commit 66ca327aa3badadc8edabbcd6440f7633fd53e1d Author: yukon Date: Mon Dec 19 17:34:19 2016 +0800 Finish code dump. Reviewed by @vongosling @stevenschew @vintagewang @lollipopjin diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..30f07f3892 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +*dependency-reduced-pom.xml +.classpath +.project +.settings/ +target/ +devenv +*.log* +*.iml +.idea/ +*.versionsBackup +*bin diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..d67d8f63bf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,5 @@ +language: java +jdk: + - oraclejdk8 + - oraclejdk7 + - openjdk6 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..a3f3baacc9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,17 @@ +## How To Contribute + +We are always very happy to have contributions, whether for trivial cleanups or big new features. +We want to have high quality, well documented codes for each programming language. These, as well as the surrounding [ecosystem](https://github.com/rocketmq) of integration tools that people use with RocketMQ, are critical aspects of the project. + +Nor is code the only way to contribute to the project. We strongly value documentation and gladly accept improvements to the documentation. + +## Contributing code + +To submit a change for inclusion, please do the following: +#### If the change is non-trivial please include some unit tests that cover the new functionality. +#### If you are introducing a completely new feature or API it is a good idea to start a wiki and get consensus on the basic design first. +#### It is our job to follow up on patches in a timely fashion. Nag us if we aren't doing our job (sometimes we drop things). + +## Becoming a Committer + +We are always interested in adding new contributors. What we look for a series of contributions, good taste, and an ongoing interest in the project. If you are interested in becoming a committer, let one of the existing committers know and they can help guide you through the process. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..715fb67f0c --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (properties) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 Alibaba Group + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..913ef3883e --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ +RocketMQ +Copyright 2016 Alibaba Group. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This distribution has producerGroup binary dependency on mysql-connector-java, which is available under gpl 2.0. +The source code of mysql-connector-java can be found at http://dev.mysql.com/downloads/connector/reconsumeTimes/. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000..9b4cee4db7 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +## RocketMQ -- Alibaba's MQ [![Build Status](https://travis-ci.org/alibaba/RocketMQ.svg?branch=master)](https://travis-ci.org/alibaba/RocketMQ) +[![Maven Central](https://img.shields.io/badge/maven--center-stable--version-green.svg)](http://search.maven.org/#search%7Cga%7C1%7Ccom.alibaba.rocketmq) +[![GitHub release](https://img.shields.io/badge/release-download-orange.svg)](https://github.com/alibaba/RocketMQ/releases) +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +**RocketMQ is a low latency, reliable, scalable, easy to use message oriented middleware born from alibaba massive messaging business.** + +It offers a variety of features as follows: + +* Pub/Sub and P2P messaging model +* Reliable FIFO and strict sequential messaging in the same queue +* Long pull queue model,also support push consumption style +* Million message accumulation ability in single queue +* Over a variety of messaging protocols.such as JMS,MQTT,HTTP etc. +* Distributed high available deploy architecture, meets at least once message delivery semantics +* Docker images for isolated testing and cloud Isolated clusters +* Feature-rich administrative dashboard for configuration,metrics and monitoring [![Cloud Charge](https://img.shields.io/badge/cloud-charge-red.svg)](https://www.aliyun.com/product/ons/?spm=5176.7960203.237031.135.67OdNO) +* Message full-link tracking [![Cloud Charge] (https://img.shields.io/badge/cloud-charge-red.svg)](https://www.aliyun.com/product/ons/?spm=5176.7960203.237031.135.67OdNO) +* Producer transaction message,making producer and local database transaction in one atomic operation [![Cloud Charge](https://img.shields.io/badge/cloud-charge-red.svg)](https://www.aliyun.com/product/ons/?spm=5176.7960203.237031.135.67OdNO) +* Message Schedule delivery,similar JMS2 spec's delivery delay [![Cloud Charge] (https://img.shields.io/badge/cloud-charge-red.svg)](https://www.aliyun.com/product/ons/?spm=5176.7960203.237031.135.67OdNO) + +---------- + +## Learn it & Contact us +* Homepage: +* Mailing list: +* Documentation: +* Wiki: +* Issues: +* Ask: or +* [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/alibaba/RocketMQ?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) + + +---------- + +## How can I develop with RocketMQ? +* [Download the latest release package](https://github.com/alibaba/RocketMQ/releases) +* [Get the latest document & Who using RocketMQ](https://github.com/alibaba/RocketMQ/issues/1) +* [`Using RocketMQ in Aliyun`](http://www.aliyun.com/product/ons) +* [`Docker images`](https://registry.hub.docker.com/u/vongosling/rocketmq/) +* [Integrate with Alibaba JStorm](https://github.com/alibaba/jstorm) +* [Integrate with Alibaba Canal](https://github.com/alibaba/canal) +* [RocketMQ Community Projects](https://github.com/rocketmq) + +---------- + +## Contributing +We are always very happy to have contributions, whether for trivial cleanups,big new features or other material rewards. more details see [here](https://github.com/alibaba/RocketMQ/wiki/How-to-donate) + +---------- +## License +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) 2010-2013 Alibaba Group Holding Limited diff --git a/benchmark/consumer.sh b/benchmark/consumer.sh new file mode 100644 index 0000000000..e516439449 --- /dev/null +++ b/benchmark/consumer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: consumer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.benchmark.Consumer $@ & diff --git a/benchmark/producer.sh b/benchmark/producer.sh new file mode 100644 index 0000000000..1462257f00 --- /dev/null +++ b/benchmark/producer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: producer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh -Dcom.alibaba.rocketmq.client.sendSmartMsg=true com.alibaba.rocketmq.example.benchmark.Producer $@ & diff --git a/benchmark/runclass.sh b/benchmark/runclass.sh new file mode 100644 index 0000000000..afe6b6ef33 --- /dev/null +++ b/benchmark/runclass.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# +# $Id: runserver.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname opts" + exit 1 +fi + +BASE_DIR=$(dirname $0)/.. +CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPermSize=320m" +JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" +JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:/dev/shm/rmq_srv_gc.log -XX:+PrintGCDetails" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" +JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages" +JAVA_OPT="${JAVA_OPT} -XX:+PerfDisableSharedMem" +#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" +JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/usr/java +fi + +JAVA="$JAVA_HOME/bin/java" + +$JAVA ${JAVA_OPT} $@ diff --git a/benchmark/tproducer.sh b/benchmark/tproducer.sh new file mode 100644 index 0000000000..f19d2c68dc --- /dev/null +++ b/benchmark/tproducer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: producer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.benchmark.TransactionProducer $@ diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml new file mode 100644 index 0000000000..ac562fac4f --- /dev/null +++ b/checkstyle/checkstyle.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/conf/2m-2s-async/broker-a-s.properties b/conf/2m-2s-async/broker-a-s.properties new file mode 100644 index 0000000000..a4401f8837 --- /dev/null +++ b/conf/2m-2s-async/broker-a-s.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-a.properties b/conf/2m-2s-async/broker-a.properties new file mode 100644 index 0000000000..6ca12f1db8 --- /dev/null +++ b/conf/2m-2s-async/broker-a.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-b-s.properties b/conf/2m-2s-async/broker-b-s.properties new file mode 100644 index 0000000000..51f8daf9a5 --- /dev/null +++ b/conf/2m-2s-async/broker-b-s.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-b.properties b/conf/2m-2s-async/broker-b.properties new file mode 100644 index 0000000000..f7f3791f0b --- /dev/null +++ b/conf/2m-2s-async/broker-b.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-a-s.properties b/conf/2m-2s-sync/broker-a-s.properties new file mode 100644 index 0000000000..a4401f8837 --- /dev/null +++ b/conf/2m-2s-sync/broker-a-s.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-a.properties b/conf/2m-2s-sync/broker-a.properties new file mode 100644 index 0000000000..135552d285 --- /dev/null +++ b/conf/2m-2s-sync/broker-a.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-b-s.properties b/conf/2m-2s-sync/broker-b-s.properties new file mode 100644 index 0000000000..51f8daf9a5 --- /dev/null +++ b/conf/2m-2s-sync/broker-b-s.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-b.properties b/conf/2m-2s-sync/broker-b.properties new file mode 100644 index 0000000000..97162a75f9 --- /dev/null +++ b/conf/2m-2s-sync/broker-b.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-noslave/broker-a.properties b/conf/2m-noslave/broker-a.properties new file mode 100644 index 0000000000..6ca12f1db8 --- /dev/null +++ b/conf/2m-noslave/broker-a.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-noslave/broker-b.properties b/conf/2m-noslave/broker-b.properties new file mode 100644 index 0000000000..f7f3791f0b --- /dev/null +++ b/conf/2m-noslave/broker-b.properties @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/broker.conf b/conf/broker.conf new file mode 100644 index 0000000000..6ca12f1db8 --- /dev/null +++ b/conf/broker.conf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/logback_broker.xml b/conf/logback_broker.xml new file mode 100644 index 0000000000..49e9d12e58 --- /dev/null +++ b/conf/logback_broker.xml @@ -0,0 +1,328 @@ + + + + + + ${user.home}/logs/rocketmqlogs/broker_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/broker_default.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/broker.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/broker.%i.log + + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/protection.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/protection.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/watermark.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/store.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/store.%i.log + + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/remoting.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/remoting.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/storeerror.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/storeerror.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/transaction.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/transaction.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/lock.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/lock.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/stats.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/stats.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/commercial.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/commercial.%i.log + + 1 + 10 + + + 500MB + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_filtersrv.xml b/conf/logback_filtersrv.xml new file mode 100644 index 0000000000..8de4e08499 --- /dev/null +++ b/conf/logback_filtersrv.xml @@ -0,0 +1,91 @@ + + + + + + ${user.home}/logs/rocketmqlogs/filtersrv_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/filtersrv_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/filtersrv.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/filtersrv.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + 0 + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,SSS} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_namesrv.xml b/conf/logback_namesrv.xml new file mode 100644 index 0000000000..7a60c7646f --- /dev/null +++ b/conf/logback_namesrv.xml @@ -0,0 +1,91 @@ + + + + + + ${user.home}/logs/rocketmqlogs/namesrv_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/namesrv_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/namesrv.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/namesrv.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + 0 + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,SSS} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_tools.xml b/conf/logback_tools.xml new file mode 100644 index 0000000000..addf211cf2 --- /dev/null +++ b/conf/logback_tools.xml @@ -0,0 +1,96 @@ + + + + + + ${user.home}/logs/rocketmqlogs/tools_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/tools_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/tools.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/tools.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy.bat b/deploy.bat new file mode 100644 index 0000000000..9cc3daebff --- /dev/null +++ b/deploy.bat @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mvn -Dmaven.test.skip=true deploy \ No newline at end of file diff --git a/eclipse.bat b/eclipse.bat new file mode 100644 index 0000000000..24e7313a01 --- /dev/null +++ b/eclipse.bat @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mvn -U eclipse:eclipse diff --git a/install.bat b/install.bat new file mode 100644 index 0000000000..550f4b081b --- /dev/null +++ b/install.bat @@ -0,0 +1,17 @@ +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +mvn -Dmaven.test.skip=true clean package install assembly:assembly -U + diff --git a/install.sh b/install.sh new file mode 100644 index 0000000000..6a536ed74d --- /dev/null +++ b/install.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git pull + +rm -rf target +rm -f devenv +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/usr/java +fi +export PATH=$JAVA_HOME/bin:$PATH +mvn clean package install assembly:assembly -U + +ln -s target/alibaba-rocketmq-broker/alibaba-rocketmq devenv diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000..87dd809278 --- /dev/null +++ b/pom.xml @@ -0,0 +1,518 @@ + + + + + + + org.apache + apache + 18 + + + 4.0.0 + + 2012 + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + pom + rocketmq-all ${project.version} + http://rocketmq.incubator.apache.org/ + + + 3.2.5 + + + + https://git-wip-us.apache.org/repos/asf/incubator-rocketmq.git + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-rocketmq.git + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-rocketmq.git + + + + + Development List + dev-subscribe@rocketmq.incubator.apache.org + dev-unsubscribe@rocketmq.incubator.apache.org + dev@rocketmq.incubator.apache.org + + + User List + users-subscribe@rocketmq.incubator.apache.org + users-unsubscribe@rocketmq.incubator.apache.org + users@rocketmq.incubator.apache.org + + + Commits List + commits-subscribe@rocketmq.incubator.apache.org + commits-unsubscribe@rocketmq.incubator.apache.org + commits@rocketmq.incubator.apache.org + + + + + + vintagewang + vintagewang + + architect + committer + + vintagewang@apache.org + https://github.com/vintagewang + 8 + + + vongosling + vongosling + + architect + committer + + vongosling@apache.org + https://github.com/vongosling + +8 + + + yukon + Xinyu Zhou + yukon@@apache.org + https://github.com/zhouxinyu + + committer + + +8 + + + stevenschew + Wei Zhou + stevenschew@@apache.org + https://github.com/zhouwei198732 + + committer + + +8 + + + lollipop + Jixiang Jin + lollipop@apache.org + https://github.com/jinjixiang + + committer + + +8 + + + lizhanhui + Zhanhui Li + lizhanhui@apache.org + https://github.com/lizhanhui + + committer + + +8 + + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + Apache Software Foundation + http://www.apache.org + + + + jira + https://issues.apache.org/jira/browse/RocketMQ + + + + UTF-8 + + false + true + + 1.7 + 1.7 + + + + rocketmq-client + rocketmq-common + rocketmq-broker + rocketmq-tools + rocketmq-store + rocketmq-namesrv + rocketmq-remoting + rocketmq-example + rocketmq-filtersrv + rocketmq-srvutil + + + + + + org.codehaus.mojo + versions-maven-plugin + 2.2 + + + com.github.vongosling + dependency-mediator-maven-plugin + 1.0.2 + + + org.codehaus.mojo + clirr-maven-plugin + 2.7 + + + maven-enforcer-plugin + 1.4.1 + + + enforce-ban-circular-dependencies + + enforce + + + + + + + + true + + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-4 + + + + + maven-compiler-plugin + 3.5.1 + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.source} + true + true + + + + maven-surefire-plugin + 2.19.1 + + -Xms512m -Xmx1024m + + **/*Test.java + + + com/alibaba/rocketmq/remoting/ExceptionTest.java + com/alibaba/rocketmq/remoting/SyncInvokeTest.java + com/alibaba/rocketmq/remoting/NettyIdleTest.java + com/alibaba/rocketmq/remoting/NettyConnectionTest.java + com/alibaba/rocketmq/common/filter/PolishExprTest.java + com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java + com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java + com/alibaba/rocketmq/store/RecoverTest.java + com/alibaba/rocketmq/broker/api/SendMessageTest.java + com/alibaba/rocketmq/test/integration/*/*.java + com/alibaba/rocketmq/test/integration/BaseTest.java + com/alibaba/rocketmq/test/*/*.java + com/alibaba/rocketmq/test/BaseTest.java + + + + + maven-assembly-plugin + 2.6 + + alibaba-rocketmq + + release.xml + + + + + + + + maven-javadoc-plugin + 2.10.4 + + UTF-8 + + + + attach-javadocs + + jar + + + + + + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + + + maven-help-plugin + 2.2 + + + generate-effective-dependencies-pom + generate-resources + + effective-pom + + + ${project.build.directory}/effective-pom/effective-dependencies.xml + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + verify + verify + + checkstyle/checkstyle.xml + UTF-8 + true + true + + + check + + + + + + maven-resources-plugin + 3.0.2 + + + + src/main/resources + false + + + ${project.build.sourceEncoding} + + + + + + + + jdk8 + + [1.8,) + + + + + + maven-javadoc-plugin + 2.10.4 + + -Xdoclint:none + + + + + + + + maven-javadoc-plugin + 2.10.4 + + -Xdoclint:none + + + + + + + release-sign-artifacts + + + performRelease + true + + + + + + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + + + + + + + ${project.groupId} + rocketmq-client + ${project.version} + + + ${project.groupId} + rocketmq-broker + ${project.version} + + + ${project.groupId} + rocketmq-common + ${project.version} + + + ${project.groupId} + rocketmq-store + ${project.version} + + + ${project.groupId} + rocketmq-namesrv + ${project.version} + + + ${project.groupId} + rocketmq-tools + ${project.version} + + + ${project.groupId} + rocketmq-remoting + ${project.version} + + + ${project.groupId} + rocketmq-qatest + ${project.version} + + + ${project.groupId} + rocketmq-filtersrv + ${project.version} + + + ${project.groupId} + rocketmq-srvutil + ${project.version} + + + junit + junit + 4.11 + test + + + org.slf4j + slf4j-api + 1.7.5 + + + ch.qos.logback + logback-classic + 1.0.13 + + + ch.qos.logback + logback-core + 1.0.13 + + + commons-cli + commons-cli + 1.2 + + + io.netty + netty-all + 4.0.36.Final + + + com.alibaba + fastjson + 1.2.12 + + + org.javassist + javassist + 3.20.0-GA + + + org.apache.commons + commons-lang3 + 3.4 + + + net.java.dev.jna + jna + 4.2.2 + + + + \ No newline at end of file diff --git a/release-client.xml b/release-client.xml new file mode 100644 index 0000000000..12e065de2f --- /dev/null +++ b/release-client.xml @@ -0,0 +1,52 @@ + + + + + client + alibaba-rocketmq-client-java + + dir + tar.gz + + + + + LICENSE + NOTICE + + + + + + + + + com.alibaba.rocketmq:rocketmq-client + + + ./ + false + + + ./ + + + + + + diff --git a/release.xml b/release.xml new file mode 100644 index 0000000000..f304e7450e --- /dev/null +++ b/release.xml @@ -0,0 +1,60 @@ + + + + + broker + alibaba-rocketmq + + dir + tar.gz + + + + + bin/* + conf/* + benchmark/* + LICENSE + NOTICE + + + + + + + + + com.alibaba.rocketmq:rocketmq-broker + com.alibaba.rocketmq:rocketmq-tools + com.alibaba.rocketmq:rocketmq-client + com.alibaba.rocketmq:rocketmq-namesrv + com.alibaba.rocketmq:rocketmq-filtersrv + com.alibaba.rocketmq:rocketmq-example + + + lib/ + false + + + lib/ + + + + + + diff --git a/rocketmq-broker/pom.xml b/rocketmq-broker/pom.xml new file mode 100644 index 0000000000..fbbf0ee0d2 --- /dev/null +++ b/rocketmq-broker/pom.xml @@ -0,0 +1,79 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-broker + rocketmq-broker ${project.version} + + + + junit + junit + test + + + + + + + + ${project.groupId} + rocketmq-common + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-remoting + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + com.alibaba + fastjson + + + org.javassist + javassist + + + diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java new file mode 100644 index 0000000000..b2b6aedefd --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java @@ -0,0 +1,773 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker; + +import com.alibaba.rocketmq.broker.client.*; +import com.alibaba.rocketmq.broker.client.net.Broker2Client; +import com.alibaba.rocketmq.broker.client.rebalance.RebalanceLockManager; +import com.alibaba.rocketmq.broker.filtersrv.FilterServerManager; +import com.alibaba.rocketmq.broker.latency.BrokerFastFailure; +import com.alibaba.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; +import com.alibaba.rocketmq.broker.longpolling.NotifyMessageArrivingListener; +import com.alibaba.rocketmq.broker.longpolling.PullRequestHoldService; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageHook; +import com.alibaba.rocketmq.broker.offset.ConsumerOffsetManager; +import com.alibaba.rocketmq.broker.out.BrokerOuterAPI; +import com.alibaba.rocketmq.broker.plugin.MessageStoreFactory; +import com.alibaba.rocketmq.broker.plugin.MessageStorePluginContext; +import com.alibaba.rocketmq.broker.processor.*; +import com.alibaba.rocketmq.broker.slave.SlaveSynchronize; +import com.alibaba.rocketmq.broker.subscription.SubscriptionGroupManager; +import com.alibaba.rocketmq.broker.topic.TopicConfigManager; +import com.alibaba.rocketmq.common.*; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.stats.MomentStatsItem; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.*; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.MessageArrivingListener; +import com.alibaba.rocketmq.store.MessageStore; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.stats.BrokerStats; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + + +/** + * @author shijia.wxr + */ +public class BrokerController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); + private static final Logger LOG_WATER_MARK = LoggerFactory.getLogger(LoggerName.WATER_MARK_LOGGER_NAME); + private final BrokerConfig brokerConfig; + private final NettyServerConfig nettyServerConfig; + private final NettyClientConfig nettyClientConfig; + private final MessageStoreConfig messageStoreConfig; + private final ConsumerOffsetManager consumerOffsetManager; + private final ConsumerManager consumerManager; + private final ProducerManager producerManager; + private final ClientHousekeepingService clientHousekeepingService; + private final PullMessageProcessor pullMessageProcessor; + private final PullRequestHoldService pullRequestHoldService; + private final MessageArrivingListener messageArrivingListener; + private final Broker2Client broker2Client; + private final SubscriptionGroupManager subscriptionGroupManager; + private final ConsumerIdsChangeListener consumerIdsChangeListener; + private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager(); + private final BrokerOuterAPI brokerOuterAPI; + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "BrokerControllerScheduledThread")); + private final SlaveSynchronize slaveSynchronize; + private final BlockingQueue sendThreadPoolQueue; + private final BlockingQueue pullThreadPoolQueue; + private final BlockingQueue clientManagerThreadPoolQueue; + private final BlockingQueue consumerManagerThreadPoolQueue; + private final FilterServerManager filterServerManager; + private final BrokerStatsManager brokerStatsManager; + private final List sendMessageHookList = new ArrayList(); + private final List consumeMessageHookList = new ArrayList(); + private MessageStore messageStore; + private RemotingServer remotingServer; + private RemotingServer fastRemotingServer; + private TopicConfigManager topicConfigManager; + private ExecutorService sendMessageExecutor; + private ExecutorService pullMessageExecutor; + private ExecutorService adminBrokerExecutor; + private ExecutorService clientManageExecutor; + private ExecutorService consumerManageExecutor; + private boolean updateMasterHAServerAddrPeriodically = false; + private BrokerStats brokerStats; + private InetSocketAddress storeHost; + private BrokerFastFailure brokerFastFailure; + private Configuration configuration; + + public BrokerController(// + final BrokerConfig brokerConfig, // + final NettyServerConfig nettyServerConfig, // + final NettyClientConfig nettyClientConfig, // + final MessageStoreConfig messageStoreConfig // + ) { + this.brokerConfig = brokerConfig; + this.nettyServerConfig = nettyServerConfig; + this.nettyClientConfig = nettyClientConfig; + this.messageStoreConfig = messageStoreConfig; + this.consumerOffsetManager = new ConsumerOffsetManager(this); + this.topicConfigManager = new TopicConfigManager(this); + this.pullMessageProcessor = new PullMessageProcessor(this); + this.pullRequestHoldService = new PullRequestHoldService(this); + this.messageArrivingListener = new NotifyMessageArrivingListener(this.pullRequestHoldService); + this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this); + this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener); + this.producerManager = new ProducerManager(); + this.clientHousekeepingService = new ClientHousekeepingService(this); + this.broker2Client = new Broker2Client(this); + this.subscriptionGroupManager = new SubscriptionGroupManager(this); + this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig); + this.filterServerManager = new FilterServerManager(this); + + if (this.brokerConfig.getNamesrvAddr() != null) { + this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr()); + log.info("user specfied name server address: {}", this.brokerConfig.getNamesrvAddr()); + } + + this.slaveSynchronize = new SlaveSynchronize(this); + + this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); + + this.pullThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity()); + this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity()); + this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity()); + + this.brokerStatsManager = new BrokerStatsManager(this.brokerConfig.getBrokerClusterName()); + this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), this.getNettyServerConfig().getListenPort())); + + this.brokerFastFailure = new BrokerFastFailure(this); + this.configuration = new Configuration( + log, + BrokerPathConfigHelper.getBrokerConfigPath(), + this.brokerConfig, this.nettyServerConfig, this.nettyClientConfig, this.messageStoreConfig + ); + } + + public BrokerConfig getBrokerConfig() { + return brokerConfig; + } + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + public BlockingQueue getPullThreadPoolQueue() { + return pullThreadPoolQueue; + } + + public boolean initialize() throws CloneNotSupportedException { + boolean result = true; + + result = result && this.topicConfigManager.load(); + + result = result && this.consumerOffsetManager.load(); + result = result && this.subscriptionGroupManager.load(); + + if (result) { + try { + this.messageStore = + new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, + this.brokerConfig); + this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); + //load plugin + MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig); + this.messageStore = MessageStoreFactory.build(context, this.messageStore); + } catch (IOException e) { + result = false; + e.printStackTrace(); + } + } + + result = result && this.messageStore.load(); + + if (result) { + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService); + NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone(); + fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2); + this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService); + this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getSendMessageThreadPoolNums(), + this.brokerConfig.getSendMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.sendThreadPoolQueue, + new ThreadFactoryImpl("SendMessageThread_")); + + this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getPullMessageThreadPoolNums(), + this.brokerConfig.getPullMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.pullThreadPoolQueue, + new ThreadFactoryImpl("PullMessageThread_")); + + this.adminBrokerExecutor = + Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl( + "AdminBrokerThread_")); + + this.clientManageExecutor = new ThreadPoolExecutor( + this.brokerConfig.getClientManageThreadPoolNums(), + this.brokerConfig.getClientManageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.clientManagerThreadPoolQueue, + new ThreadFactoryImpl("ClientManageThread_")); + + this.consumerManageExecutor = + Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl( + "ConsumerManageThread_")); + + this.registerProcessor(); + + + // TODO remove in future + final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis(); + final long period = 1000 * 60 * 60 * 24; + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.getBrokerStats().record(); + } catch (Throwable e) { + log.error("schedule record error.", e); + } + } + }, initialDelay, period, TimeUnit.MILLISECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.consumerOffsetManager.persist(); + } catch (Throwable e) { + log.error("schedule persist consumerOffset error.", e); + } + } + }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.protectBroker(); + } catch (Exception e) { + log.error("protectBroker error.", e); + } + } + }, 3, 3, TimeUnit.MINUTES); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.printWaterMark(); + } catch (Exception e) { + log.error("printWaterMark error.", e); + } + } + }, 10, 1, TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes()); + } catch (Throwable e) { + log.error("schedule dispatchBehindBytes error.", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + + if (this.brokerConfig.getNamesrvAddr() != null) { + this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr()); + } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.brokerOuterAPI.fetchNameServerAddr(); + } catch (Throwable e) { + log.error("ScheduledTask fetchNameServerAddr exception", e); + } + } + }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + } + + if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { + if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) { + this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress()); + this.updateMasterHAServerAddrPeriodically = false; + } else { + this.updateMasterHAServerAddrPeriodically = true; + } + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.slaveSynchronize.syncAll(); + } catch (Throwable e) { + log.error("ScheduledTask syncAll slave exception", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } else { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.printMasterAndSlaveDiff(); + } catch (Throwable e) { + log.error("schedule printMasterAndSlaveDiff error.", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } + } + + return result; + } + + public void registerProcessor() { + /** + * SendMessageProcessor + */ + SendMessageProcessor sendProcessor = new SendMessageProcessor(this); + sendProcessor.registerSendMessageHook(sendMessageHookList); + sendProcessor.registerConsumeMessageHook(consumeMessageHookList); + + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, this.sendMessageExecutor); + /** + * PullMessageProcessor + */ + this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor); + this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + + /** + * QueryMessageProcessor + */ + NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this); + this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.pullMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.pullMessageExecutor); + + this.fastRemotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, this.pullMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, this.pullMessageExecutor); + + /** + * ClientManageProcessor + */ + ClientManageProcessor clientProcessor = new ClientManageProcessor(this); + this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); + + this.fastRemotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, this.clientManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, this.clientManageExecutor); + + /** + * ConsumerManageProcessor + */ + ConsumerManageProcessor consumerManageProcessor = new ConsumerManageProcessor(this); + this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor); + this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); + this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); + + this.fastRemotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, consumerManageProcessor, this.consumerManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, consumerManageProcessor, this.consumerManageExecutor); + + + /** + * EndTransactionProcessor + */ + this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.sendMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.sendMessageExecutor); + + /** + * Default + */ + AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this); + this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor); + this.fastRemotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor); + } + + public BrokerStats getBrokerStats() { + return brokerStats; + } + + public void setBrokerStats(BrokerStats brokerStats) { + this.brokerStats = brokerStats; + } + + public void protectBroker() { + if (this.brokerConfig.isDisableConsumeIfConsumerReadSlowly()) { + final Iterator> it = this.brokerStatsManager.getMomentStatsItemSetFallSize().getStatsItemTable().entrySet().iterator(); + while (it.hasNext()) { + final Map.Entry next = it.next(); + final long fallBehindBytes = next.getValue().getValue().get(); + if (fallBehindBytes > this.brokerConfig.getConsumerFallbehindThreshold()) { + final String[] split = next.getValue().getStatsKey().split("@"); + final String group = split[2]; + LOG_PROTECTION.info("[PROTECT_BROKER] the consumer[{}] consume slowly, {} bytes, disable it", group, fallBehindBytes); + this.subscriptionGroupManager.disableConsume(group); + } + } + } + } + + public long headSlowTimeMills(BlockingQueue q) { + long slowTimeMills = 0; + final Runnable peek = q.peek(); + if (peek != null) { + RequestTask rt = BrokerFastFailure.castRunnable(peek); + slowTimeMills = this.messageStore.now() - rt.getCreateTimestamp(); + } + + if (slowTimeMills < 0) slowTimeMills = 0; + + return slowTimeMills; + } + + public long headSlowTimeMills4SendThreadPoolQueue() { + return this.headSlowTimeMills(this.sendThreadPoolQueue); + } + + public long headSlowTimeMills4PullThreadPoolQueue() { + return this.headSlowTimeMills(this.pullThreadPoolQueue); + } + + public void printWaterMark() { + LOG_WATER_MARK.info("[WATERMARK] Send Queue Size: {} SlowTimeMills: {}", this.sendThreadPoolQueue.size(), headSlowTimeMills4SendThreadPoolQueue()); + LOG_WATER_MARK.info("[WATERMARK] Pull Queue Size: {} SlowTimeMills: {}", this.pullThreadPoolQueue.size(), headSlowTimeMills4PullThreadPoolQueue()); + } + + public MessageStore getMessageStore() { + return messageStore; + } + + public void setMessageStore(MessageStore messageStore) { + this.messageStore = messageStore; + } + + private void printMasterAndSlaveDiff() { + long diff = this.messageStore.slaveFallBehindMuch(); + + // XXX: warn and notify me + log.info("slave fall behind master, how much, {} bytes", diff); + } + + public Broker2Client getBroker2Client() { + return broker2Client; + } + + public ConsumerManager getConsumerManager() { + return consumerManager; + } + + public ConsumerOffsetManager getConsumerOffsetManager() { + return consumerOffsetManager; + } + + public MessageStoreConfig getMessageStoreConfig() { + return messageStoreConfig; + } + + public ProducerManager getProducerManager() { + return producerManager; + } + + public void setFastRemotingServer(RemotingServer fastRemotingServer) { + this.fastRemotingServer = fastRemotingServer; + } + + public PullMessageProcessor getPullMessageProcessor() { + return pullMessageProcessor; + } + + public PullRequestHoldService getPullRequestHoldService() { + return pullRequestHoldService; + } + + public SubscriptionGroupManager getSubscriptionGroupManager() { + return subscriptionGroupManager; + } + + public void shutdown() { + if (this.brokerStatsManager != null) { + this.brokerStatsManager.shutdown(); + } + + if (this.clientHousekeepingService != null) { + this.clientHousekeepingService.shutdown(); + } + + if (this.pullRequestHoldService != null) { + this.pullRequestHoldService.shutdown(); + } + + if (this.remotingServer != null) { + this.remotingServer.shutdown(); + } + + if (this.fastRemotingServer != null) { + this.fastRemotingServer.shutdown(); + } + + if (this.messageStore != null) { + this.messageStore.shutdown(); + } + + this.scheduledExecutorService.shutdown(); + try { + this.scheduledExecutorService.awaitTermination(5000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + } + + this.unregisterBrokerAll(); + + if (this.sendMessageExecutor != null) { + this.sendMessageExecutor.shutdown(); + } + + if (this.pullMessageExecutor != null) { + this.pullMessageExecutor.shutdown(); + } + + if (this.adminBrokerExecutor != null) { + this.adminBrokerExecutor.shutdown(); + } + + if (this.brokerOuterAPI != null) { + this.brokerOuterAPI.shutdown(); + } + + this.consumerOffsetManager.persist(); + + if (this.filterServerManager != null) { + this.filterServerManager.shutdown(); + } + + if (this.brokerFastFailure != null) { + this.brokerFastFailure.shutdown(); + } + } + + private void unregisterBrokerAll() { + this.brokerOuterAPI.unregisterBrokerAll( + this.brokerConfig.getBrokerClusterName(), + this.getBrokerAddr(), + this.brokerConfig.getBrokerName(), + this.brokerConfig.getBrokerId()); + } + + public String getBrokerAddr() { + return this.brokerConfig.getBrokerIP1() + ":" + this.nettyServerConfig.getListenPort(); + } + + public void start() throws Exception { + if (this.messageStore != null) { + this.messageStore.start(); + } + + if (this.remotingServer != null) { + this.remotingServer.start(); + } + + if (this.fastRemotingServer != null) { + this.fastRemotingServer.start(); + } + + if (this.brokerOuterAPI != null) { + this.brokerOuterAPI.start(); + } + + if (this.pullRequestHoldService != null) { + this.pullRequestHoldService.start(); + } + + if (this.clientHousekeepingService != null) { + this.clientHousekeepingService.start(); + } + + if (this.filterServerManager != null) { + this.filterServerManager.start(); + } + + this.registerBrokerAll(true, false); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.registerBrokerAll(true, false); + } catch (Throwable e) { + log.error("registerBrokerAll Exception", e); + } + } + }, 1000 * 10, 1000 * 30, TimeUnit.MILLISECONDS); + + if (this.brokerStatsManager != null) { + this.brokerStatsManager.start(); + } + + if (this.brokerFastFailure != null) { + this.brokerFastFailure.start(); + } + } + + public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway) { + TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); + + if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) + || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { + ConcurrentHashMap topicConfigTable = new ConcurrentHashMap(); + for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { + TopicConfig tmp = + new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), + this.brokerConfig.getBrokerPermission()); + topicConfigTable.put(topicConfig.getTopicName(), tmp); + } + topicConfigWrapper.setTopicConfigTable(topicConfigTable); + } + + RegisterBrokerResult registerBrokerResult = this.brokerOuterAPI.registerBrokerAll( + this.brokerConfig.getBrokerClusterName(), + this.getBrokerAddr(), + this.brokerConfig.getBrokerName(), + this.brokerConfig.getBrokerId(), + this.getHAServerAddr(), + topicConfigWrapper, + this.filterServerManager.buildNewFilterServerList(), + oneway, + this.brokerConfig.getRegisterBrokerTimeoutMills()); + + if (registerBrokerResult != null) { + if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { + this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); + } + + this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); + + if (checkOrderConfig) { + this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); + } + } + } + + public TopicConfigManager getTopicConfigManager() { + return topicConfigManager; + } + + public void setTopicConfigManager(TopicConfigManager topicConfigManager) { + this.topicConfigManager = topicConfigManager; + } + + public String getHAServerAddr() { + return this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort(); + } + + public RebalanceLockManager getRebalanceLockManager() { + return rebalanceLockManager; + } + + public SlaveSynchronize getSlaveSynchronize() { + return slaveSynchronize; + } + + public ExecutorService getPullMessageExecutor() { + return pullMessageExecutor; + } + + public void setPullMessageExecutor(ExecutorService pullMessageExecutor) { + this.pullMessageExecutor = pullMessageExecutor; + } + + public BlockingQueue getSendThreadPoolQueue() { + return sendThreadPoolQueue; + } + + public FilterServerManager getFilterServerManager() { + return filterServerManager; + } + + public BrokerStatsManager getBrokerStatsManager() { + return brokerStatsManager; + } + + public List getSendMessageHookList() { + return sendMessageHookList; + } + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register SendMessageHook Hook, {}", hook.hookName()); + } + + public List getConsumeMessageHookList() { + return consumeMessageHookList; + } + + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register ConsumeMessageHook Hook, {}", hook.hookName()); + } + + public void registerServerRPCHook(RPCHook rpcHook) { + getRemotingServer().registerRPCHook(rpcHook); + } + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } + + public void registerClientRPCHook(RPCHook rpcHook) { + this.getBrokerOuterAPI().registerRPCHook(rpcHook); + } + + public BrokerOuterAPI getBrokerOuterAPI() { + return brokerOuterAPI; + } + + public InetSocketAddress getStoreHost() { + return storeHost; + } + + public void setStoreHost(InetSocketAddress storeHost) { + this.storeHost = storeHost; + } + + public Configuration getConfiguration() { + return this.configuration; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java new file mode 100644 index 0000000000..055e8dc140 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker; + +import java.io.File; + + +public class BrokerPathConfigHelper { + private static String brokerConfigPath = System.getProperty("user.home") + File.separator + "store" + + File.separator + "config" + File.separator + "broker.properties"; + + + public static String getBrokerConfigPath() { + return brokerConfigPath; + } + + + public static void setBrokerConfigPath(String path) { + brokerConfigPath = path; + } + + + public static String getTopicConfigPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "topics.json"; + } + + + public static String getConsumerOffsetPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "consumerOffset.json"; + } + + + public static String getSubscriptionGroupPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json"; + } + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java new file mode 100644 index 0000000000..7e81117fa2 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class BrokerStartup { + public static Properties properties = null; + public static CommandLine commandLine = null; + public static String configFile = null; + public static Logger log; + + public static void main(String[] args) { + start(createBrokerController(args)); + } + + public static BrokerController start(BrokerController controller) { + try { + controller.start(); + String tip = "The broker[" + controller.getBrokerConfig().getBrokerName() + ", " + + controller.getBrokerAddr() + "] boot success. serializeType=" + RemotingCommand.getSerializeTypeConfigInThisServer(); + + if (null != controller.getBrokerConfig().getNamesrvAddr()) { + tip += " and name server is " + controller.getBrokerConfig().getNamesrvAddr(); + } + + log.info(tip); + return controller; + } catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } + + public static BrokerController createBrokerController(String[] args) { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE)) { + NettySystemConfig.socketSndbufSize = 131072; + } + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE)) { + NettySystemConfig.socketRcvbufSize = 131072; + } + + try { + //PackageConflictDetect.detectFastjson(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + commandLine = ServerUtil.parseCmdLine("mqbroker", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + } + + final BrokerConfig brokerConfig = new BrokerConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + final NettyClientConfig nettyClientConfig = new NettyClientConfig(); + nettyServerConfig.setListenPort(10911); + final MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + + if (BrokerRole.SLAVE == messageStoreConfig.getBrokerRole()) { + int ratio = messageStoreConfig.getAccessMessageInMemoryMaxRatio() - 10; + messageStoreConfig.setAccessMessageInMemoryMaxRatio(ratio); + } + + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, brokerConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + MixAll.printObjectProperties(null, nettyClientConfig); + MixAll.printObjectProperties(null, messageStoreConfig); + System.exit(0); + } else if (commandLine.hasOption('m')) { + MixAll.printObjectProperties(null, brokerConfig, true); + MixAll.printObjectProperties(null, nettyServerConfig, true); + MixAll.printObjectProperties(null, nettyClientConfig, true); + MixAll.printObjectProperties(null, messageStoreConfig, true); + System.exit(0); + } + + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + configFile = file; + InputStream in = new BufferedInputStream(new FileInputStream(file)); + properties = new Properties(); + properties.load(in); + + parsePropertie2SystemEnv(properties); + MixAll.properties2Object(properties, brokerConfig); + MixAll.properties2Object(properties, nettyServerConfig); + MixAll.properties2Object(properties, nettyClientConfig); + MixAll.properties2Object(properties, messageStoreConfig); + + BrokerPathConfigHelper.setBrokerConfigPath(file); + in.close(); + } + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig); + + if (null == brokerConfig.getRocketmqHome()) { + System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation"); + System.exit(-2); + } + + String namesrvAddr = brokerConfig.getNamesrvAddr(); + if (null != namesrvAddr) { + try { + String[] addrArray = namesrvAddr.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + RemotingUtil.string2SocketAddress(addr); + } + } + } catch (Exception e) { + System.out.printf( + "The Name Server Address[%s] illegal, please set it as follows, \"127.0.0.1:9876;192.168.0.1:9876\"%n", + namesrvAddr); + System.exit(-3); + } + } + + + switch (messageStoreConfig.getBrokerRole()) { + case ASYNC_MASTER: + case SYNC_MASTER: + brokerConfig.setBrokerId(MixAll.MASTER_ID); + break; + case SLAVE: + if (brokerConfig.getBrokerId() <= 0) { + System.out.printf("Slave's brokerId must be > 0"); + System.exit(-3); + } + + break; + default: + break; + } + + messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1); + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml"); + log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + MixAll.printObjectProperties(log, brokerConfig); + MixAll.printObjectProperties(log, nettyServerConfig); + MixAll.printObjectProperties(log, nettyClientConfig); + MixAll.printObjectProperties(log, messageStoreConfig); + + final BrokerController controller = new BrokerController(// + brokerConfig, // + nettyServerConfig, // + nettyClientConfig, // + messageStoreConfig); + // remember all configs to prevent discard + controller.getConfiguration().registerConfig(properties); + + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + return controller; + } catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } + + private static void parsePropertie2SystemEnv(Properties properties) { + if (properties == null) { + return; + } + String rmqAddressServerDomain = properties.getProperty("rmqAddressServerDomain", "jmenv.tbsite.net"); + String rmqAddressServerSubGroup = properties.getProperty("rmqAddressServerSubGroup", "nsaddr"); + System.setProperty("rocketmq.namesrv.domain", rmqAddressServerDomain); + System.setProperty("rocketmq.namesrv.domain.subgroup", rmqAddressServerSubGroup); + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Broker config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "printImportantConfig", false, "Print important config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java new file mode 100644 index 0000000000..babf4b7eb5 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; +import io.netty.channel.Channel; + + +/** + * @author shijia.wxr + */ +public class ClientChannelInfo { + private final Channel channel; + private final String clientId; + private final LanguageCode language; + private final int version; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public ClientChannelInfo(Channel channel) { + this(channel, null, null, 0); + } + + + public ClientChannelInfo(Channel channel, String clientId, LanguageCode language, int version) { + this.channel = channel; + this.clientId = clientId; + this.language = language; + this.version = version; + } + + + public Channel getChannel() { + return channel; + } + + + public String getClientId() { + return clientId; + } + + + public LanguageCode getLanguage() { + return language; + } + + + public int getVersion() { + return version; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((channel == null) ? 0 : channel.hashCode()); + result = prime * result + ((clientId == null) ? 0 : clientId.hashCode()); + result = prime * result + ((language == null) ? 0 : language.hashCode()); + result = prime * result + (int) (lastUpdateTimestamp ^ (lastUpdateTimestamp >>> 32)); + result = prime * result + version; + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ClientChannelInfo other = (ClientChannelInfo) obj; + if (channel == null) { + if (other.channel != null) + return false; + } else if (this.channel != other.channel) { + return false; + } + + return true; + } + + + @Override + public String toString() { + return "ClientChannelInfo [channel=" + channel + ", clientId=" + clientId + ", language=" + language + + ", version=" + version + ", lastUpdateTimestamp=" + lastUpdateTimestamp + "]"; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java new file mode 100644 index 0000000000..4ac7532454 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class ClientHousekeepingService implements ChannelEventListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + + private ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ClientHousekeepingScheduledThread")); + + + public ClientHousekeepingService(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public void start() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + ClientHousekeepingService.this.scanExceptionChannel(); + } catch (Exception e) { + log.error("", e); + } + } + }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS); + } + + private void scanExceptionChannel() { + this.brokerController.getProducerManager().scanNotActiveChannel(); + this.brokerController.getConsumerManager().scanNotActiveChannel(); + this.brokerController.getFilterServerManager().scanNotActiveChannel(); + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + + } + + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } + + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } + + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java new file mode 100644 index 0000000000..410b703fa8 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java @@ -0,0 +1,273 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerGroupInfo { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final String groupName; + private final ConcurrentHashMap subscriptionTable = + new ConcurrentHashMap(); + private final ConcurrentHashMap channelInfoTable = + new ConcurrentHashMap(16); + private volatile ConsumeType consumeType; + private volatile MessageModel messageModel; + private volatile ConsumeFromWhere consumeFromWhere; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public ConsumerGroupInfo(String groupName, ConsumeType consumeType, MessageModel messageModel, + ConsumeFromWhere consumeFromWhere) { + this.groupName = groupName; + this.consumeType = consumeType; + this.messageModel = messageModel; + this.consumeFromWhere = consumeFromWhere; + } + + + public ClientChannelInfo findChannel(final String clientId) { + Iterator> it = this.channelInfoTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getValue().getClientId().equals(clientId)) { + return next.getValue(); + } + } + + return null; + } + + + public ConcurrentHashMap getSubscriptionTable() { + return subscriptionTable; + } + + + public ConcurrentHashMap getChannelInfoTable() { + return channelInfoTable; + } + + + public List getAllChannel() { + List result = new ArrayList(); + + result.addAll(this.channelInfoTable.keySet()); + + return result; + } + + + public List getAllClientId() { + List result = new ArrayList(); + + Iterator> it = this.channelInfoTable.entrySet().iterator(); + + while (it.hasNext()) { + Entry entry = it.next(); + ClientChannelInfo clientChannelInfo = entry.getValue(); + result.add(clientChannelInfo.getClientId()); + } + + return result; + } + + + public void unregisterChannel(final ClientChannelInfo clientChannelInfo) { + ClientChannelInfo old = this.channelInfoTable.remove(clientChannelInfo.getChannel()); + if (old != null) { + log.info("unregister a consumer[{}] from consumerGroupInfo {}", this.groupName, old.toString()); + } + } + + + public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) { + final ClientChannelInfo info = this.channelInfoTable.remove(channel); + if (info != null) { + log.warn( + "NETTY EVENT: remove not active channel[{}] from ConsumerGroupInfo groupChannelTable, consumer group: {}", + info.toString(), groupName); + return true; + } + + return false; + } + + public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType, + MessageModel messageModel, ConsumeFromWhere consumeFromWhere) { + boolean updated = false; + this.consumeType = consumeType; + this.messageModel = messageModel; + this.consumeFromWhere = consumeFromWhere; + + ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel()); + if (null == infoOld) { + ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew); + if (null == prev) { + log.info("new consumer connected, group: {} {} {} channel: {}", this.groupName, consumeType, + messageModel, infoNew.toString()); + updated = true; + } + + infoOld = infoNew; + } else { + if (!infoOld.getClientId().equals(infoNew.getClientId())) { + log.error("[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ", + this.groupName, + infoOld.toString(), + infoNew.toString()); + this.channelInfoTable.put(infoNew.getChannel(), infoNew); + } + } + + this.lastUpdateTimestamp = System.currentTimeMillis(); + infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp); + + return updated; + } + + + public boolean updateSubscription(final Set subList) { + boolean updated = false; + + for (SubscriptionData sub : subList) { + SubscriptionData old = this.subscriptionTable.get(sub.getTopic()); + if (old == null) { + SubscriptionData prev = this.subscriptionTable.putIfAbsent(sub.getTopic(), sub); + if (null == prev) { + updated = true; + log.info("subscription changed, add new topic, group: {} {}", + this.groupName, + sub.toString()); + } + } else if (sub.getSubVersion() > old.getSubVersion()) { + if (this.consumeType == ConsumeType.CONSUME_PASSIVELY) { + log.info("subscription changed, group: {} OLD: {} NEW: {}", + this.groupName, + old.toString(), + sub.toString() + ); + } + + this.subscriptionTable.put(sub.getTopic(), sub); + } + } + + + Iterator> it = this.subscriptionTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String oldTopic = next.getKey(); + + boolean exist = false; + for (SubscriptionData sub : subList) { + if (sub.getTopic().equals(oldTopic)) { + exist = true; + break; + } + } + + if (!exist) { + log.warn("subscription changed, group: {} remove topic {} {}", + this.groupName, + oldTopic, + next.getValue().toString() + ); + + it.remove(); + updated = true; + } + } + + this.lastUpdateTimestamp = System.currentTimeMillis(); + + return updated; + } + + + public Set getSubscribeTopics() { + return subscriptionTable.keySet(); + } + + + public SubscriptionData findSubscriptionData(final String topic) { + return this.subscriptionTable.get(topic); + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public String getGroupName() { + return groupName; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java new file mode 100644 index 0000000000..4da2eb3dc3 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public interface ConsumerIdsChangeListener { + public void consumerIdsChanged(final String group, final List channels); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java new file mode 100644 index 0000000000..48e967371e --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; + private final ConcurrentHashMap consumerTable = + new ConcurrentHashMap(1024); + private final ConsumerIdsChangeListener consumerIdsChangeListener; + + public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener) { + this.consumerIdsChangeListener = consumerIdsChangeListener; + } + + public ClientChannelInfo findChannel(final String group, final String clientId) { + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.findChannel(clientId); + } + return null; + } + + public SubscriptionData findSubscriptionData(final String group, final String topic) { + ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.findSubscriptionData(topic); + } + + return null; + } + + public ConsumerGroupInfo getConsumerGroupInfo(final String group) { + return this.consumerTable.get(group); + } + + public int findSubscriptionDataCount(final String group) { + ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.getSubscriptionTable().size(); + } + + return 0; + } + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ConsumerGroupInfo info = next.getValue(); + boolean removed = info.doChannelCloseEvent(remoteAddr, channel); + if (removed) { + if (info.getChannelInfoTable().isEmpty()) { + ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey()); + if (remove != null) { + log.info("unregister consumer ok, no any connection, and remove consumer group, {}", + next.getKey()); + } + } + + this.consumerIdsChangeListener.consumerIdsChanged(next.getKey(), info.getAllChannel()); + } + } + } + + public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, + final Set subList, boolean isNotifyConsumerIdsChangedEnable) { + + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (null == consumerGroupInfo) { + ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere); + ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp); + consumerGroupInfo = prev != null ? prev : tmp; + } + + boolean r1 = + consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, + consumeFromWhere); + boolean r2 = consumerGroupInfo.updateSubscription(subList); + + if (r1 || r2) { + if (isNotifyConsumerIdsChangedEnable) { + this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel()); + } + } + + return r1 || r2; + } + + public void unregisterConsumer(final String group, final ClientChannelInfo clientChannelInfo, boolean isNotifyConsumerIdsChangedEnable) { + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (null != consumerGroupInfo) { + consumerGroupInfo.unregisterChannel(clientChannelInfo); + if (consumerGroupInfo.getChannelInfoTable().isEmpty()) { + ConsumerGroupInfo remove = this.consumerTable.remove(group); + if (remove != null) { + log.info("unregister consumer ok, no any connection, and remove consumer group, {}", group); + } + } + if (isNotifyConsumerIdsChangedEnable) { + this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel()); + } + } + } + + public void scanNotActiveChannel() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String group = next.getKey(); + ConsumerGroupInfo consumerGroupInfo = next.getValue(); + ConcurrentHashMap channelInfoTable = + consumerGroupInfo.getChannelInfoTable(); + + Iterator> itChannel = channelInfoTable.entrySet().iterator(); + while (itChannel.hasNext()) { + Entry nextChannel = itChannel.next(); + ClientChannelInfo clientChannelInfo = nextChannel.getValue(); + long diff = System.currentTimeMillis() - clientChannelInfo.getLastUpdateTimestamp(); + if (diff > CHANNEL_EXPIRED_TIMEOUT) { + log.warn( + "SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}", + RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group); + RemotingUtil.closeChannel(clientChannelInfo.getChannel()); + itChannel.remove(); + } + } + + if (channelInfoTable.isEmpty()) { + log.warn( + "SCAN: remove expired channel from ConsumerManager consumerTable, all clear, consumerGroup={}", + group); + it.remove(); + } + } + } + + public HashSet queryTopicConsumeByWho(final String topic) { + HashSet groups = new HashSet(); + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + ConcurrentHashMap subscriptionTable = + entry.getValue().getSubscriptionTable(); + if (subscriptionTable.containsKey(topic)) { + groups.add(entry.getKey()); + } + } + return groups; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java new file mode 100644 index 0000000000..0095913ee5 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.broker.BrokerController; +import io.netty.channel.Channel; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener { + private final BrokerController brokerController; + + + public DefaultConsumerIdsChangeListener(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public void consumerIdsChanged(String group, List channels) { + if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) { + for (Channel chl : channels) { + this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group); + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java new file mode 100644 index 0000000000..74e7ea7d85 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class ProducerManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final long LOCK_TIMEOUT_MILLIS = 3000; + private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; + private final Lock groupChannelLock = new ReentrantLock(); + private final HashMap> groupChannelTable = + new HashMap>(); + + + public ProducerManager() { + } + + + public HashMap> getGroupChannelTable() { + HashMap> newGroupChannelTable = + new HashMap>(); + try { + if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + newGroupChannelTable.putAll(groupChannelTable); + } finally { + groupChannelLock.unlock(); + } + } + } catch (InterruptedException e) { + log.error("", e); + } + return newGroupChannelTable; + } + + + public void scanNotActiveChannel() { + try { + if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final HashMap chlMap = entry.getValue(); + + Iterator> it = chlMap.entrySet().iterator(); + while (it.hasNext()) { + Entry item = it.next(); + // final Integer id = item.getKey(); + final ClientChannelInfo info = item.getValue(); + + long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp(); + if (diff > CHANNEL_EXPIRED_TIMEOUT) { + it.remove(); + log.warn( + "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", + RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); + RemotingUtil.closeChannel(info.getChannel()); + } + } + } + } finally { + this.groupChannelLock.unlock(); + } + } else { + log.warn("ProducerManager scanNotActiveChannel lock timeout"); + } + } catch (InterruptedException e) { + log.error("", e); + } + } + + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + if (channel != null) { + try { + if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final HashMap clientChannelInfoTable = + entry.getValue(); + final ClientChannelInfo clientChannelInfo = + clientChannelInfoTable.remove(channel); + if (clientChannelInfo != null) { + log.info( + "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", + clientChannelInfo.toString(), remoteAddr, group); + } + + } + } finally { + this.groupChannelLock.unlock(); + } + } else { + log.warn("ProducerManager doChannelCloseEvent lock timeout"); + } + } catch (InterruptedException e) { + log.error("", e); + } + } + } + + + public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { + try { + ClientChannelInfo clientChannelInfoFound = null; + + if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + HashMap channelTable = this.groupChannelTable.get(group); + if (null == channelTable) { + channelTable = new HashMap(); + this.groupChannelTable.put(group, channelTable); + } + + clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); + if (null == clientChannelInfoFound) { + channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); + log.info("new producer connected, group: {} channel: {}", group, + clientChannelInfo.toString()); + } + } finally { + this.groupChannelLock.unlock(); + } + + if (clientChannelInfoFound != null) { + clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis()); + } + } else { + log.warn("ProducerManager registerProducer lock timeout"); + } + } catch (InterruptedException e) { + log.error("", e); + } + } + + + public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { + try { + if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + HashMap channelTable = this.groupChannelTable.get(group); + if (null != channelTable && !channelTable.isEmpty()) { + ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); + if (old != null) { + log.info("unregister a producer[{}] from groupChannelTable {}", group, + clientChannelInfo.toString()); + } + + if (channelTable.isEmpty()) { + this.groupChannelTable.remove(group); + log.info("unregister a producer group[{}] from groupChannelTable", group); + } + } + } finally { + this.groupChannelLock.unlock(); + } + } else { + log.warn("ProducerManager unregisterProducer lock timeout"); + } + } catch (InterruptedException e) { + log.error("", e); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java new file mode 100644 index 0000000000..a38c9cbcd0 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java @@ -0,0 +1,317 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client.net; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.broker.pagecache.OneMessageTransfer; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.message.MessageQueueForC; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.GetConsumerStatusBody; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBody; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBodyForC; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.FileRegion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class Broker2Client { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + + public Broker2Client(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void checkProducerTransactionState( + final Channel channel, + final CheckTransactionStateRequestHeader requestHeader, + final SelectMappedBufferResult selectMappedBufferResult) { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader); + request.markOnewayRPC(); + + try { + FileRegion fileRegion = + new OneMessageTransfer(request.encodeHeader(selectMappedBufferResult.getSize()), + selectMappedBufferResult); + channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + selectMappedBufferResult.release(); + if (!future.isSuccess()) { + log.error("invokeProducer failed,", future.cause()); + } + } + }); + } catch (Throwable e) { + log.error("invokeProducer exception", e); + selectMappedBufferResult.release(); + } + } + + public RemotingCommand callClient(final Channel channel, + final RemotingCommand request + ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000); + } + + public void notifyConsumerIdsChanged( + final Channel channel, + final String consumerGroup) { + if (null == consumerGroup) { + log.error("notifyConsumerIdsChanged consumerGroup is null"); + return; + } + + NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, requestHeader); + + try { + this.brokerController.getRemotingServer().invokeOneway(channel, request, 10); + } catch (Exception e) { + log.error("notifyConsumerIdsChanged exception, " + consumerGroup, e.getMessage()); + } + } + + public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) { + return resetOffset(topic, group, timeStamp, isForce, false); + } + + public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, + boolean isC) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.error("[reset-offset] reset offset failed, no topic in this broker. topic={}", topic); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("[reset-offset] reset offset failed, no topic in this broker. topic=" + topic); + return response; + } + + Map offsetTable = new HashMap(); + + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setTopic(topic); + mq.setQueueId(i); + + long consumerOffset = + this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, i); + if (-1 == consumerOffset) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("THe consumer group <%s> not exist", group)); + return response; + } + + long timeStampOffset; + if (timeStamp == -1) { + + timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + } else { + timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp); + } + + if (timeStampOffset < 0) { + log.warn("reset offset is invalid. topic={}, queueId={}, timeStampOffset={}", topic, i, timeStampOffset); + timeStampOffset = 0; + } + + if (isForce || timeStampOffset < consumerOffset) { + offsetTable.put(mq, timeStampOffset); + } else { + offsetTable.put(mq, consumerOffset); + } + } + + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setTimestamp(timeStamp); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, requestHeader); + if (isC) { + // c++ language + ResetOffsetBodyForC body = new ResetOffsetBodyForC(); + List offsetList = convertOffsetTable2OffsetList(offsetTable); + body.setOffsetTable(offsetList); + request.setBody(body.encode()); + } else { + // other language + ResetOffsetBody body = new ResetOffsetBody(); + body.setOffsetTable(offsetTable); + request.setBody(body.encode()); + } + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo(group); + + if (consumerGroupInfo != null && !consumerGroupInfo.getAllChannel().isEmpty()) { + ConcurrentHashMap channelInfoTable = + consumerGroupInfo.getChannelInfoTable(); + for (Map.Entry entry : channelInfoTable.entrySet()) { + int version = entry.getValue().getVersion(); + if (version >= MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) { + try { + this.brokerController.getRemotingServer().invokeOneway(entry.getKey(), request, 5000); + log.info("[reset-offset] reset offset success. topic={}, group={}, clientId={}", + new Object[]{topic, group, entry.getValue().getClientId()}); + } catch (Exception e) { + log.error("[reset-offset] reset offset exception. topic={}, group={}", + new Object[]{topic, group}, e); + } + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the client does not support this feature. version=" + + MQVersion.getVersionDesc(version)); + log.warn("[reset-offset] the client does not support this feature. version={}", + RemotingHelper.parseChannelRemoteAddr(entry.getKey()), MQVersion.getVersionDesc(version)); + return response; + } + } + } else { + String errorInfo = + String.format("Consumer not online, so can not reset offset, Group: %s Topic: %s Timestamp: %d", + requestHeader.getGroup(), + requestHeader.getTopic(), + requestHeader.getTimestamp()); + log.error(errorInfo); + response.setCode(ResponseCode.CONSUMER_NOT_ONLINE); + response.setRemark(errorInfo); + return response; + } + response.setCode(ResponseCode.SUCCESS); + ResetOffsetBody resBody = new ResetOffsetBody(); + resBody.setOffsetTable(offsetTable); + response.setBody(resBody.encode()); + return response; + } + + private List convertOffsetTable2OffsetList(Map table) { + List list = new ArrayList(); + for (Entry entry : table.entrySet()) { + MessageQueue mq = entry.getKey(); + MessageQueueForC tmp = + new MessageQueueForC(mq.getTopic(), mq.getBrokerName(), mq.getQueueId(), entry.getValue()); + list.add(tmp); + } + return list; + } + + public RemotingCommand getConsumeStatus(String topic, String group, String originClientId) { + final RemotingCommand result = RemotingCommand.createResponseCommand(null); + + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, + requestHeader); + + Map> consumerStatusTable = + new HashMap>(); + ConcurrentHashMap channelInfoTable = + this.brokerController.getConsumerManager().getConsumerGroupInfo(group).getChannelInfoTable(); + if (null == channelInfoTable || channelInfoTable.isEmpty()) { + result.setCode(ResponseCode.SYSTEM_ERROR); + result.setRemark(String.format("No Any Consumer online in the consumer group: [%s]", group)); + return result; + } + + for (Map.Entry entry : channelInfoTable.entrySet()) { + int version = entry.getValue().getVersion(); + String clientId = entry.getValue().getClientId(); + if (version < MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) { + result.setCode(ResponseCode.SYSTEM_ERROR); + result.setRemark("the client does not support this feature. version=" + + MQVersion.getVersionDesc(version)); + log.warn("[get-consumer-status] the client does not support this feature. version={}", + RemotingHelper.parseChannelRemoteAddr(entry.getKey()), MQVersion.getVersionDesc(version)); + return result; + } else if (UtilAll.isBlank(originClientId) || originClientId.equals(clientId)) { + try { + RemotingCommand response = + this.brokerController.getRemotingServer().invokeSync(entry.getKey(), request, 5000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerStatusBody body = + GetConsumerStatusBody.decode(response.getBody(), + GetConsumerStatusBody.class); + + consumerStatusTable.put(clientId, body.getMessageQueueTable()); + log.info( + "[get-consumer-status] get consumer status success. topic={}, group={}, channelRemoteAddr={}", + new Object[]{topic, group, clientId}); + } + } + default: + break; + } + } catch (Exception e) { + log.error( + "[get-consumer-status] get consumer status exception. topic={}, group={}, offset={}", + new Object[]{topic, group}, e); + } + + if (!UtilAll.isBlank(originClientId) && originClientId.equals(clientId)) { + break; + } + } + } + + result.setCode(ResponseCode.SUCCESS); + GetConsumerStatusBody resBody = new GetConsumerStatusBody(); + resBody.setConsumerTable(consumerStatusTable); + result.setBody(resBody.encode()); + return result; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java new file mode 100644 index 0000000000..84be628f91 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java @@ -0,0 +1,281 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client.rebalance; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class RebalanceLockManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME); + private final static long REBALANCE_LOCK_MAX_LIVE_TIME = Long.parseLong(System.getProperty( + "rocketmq.broker.rebalance.lockMaxLiveTime", "60000")); + private final Lock lock = new ReentrantLock(); + private final ConcurrentHashMap> mqLockTable = + new ConcurrentHashMap>(1024); + + public boolean tryLock(final String group, final MessageQueue mq, final String clientId) { + + if (!this.isLocked(group, mq, clientId)) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null == groupValue) { + groupValue = new ConcurrentHashMap(32); + this.mqLockTable.put(group, groupValue); + } + + LockEntry lockEntry = groupValue.get(mq); + if (null == lockEntry) { + lockEntry = new LockEntry(); + lockEntry.setClientId(clientId); + groupValue.put(mq, lockEntry); + log.info("tryLock, message queue not locked, I got it. Group: {} NewClientId: {} {}", // + group, // + clientId, // + mq); + } + + if (lockEntry.isLocked(clientId)) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + return true; + } + + String oldClientId = lockEntry.getClientId(); + + + if (lockEntry.isExpired()) { + lockEntry.setClientId(clientId); + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + log.warn( + "tryLock, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + return true; + } + + + log.warn( + "tryLock, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + return false; + } finally { + this.lock.unlock(); + } + } catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } else { + + } + + return true; + } + + private boolean isLocked(final String group, final MessageQueue mq, final String clientId) { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (groupValue != null) { + LockEntry lockEntry = groupValue.get(mq); + if (lockEntry != null) { + boolean locked = lockEntry.isLocked(clientId); + if (locked) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + } + + return locked; + } + } + + return false; + } + + public Set tryLockBatch(final String group, final Set mqs, + final String clientId) { + Set lockedMqs = new HashSet(mqs.size()); + Set notLockedMqs = new HashSet(mqs.size()); + + + for (MessageQueue mq : mqs) { + if (this.isLocked(group, mq, clientId)) { + lockedMqs.add(mq); + } else { + notLockedMqs.add(mq); + } + } + + if (!notLockedMqs.isEmpty()) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null == groupValue) { + groupValue = new ConcurrentHashMap(32); + this.mqLockTable.put(group, groupValue); + } + + + for (MessageQueue mq : notLockedMqs) { + LockEntry lockEntry = groupValue.get(mq); + if (null == lockEntry) { + lockEntry = new LockEntry(); + lockEntry.setClientId(clientId); + groupValue.put(mq, lockEntry); + log.info( + "tryLockBatch, message queue not locked, I got it. Group: {} NewClientId: {} {}", // + group, // + clientId, // + mq); + } + + + if (lockEntry.isLocked(clientId)) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + lockedMqs.add(mq); + continue; + } + + String oldClientId = lockEntry.getClientId(); + + + if (lockEntry.isExpired()) { + lockEntry.setClientId(clientId); + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + log.warn( + "tryLockBatch, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + lockedMqs.add(mq); + continue; + } + + + log.warn( + "tryLockBatch, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + } + } finally { + this.lock.unlock(); + } + } catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } + + return lockedMqs; + } + + public void unlockBatch(final String group, final Set mqs, final String clientId) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null != groupValue) { + for (MessageQueue mq : mqs) { + LockEntry lockEntry = groupValue.get(mq); + if (null != lockEntry) { + if (lockEntry.getClientId().equals(clientId)) { + groupValue.remove(mq); + log.info("unlockBatch, Group: {} {} {}", + group, + mq, + clientId); + } else { + log.warn("unlockBatch, but mq locked by other client: {}, Group: {} {} {}", + lockEntry.getClientId(), + group, + mq, + clientId); + } + } else { + log.warn("unlockBatch, but mq not locked, Group: {} {} {}", + group, + mq, + clientId); + } + } + } else { + log.warn("unlockBatch, group not exist, Group: {} {}", + group, + clientId); + } + } finally { + this.lock.unlock(); + } + } catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } + + static class LockEntry { + private String clientId; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + public boolean isLocked(final String clientId) { + boolean eq = this.clientId.equals(clientId); + return eq && !this.isExpired(); + } + + public boolean isExpired() { + boolean expired = + (System.currentTimeMillis() - this.lastUpdateTimestamp) > REBALANCE_LOCK_MAX_LIVE_TIME; + + return expired; + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java new file mode 100644 index 0000000000..b2e7e82117 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.filtersrv; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerStartup; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class FilterServerManager { + + public static final long FILTER_SERVER_MAX_IDLE_TIME_MILLS = 30000; + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final ConcurrentHashMap filterServerTable = + new ConcurrentHashMap(16); + private final BrokerController brokerController; + + private ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FilterServerManagerScheduledThread")); + + public FilterServerManager(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void start() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + FilterServerManager.this.createFilterServer(); + } catch (Exception e) { + log.error("", e); + } + } + }, 1000 * 5, 1000 * 30, TimeUnit.MILLISECONDS); + } + + public void createFilterServer() { + int more = + this.brokerController.getBrokerConfig().getFilterServerNums() - this.filterServerTable.size(); + String cmd = this.buildStartCommand(); + for (int i = 0; i < more; i++) { + FilterServerUtil.callShell(cmd, log); + } + } + + private String buildStartCommand() { + String config = ""; + if (BrokerStartup.configFile != null) { + config = String.format("-c %s", BrokerStartup.configFile); + } + + if (this.brokerController.getBrokerConfig().getNamesrvAddr() != null) { + config += String.format(" -n %s", this.brokerController.getBrokerConfig().getNamesrvAddr()); + } + + if (RemotingUtil.isWindowsPlatform()) { + return String.format("start /b %s\\bin\\mqfiltersrv.exe %s", + this.brokerController.getBrokerConfig().getRocketmqHome(), + config); + } else { + return String.format("sh %s/bin/startfsrv.sh %s", + this.brokerController.getBrokerConfig().getRocketmqHome(), + config); + } + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + public void registerFilterServer(final Channel channel, final String filterServerAddr) { + FilterServerInfo filterServerInfo = this.filterServerTable.get(channel); + if (filterServerInfo != null) { + filterServerInfo.setLastUpdateTimestamp(System.currentTimeMillis()); + } else { + filterServerInfo = new FilterServerInfo(); + filterServerInfo.setFilterServerAddr(filterServerAddr); + filterServerInfo.setLastUpdateTimestamp(System.currentTimeMillis()); + this.filterServerTable.put(channel, filterServerInfo); + log.info("Receive a New Filter Server<{}>", filterServerAddr); + } + } + + /** + + */ + public void scanNotActiveChannel() { + + Iterator> it = this.filterServerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long timestamp = next.getValue().getLastUpdateTimestamp(); + Channel channel = next.getKey(); + if ((System.currentTimeMillis() - timestamp) > FILTER_SERVER_MAX_IDLE_TIME_MILLS) { + log.info("The Filter Server<{}> expired, remove it", next.getKey()); + it.remove(); + RemotingUtil.closeChannel(channel); + } + } + } + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + FilterServerInfo old = this.filterServerTable.remove(channel); + if (old != null) { + log.warn("The Filter Server<{}> connection<{}> closed, remove it", old.getFilterServerAddr(), + remoteAddr); + } + } + + public List buildNewFilterServerList() { + List addr = new ArrayList(); + Iterator> it = this.filterServerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + addr.add(next.getValue().getFilterServerAddr()); + } + return addr; + } + + static class FilterServerInfo { + private String filterServerAddr; + private long lastUpdateTimestamp; + + + public String getFilterServerAddr() { + return filterServerAddr; + } + + + public void setFilterServerAddr(String filterServerAddr) { + this.filterServerAddr = filterServerAddr; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java new file mode 100644 index 0000000000..c5ace191c7 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.filtersrv; + +import org.slf4j.Logger; + + +public class FilterServerUtil { + public static void callShell(final String shellString, final Logger log) { + Process process = null; + try { + String[] cmdArray = splitShellString(shellString); + process = Runtime.getRuntime().exec(cmdArray); + process.waitFor(); + log.info("callShell: <{}> OK", shellString); + } catch (Throwable e) { + log.error("callShell: readLine IOException, " + shellString, e); + } finally { + if (null != process) + process.destroy(); + } + } + + private static String[] splitShellString(final String shellString) { + String[] split = shellString.split(" "); + return split; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFastFailure.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFastFailure.java new file mode 100644 index 0000000000..586bed099d --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFastFailure.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.latency; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.netty.RequestTask; +import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class BrokerFastFailure { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "BrokerFastFailureScheduledThread")); + private final BrokerController brokerController; + + public BrokerFastFailure(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void start() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + cleanExpiredRequest(); + } + }, 1000, 10, TimeUnit.MILLISECONDS); + } + + private void cleanExpiredRequest() { + while (this.brokerController.getMessageStore().isOSPageCacheBusy()) { + try { + if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) { + final Runnable runnable = this.brokerController.getSendThreadPoolQueue().poll(0, TimeUnit.SECONDS); + if (null == runnable) { + break; + } + + final RequestTask rt = castRunnable(runnable); + rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", System.currentTimeMillis() - rt.getCreateTimestamp(), this.brokerController.getSendThreadPoolQueue().size())); + } else { + break; + } + } catch (Throwable e) { + } + } + + while (true) { + try { + if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) { + final Runnable runnable = this.brokerController.getSendThreadPoolQueue().peek(); + if (null == runnable) { + break; + } + final RequestTask rt = castRunnable(runnable); + if (rt.isStopRun()) { + break; + } + + final long behind = System.currentTimeMillis() - rt.getCreateTimestamp(); + if (behind >= this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()) { + if (this.brokerController.getSendThreadPoolQueue().remove(runnable)) { + rt.setStopRun(true); + rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", behind, this.brokerController.getSendThreadPoolQueue().size())); + } + } else { + break; + } + } else { + break; + } + } catch (Throwable e) { + } + } + } + + public static RequestTask castRunnable(final Runnable runnable) { + try { + FutureTaskExt object = (FutureTaskExt) runnable; + return (RequestTask) object.getRunnable(); + } catch (Throwable e) { + log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e); + } + + return null; + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java new file mode 100644 index 0000000000..f81d48a6b5 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.latency; + +import java.util.concurrent.*; + +/** + * @author shijia.wxr + */ +public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor { + public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, final TimeUnit unit, final BlockingQueue workQueue) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); + } + + public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, final TimeUnit unit, final BlockingQueue workQueue, final ThreadFactory threadFactory) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + } + + public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, final TimeUnit unit, final BlockingQueue workQueue, final RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); + } + + public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, final TimeUnit unit, final BlockingQueue workQueue, final ThreadFactory threadFactory, final RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + } + + @Override + protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { + return new FutureTaskExt(runnable, value); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/FutureTaskExt.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/FutureTaskExt.java new file mode 100644 index 0000000000..6ec7bb5712 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/latency/FutureTaskExt.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.latency; + +import java.util.concurrent.Callable; +import java.util.concurrent.FutureTask; + +/** + * @author shijia.wxr + */ +public class FutureTaskExt extends FutureTask { + private final Runnable runnable; + + public FutureTaskExt(final Callable callable) { + super(callable); + this.runnable = null; + } + + public FutureTaskExt(final Runnable runnable, final V result) { + super(runnable, result); + this.runnable = runnable; + } + + public Runnable getRunnable() { + return runnable; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java new file mode 100644 index 0000000000..bc9c58d8b8 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class ManyPullRequest { + private final ArrayList pullRequestList = new ArrayList(); + + + public synchronized void addPullRequest(final PullRequest pullRequest) { + this.pullRequestList.add(pullRequest); + } + + + public synchronized void addPullRequest(final List many) { + this.pullRequestList.addAll(many); + } + + + public synchronized List cloneListAndClear() { + if (!this.pullRequestList.isEmpty()) { + List result = (ArrayList) this.pullRequestList.clone(); + this.pullRequestList.clear(); + return result; + } + + return null; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java new file mode 100644 index 0000000000..15ee05019a --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.longpolling; + +import com.alibaba.rocketmq.store.MessageArrivingListener; + + +public class NotifyMessageArrivingListener implements MessageArrivingListener { + private final PullRequestHoldService pullRequestHoldService; + + + public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService) { + this.pullRequestHoldService = pullRequestHoldService; + } + + + @Override + public void arriving(String topic, int queueId, long logicOffset, long tagsCode) { + this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java new file mode 100644 index 0000000000..b4f1e11dab --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.Channel; + + +/** + * @author shijia.wxr + */ +public class PullRequest { + private final RemotingCommand requestCommand; + private final Channel clientChannel; + private final long timeoutMillis; + private final long suspendTimestamp; + private final long pullFromThisOffset; + private final SubscriptionData subscriptionData; + + + public PullRequest(RemotingCommand requestCommand, Channel clientChannel, long timeoutMillis, long suspendTimestamp, + long pullFromThisOffset, SubscriptionData subscriptionData) { + this.requestCommand = requestCommand; + this.clientChannel = clientChannel; + this.timeoutMillis = timeoutMillis; + this.suspendTimestamp = suspendTimestamp; + this.pullFromThisOffset = pullFromThisOffset; + this.subscriptionData = subscriptionData; + } + + + public RemotingCommand getRequestCommand() { + return requestCommand; + } + + + public Channel getClientChannel() { + return clientChannel; + } + + + public long getTimeoutMillis() { + return timeoutMillis; + } + + + public long getSuspendTimestamp() { + return suspendTimestamp; + } + + + public long getPullFromThisOffset() { + return pullFromThisOffset; + } + + public SubscriptionData getSubscriptionData() { + return subscriptionData; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java new file mode 100644 index 0000000000..888c5f2813 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.SystemClock; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.DefaultMessageFilter; +import com.alibaba.rocketmq.store.MessageFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class PullRequestHoldService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final String TOPIC_QUEUEID_SEPARATOR = "@"; + private final BrokerController brokerController; + private final SystemClock systemClock = new SystemClock(); + private final MessageFilter messageFilter = new DefaultMessageFilter(); + private ConcurrentHashMap pullRequestTable = + new ConcurrentHashMap(1024); + + + public PullRequestHoldService(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void suspendPullRequest(final String topic, final int queueId, final PullRequest pullRequest) { + String key = this.buildKey(topic, queueId); + ManyPullRequest mpr = this.pullRequestTable.get(key); + if (null == mpr) { + mpr = new ManyPullRequest(); + ManyPullRequest prev = this.pullRequestTable.putIfAbsent(key, mpr); + if (prev != null) { + mpr = prev; + } + } + + mpr.addPullRequest(pullRequest); + } + + private String buildKey(final String topic, final int queueId) { + StringBuilder sb = new StringBuilder(); + sb.append(topic); + sb.append(TOPIC_QUEUEID_SEPARATOR); + sb.append(queueId); + return sb.toString(); + } + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + while (!this.isStopped()) { + try { + if (this.brokerController.getBrokerConfig().isLongPollingEnable()) { + this.waitForRunning(5 * 1000); + } else { + this.waitForRunning(this.brokerController.getBrokerConfig().getShortPollingTimeMills()); + } + + long beginLockTimestamp = this.systemClock.now(); + this.checkHoldRequest(); + long costTime = this.systemClock.now() - beginLockTimestamp; + if (costTime > 5 * 1000) { + log.info("[NOTIFYME] check hold request cost {} ms.", costTime); + } + } catch (Throwable e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + @Override + public String getServiceName() { + return PullRequestHoldService.class.getSimpleName(); + } + + private void checkHoldRequest() { + for (String key : this.pullRequestTable.keySet()) { + String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR); + if (kArray != null && 2 == kArray.length) { + String topic = kArray[0]; + int queueId = Integer.parseInt(kArray[1]); + final long offset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId); + try { + this.notifyMessageArriving(topic, queueId, offset); + } catch (Throwable e) { + log.error("check hold request failed. topic={}, queueId={}", topic, queueId, e); + } + } + } + } + + public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) { + notifyMessageArriving(topic, queueId, maxOffset, null); + } + + public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset, final Long tagsCode) { + String key = this.buildKey(topic, queueId); + ManyPullRequest mpr = this.pullRequestTable.get(key); + if (mpr != null) { + List requestList = mpr.cloneListAndClear(); + if (requestList != null) { + List replayList = new ArrayList(); + + for (PullRequest request : requestList) { + long newestOffset = maxOffset; + if (newestOffset <= request.getPullFromThisOffset()) { + newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId); + } + + Long tmp = tagsCode; + if (newestOffset > request.getPullFromThisOffset()) { + if (this.messageFilter.isMessageMatched(request.getSubscriptionData(), tmp)) { + try { + this.brokerController.getPullMessageProcessor().excuteRequestWhenWakeup(request.getClientChannel(), + request.getRequestCommand()); + } catch (Throwable e) { + log.error("execute request when wakeup failed.", e); + } + continue; + } + } + + if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request.getTimeoutMillis())) { + try { + this.brokerController.getPullMessageProcessor().excuteRequestWhenWakeup(request.getClientChannel(), + request.getRequestCommand()); + } catch (Throwable e) { + log.error("execute request when wakeup failed.", e); + } + continue; + } + + + replayList.add(request); + } + + if (!replayList.isEmpty()) { + mpr.addPullRequest(replayList); + } + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java new file mode 100644 index 0000000000..b7f9c6ea0a --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; + +import java.util.Map; + + +public class ConsumeMessageContext { + private String consumerGroup; + private String topic; + private Integer queueId; + private String clientHost; + private String storeHost; + private Map messageIds; + private int bodyLength; + private boolean success; + private String status; + private Object mqTraceContext; + + private String commercialOwner; + private BrokerStatsManager.StatsType commercialRcvStats; + private int commercialRcvTimes; + private int commercialRcvSize; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public String getClientHost() { + return clientHost; + } + + + public void setClientHost(String clientHost) { + this.clientHost = clientHost; + } + + + public String getStoreHost() { + return storeHost; + } + + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + + public Map getMessageIds() { + return messageIds; + } + + + public void setMessageIds(Map messageIds) { + this.messageIds = messageIds; + } + + + public boolean isSuccess() { + return success; + } + + + public void setSuccess(boolean success) { + this.success = success; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public int getBodyLength() { + return bodyLength; + } + + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } + + public String getCommercialOwner() { + return commercialOwner; + } + + public void setCommercialOwner(final String commercialOwner) { + this.commercialOwner = commercialOwner; + } + + public BrokerStatsManager.StatsType getCommercialRcvStats() { + return commercialRcvStats; + } + + public void setCommercialRcvStats(final BrokerStatsManager.StatsType commercialRcvStats) { + this.commercialRcvStats = commercialRcvStats; + } + + public int getCommercialRcvTimes() { + return commercialRcvTimes; + } + + public void setCommercialRcvTimes(final int commercialRcvTimes) { + this.commercialRcvTimes = commercialRcvTimes; + } + + public int getCommercialRcvSize() { + return commercialRcvSize; + } + + public void setCommercialRcvSize(final int commercialRcvSize) { + this.commercialRcvSize = commercialRcvSize; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java new file mode 100644 index 0000000000..4a74db3ecc --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +public interface ConsumeMessageHook { + String hookName(); + + + void consumeMessageBefore(final ConsumeMessageContext context); + + + void consumeMessageAfter(final ConsumeMessageContext context); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java new file mode 100644 index 0000000000..5bd29cf392 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +import com.alibaba.rocketmq.common.message.MessageType; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; + +import java.util.Properties; + + +public class SendMessageContext { + private String producerGroup; + private String topic; + private String msgId; + private String originMsgId; + private Integer queueId; + private Long queueOffset; + private String brokerAddr; + private String bornHost; + private int bodyLength; + private int code; + private String errorMsg; + private String msgProps; + private Object mqTraceContext; + private Properties extProps; + private String brokerRegionId; + private String msgUniqueKey; + private long bornTimeStamp; + private MessageType msgType = MessageType.Trans_msg_Commit; + private boolean isSuccess = false; + //For Commercial + private String commercialOwner; + private BrokerStatsManager.StatsType commercialSendStats; + private int commercialSendSize; + private int commercialSendTimes; + + public boolean isSuccess() { + return isSuccess; + } + + public void setSuccess(final boolean success) { + isSuccess = success; + } + + public MessageType getMsgType() { + return msgType; + } + + public void setMsgType(final MessageType msgType) { + this.msgType = msgType; + } + + public String getMsgUniqueKey() { + return msgUniqueKey; + } + + public void setMsgUniqueKey(final String msgUniqueKey) { + this.msgUniqueKey = msgUniqueKey; + } + + public long getBornTimeStamp() { + return bornTimeStamp; + } + + public void setBornTimeStamp(final long bornTimeStamp) { + this.bornTimeStamp = bornTimeStamp; + } + + public String getBrokerRegionId() { + return brokerRegionId; + } + + public void setBrokerRegionId(final String brokerRegionId) { + this.brokerRegionId = brokerRegionId; + } + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public String getOriginMsgId() { + return originMsgId; + } + + + public void setOriginMsgId(String originMsgId) { + this.originMsgId = originMsgId; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getBornHost() { + return bornHost; + } + + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } + + + public int getBodyLength() { + return bodyLength; + } + + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } + + + public int getCode() { + return code; + } + + + public void setCode(int code) { + this.code = code; + } + + + public String getErrorMsg() { + return errorMsg; + } + + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + + public String getMsgProps() { + return msgProps; + } + + + public void setMsgProps(String msgProps) { + this.msgProps = msgProps; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Properties getExtProps() { + return extProps; + } + + + public void setExtProps(Properties extProps) { + this.extProps = extProps; + } + + public String getCommercialOwner() { + return commercialOwner; + } + + public void setCommercialOwner(final String commercialOwner) { + this.commercialOwner = commercialOwner; + } + + public BrokerStatsManager.StatsType getCommercialSendStats() { + return commercialSendStats; + } + + public void setCommercialSendStats(final BrokerStatsManager.StatsType commercialSendStats) { + this.commercialSendStats = commercialSendStats; + } + + public int getCommercialSendSize() { + return commercialSendSize; + } + + public void setCommercialSendSize(final int commercialSendSize) { + this.commercialSendSize = commercialSendSize; + } + + public int getCommercialSendTimes() { + return commercialSendTimes; + } + + public void setCommercialSendTimes(final int commercialSendTimes) { + this.commercialSendTimes = commercialSendTimes; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java new file mode 100644 index 0000000000..e079b9f45a --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +public interface SendMessageHook { + public String hookName(); + + + public void sendMessageBefore(final SendMessageContext context); + + + public void sendMessageAfter(final SendMessageContext context); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java new file mode 100644 index 0000000000..269918a93b --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -0,0 +1,249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.offset; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerOffsetManager extends ConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final String TOPIC_GROUP_SEPARATOR = "@"; + + private ConcurrentHashMap> offsetTable = + new ConcurrentHashMap>(512); + + private transient BrokerController brokerController; + + + public ConsumerOffsetManager() { + } + + + public ConsumerOffsetManager(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public void scanUnsubscribedTopic() { + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + String topic = arrays[0]; + String group = arrays[1]; + + if (null == brokerController.getConsumerManager().findSubscriptionData(group, topic) + && this.offsetBehindMuchThanData(topic, next.getValue())) { + it.remove(); + log.warn("remove topic offset, {}", topicAtGroup); + } + } + } + } + + + private boolean offsetBehindMuchThanData(final String topic, ConcurrentHashMap table) { + Iterator> it = table.entrySet().iterator(); + boolean result = !table.isEmpty(); + + while (it.hasNext() && result) { + Entry next = it.next(); + long minOffsetInStore = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, next.getKey()); + long offsetInPersist = next.getValue(); + if (offsetInPersist > minOffsetInStore) { + result = false; + } else { + result = true; + } + } + + return result; + } + + + public Set whichTopicByConsumer(final String group) { + Set topics = new HashSet(); + + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + if (group.equals(arrays[1])) { + topics.add(arrays[0]); + } + } + } + + return topics; + } + + + public Set whichGroupByTopic(final String topic) { + Set groups = new HashSet(); + + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + if (topic.equals(arrays[0])) { + groups.add(arrays[1]); + } + } + } + + return groups; + } + + + public void commitOffset(final String clientHost, final String group, final String topic, final int queueId, final long offset) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + this.commitOffset(clientHost, key, queueId, offset); + } + + private void commitOffset(final String clientHost, final String key, final int queueId, final long offset) { + ConcurrentHashMap map = this.offsetTable.get(key); + if (null == map) { + map = new ConcurrentHashMap(32); + map.put(queueId, offset); + this.offsetTable.put(key, map); + } else { + Long storeOffset = map.put(queueId, offset); + if (storeOffset != null && offset < storeOffset) { + log.warn("[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}", clientHost, key, queueId, offset, storeOffset); + } + } + } + + public long queryOffset(final String group, final String topic, final int queueId) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap map = this.offsetTable.get(key); + if (null != map) { + Long offset = map.get(queueId); + if (offset != null) + return offset; + } + + return -1; + } + + public String encode() { + return this.encode(false); + } + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getConsumerOffsetPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class); + if (obj != null) { + this.offsetTable = obj.offsetTable; + } + } + } + + public String encode(final boolean prettyFormat) { + return RemotingSerializable.toJson(this, prettyFormat); + } + + public ConcurrentHashMap> getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap> offsetTable) { + this.offsetTable = offsetTable; + } + + + public Map queryMinOffsetInAllGroup(final String topic, final String filterGroups) { + + Map queueMinOffset = new HashMap(); + Set topicGroups = this.offsetTable.keySet(); + if (!UtilAll.isBlank(filterGroups)) { + for (String group : filterGroups.split(",")) { + Iterator it = topicGroups.iterator(); + while (it.hasNext()) { + if (group.equals(it.next().split(TOPIC_GROUP_SEPARATOR)[1])) { + it.remove(); + } + } + } + } + + for (Map.Entry> offSetEntry : this.offsetTable.entrySet()) { + String topicGroup = offSetEntry.getKey(); + String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR); + if (topic.equals(topicGroupArr[0])) { + for (Entry entry : offSetEntry.getValue().entrySet()) { + long minOffset = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, entry.getKey()); + if (entry.getValue() >= minOffset) { + Long offset = queueMinOffset.get(entry.getKey()); + if (offset == null) { + queueMinOffset.put(entry.getKey(), Math.min(Long.MAX_VALUE, entry.getValue())); + } else { + queueMinOffset.put(entry.getKey(), Math.min(entry.getValue(), offset)); + } + } + } + } + + } + return queueMinOffset; + } + + + public Map queryOffset(final String group, final String topic) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + return this.offsetTable.get(key); + } + + + public void cloneOffset(final String srcGroup, final String destGroup, final String topic) { + ConcurrentHashMap offsets = this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup); + if (offsets != null) { + this.offsetTable.put(topic + TOPIC_GROUP_SEPARATOR + destGroup, new ConcurrentHashMap(offsets)); + } + } + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java new file mode 100644 index 0000000000..f051d297ba --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java @@ -0,0 +1,302 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.out; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.namesrv.TopAddressing; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + * @author manhong.yqd + */ +public class BrokerOuterAPI { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final RemotingClient remotingClient; + private final TopAddressing topAddressing = new TopAddressing(MixAll.WS_ADDR); + private String nameSrvAddr = null; + + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { + this(nettyClientConfig, null); + } + + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) { + this.remotingClient = new NettyRemotingClient(nettyClientConfig); + this.remotingClient.registerRPCHook(rpcHook); + } + + public void start() { + this.remotingClient.start(); + } + + public void shutdown() { + this.remotingClient.shutdown(); + } + + public String fetchNameServerAddr() { + try { + String addrs = this.topAddressing.fetchNSAddr(); + if (addrs != null) { + if (!addrs.equals(this.nameSrvAddr)) { + log.info("name server address changed, old: " + this.nameSrvAddr + " new: " + addrs); + this.updateNameServerAddressList(addrs); + this.nameSrvAddr = addrs; + return nameSrvAddr; + } + } + } catch (Exception e) { + log.error("fetchNameServerAddr Exception", e); + } + return nameSrvAddr; + } + + public void updateNameServerAddressList(final String addrs) { + List lst = new ArrayList(); + String[] addrArray = addrs.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + lst.add(addr); + } + + this.remotingClient.updateNameServerAddressList(lst); + } + } + + public RegisterBrokerResult registerBrokerAll( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final String haServerAddr, + final TopicConfigSerializeWrapper topicConfigWrapper, + final List filterServerList, + final boolean oneway, + final int timeoutMills) { + RegisterBrokerResult registerBrokerResult = null; + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + for (String namesrvAddr : nameServerAddressList) { + try { + RegisterBrokerResult result = this.registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, + haServerAddr, topicConfigWrapper, filterServerList, oneway, timeoutMills); + if (result != null) { + registerBrokerResult = result; + } + + log.info("register broker to name server {} OK", namesrvAddr); + } catch (Exception e) { + log.warn("registerBroker Exception, " + namesrvAddr, e); + } + } + } + + return registerBrokerResult; + } + + private RegisterBrokerResult registerBroker( + final String namesrvAddr, + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final String haServerAddr, + final TopicConfigSerializeWrapper topicConfigWrapper, + final List filterServerList, + final boolean oneway, + final int timeoutMills + ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException { + RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + requestHeader.setHaServerAddr(haServerAddr); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); + + RegisterBrokerBody requestBody = new RegisterBrokerBody(); + requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); + requestBody.setFilterServerList(filterServerList); + request.setBody(requestBody.encode()); + + if (oneway) { + try { + this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills); + } catch (RemotingTooMuchRequestException e) { + } + return null; + } + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMills); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + RegisterBrokerResponseHeader responseHeader = + (RegisterBrokerResponseHeader) response.decodeCommandCustomHeader(RegisterBrokerResponseHeader.class); + RegisterBrokerResult result = new RegisterBrokerResult(); + result.setMasterAddr(responseHeader.getMasterAddr()); + result.setHaServerAddr(responseHeader.getHaServerAddr()); + result.setHaServerAddr(responseHeader.getHaServerAddr()); + if (response.getBody() != null) { + result.setKvTable(KVTable.decode(response.getBody(), KVTable.class)); + } + return result; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public void unregisterBrokerAll( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId + ) { + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + for (String namesrvAddr : nameServerAddressList) { + try { + this.unregisterBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId); + log.info("unregisterBroker OK, NamesrvAddr: {}", namesrvAddr); + } catch (Exception e) { + log.warn("unregisterBroker Exception, " + namesrvAddr, e); + } + } + } + } + + public void unregisterBroker( + final String namesrvAddr, + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId + ) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + UnRegisterBrokerRequestHeader requestHeader = new UnRegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public TopicConfigSerializeWrapper getAllTopicConfig(final String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(true, addr), request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public ConsumerOffsetSerializeWrapper getAllConsumerOffset(final String addr) throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return ConsumerOffsetSerializeWrapper.decode(response.getBody(), ConsumerOffsetSerializeWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public String getAllDelayOffset(final String addr) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException, UnsupportedEncodingException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return new String(response.getBody(), MixAll.DEFAULT_CHARSET); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public SubscriptionGroupWrapper getAllSubscriptionGroupConfig(final String addr) throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return SubscriptionGroupWrapper.decode(response.getBody(), SubscriptionGroupWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public void registerRPCHook(RPCHook rpcHook) { + remotingClient.registerRPCHook(rpcHook); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java new file mode 100644 index 0000000000..8050bc119c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import com.alibaba.rocketmq.store.GetMessageResult; +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class ManyMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final GetMessageResult getMessageResult; + private long transfered; // the bytes which was transfered already + + + public ManyMessageTransfer(ByteBuffer byteBufferHeader, GetMessageResult getMessageResult) { + this.byteBufferHeader = byteBufferHeader; + this.getMessageResult = getMessageResult; + } + + + @Override + public long position() { + int pos = byteBufferHeader.position(); + List messageBufferList = this.getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + pos += bb.position(); + } + return pos; + } + + @Override + public long transfered() { + return transfered; + } + + @Override + public long count() { + return byteBufferHeader.limit() + this.getMessageResult.getBufferTotalSize(); + } + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } else { + List messageBufferList = this.getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + if (bb.hasRemaining()) { + transfered += target.write(bb); + return transfered; + } + } + } + + return 0; + } + + public void close() { + this.deallocate(); + } + + @Override + protected void deallocate() { + this.getMessageResult.release(); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java new file mode 100644 index 0000000000..df742c52db --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + + +/** + * @author shijia.wxr + */ +public class OneMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final SelectMappedBufferResult selectMappedBufferResult; + private long transfered; // the bytes which was transfered already + + + public OneMessageTransfer(ByteBuffer byteBufferHeader, SelectMappedBufferResult selectMappedBufferResult) { + this.byteBufferHeader = byteBufferHeader; + this.selectMappedBufferResult = selectMappedBufferResult; + } + + + @Override + public long position() { + return this.byteBufferHeader.position() + this.selectMappedBufferResult.getByteBuffer().position(); + } + + @Override + public long transfered() { + return transfered; + } + + @Override + public long count() { + return this.byteBufferHeader.limit() + this.selectMappedBufferResult.getSize(); + } + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } else if (this.selectMappedBufferResult.getByteBuffer().hasRemaining()) { + transfered += target.write(this.selectMappedBufferResult.getByteBuffer()); + return transfered; + } + + return 0; + } + + public void close() { + this.deallocate(); + } + + @Override + protected void deallocate() { + this.selectMappedBufferResult.release(); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java new file mode 100644 index 0000000000..cbcbc742d8 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import com.alibaba.rocketmq.store.QueryMessageResult; +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class QueryMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final QueryMessageResult queryMessageResult; + private long transfered; // the bytes which was transfered already + + + public QueryMessageTransfer(ByteBuffer byteBufferHeader, QueryMessageResult queryMessageResult) { + this.byteBufferHeader = byteBufferHeader; + this.queryMessageResult = queryMessageResult; + } + + + @Override + public long position() { + int pos = byteBufferHeader.position(); + List messageBufferList = this.queryMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + pos += bb.position(); + } + return pos; + } + + @Override + public long transfered() { + return transfered; + } + + @Override + public long count() { + return byteBufferHeader.limit() + this.queryMessageResult.getBufferTotalSize(); + } + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } else { + List messageBufferList = this.queryMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + if (bb.hasRemaining()) { + transfered += target.write(bb); + return transfered; + } + } + } + + return 0; + } + + public void close() { + this.deallocate(); + } + + @Override + protected void deallocate() { + this.queryMessageResult.release(); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/AbstractPluginMessageStore.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/AbstractPluginMessageStore.java new file mode 100644 index 0000000000..141ba695d3 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/AbstractPluginMessageStore.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.broker.plugin; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.store.*; + +import java.util.HashMap; +import java.util.Set; + +public abstract class AbstractPluginMessageStore implements MessageStore { + protected MessageStore next = null; + protected MessageStorePluginContext context; + + public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) { + this.next = next; + this.context = context; + } + + @Override + public long getEarliestMessageTime() { + return next.getEarliestMessageTime(); + } + + @Override + public long lockTimeMills() { + return next.lockTimeMills(); + } + + @Override + public boolean isOSPageCacheBusy() { + return next.isOSPageCacheBusy(); + } + + @Override + public boolean isTransientStorePoolDeficient() { + return next.isTransientStorePoolDeficient(); + } + + @Override + public boolean load() { + return next.load(); + } + + @Override + public void start() throws Exception { + next.start(); + } + + @Override + public void shutdown() { + next.shutdown(); + } + + @Override + public void destroy() { + next.destroy(); + } + + @Override + public PutMessageResult putMessage(MessageExtBrokerInner msg) { + return next.putMessage(msg); + } + + @Override + public GetMessageResult getMessage(String group, String topic, int queueId, long offset, + int maxMsgNums, SubscriptionData subscriptionData) { + return next.getMessage(group, topic, queueId, offset, maxMsgNums, subscriptionData); + } + + @Override + public long getMaxOffsetInQuque(String topic, int queueId) { + return next.getMaxOffsetInQuque(topic, queueId); + } + + @Override + public long getMinOffsetInQuque(String topic, int queueId) { + return next.getMinOffsetInQuque(topic, queueId); + } + + @Override + public long getCommitLogOffsetInQueue(String topic, int queueId, long cqOffset) { + return next.getCommitLogOffsetInQueue(topic, queueId, cqOffset); + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { + return next.getOffsetInQueueByTime(topic, queueId, timestamp); + } + + @Override + public MessageExt lookMessageByOffset(long commitLogOffset) { + return next.lookMessageByOffset(commitLogOffset); + } + + @Override + public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset) { + return next.selectOneMessageByOffset(commitLogOffset); + } + + @Override + public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset, int msgSize) { + return next.selectOneMessageByOffset(commitLogOffset, msgSize); + } + + @Override + public String getRunningDataInfo() { + return next.getRunningDataInfo(); + } + + @Override + public HashMap getRuntimeInfo() { + return next.getRuntimeInfo(); + } + + @Override + public long getMaxPhyOffset() { + return next.getMaxPhyOffset(); + } + + @Override + public long getMinPhyOffset() { + return next.getMinPhyOffset(); + } + + @Override + public long getEarliestMessageTime(String topic, int queueId) { + return next.getEarliestMessageTime(topic, queueId); + } + + @Override + public long getMessageStoreTimeStamp(String topic, int queueId, long offset) { + return next.getMessageStoreTimeStamp(topic, queueId, offset); + } + + @Override + public long getMessageTotalInQueue(String topic, int queueId) { + return next.getMessageTotalInQueue(topic, queueId); + } + + @Override + public SelectMappedBufferResult getCommitLogData(long offset) { + return next.getCommitLogData(offset); + } + + @Override + public boolean appendToCommitLog(long startOffset, byte[] data) { + return next.appendToCommitLog(startOffset, data); + } + + @Override + public void excuteDeleteFilesManualy() { + next.excuteDeleteFilesManualy(); + } + + @Override + public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, + long end) { + return next.queryMessage(topic, key, maxNum, begin, end); + } + + @Override + public void updateHaMasterAddress(String newAddr) { + next.updateHaMasterAddress(newAddr); + } + + @Override + public long slaveFallBehindMuch() { + return next.slaveFallBehindMuch(); + } + + @Override + public long now() { + return next.now(); + } + + @Override + public int cleanUnusedTopic(Set topics) { + return next.cleanUnusedTopic(topics); + } + + @Override + public void cleanExpiredConsumerQueue() { + next.cleanExpiredConsumerQueue(); + } + + @Override + public boolean checkInDiskByConsumeOffset(String topic, int queueId, long consumeOffset) { + return next.checkInDiskByConsumeOffset(topic, queueId, consumeOffset); + } + + @Override + public long dispatchBehindBytes() { + return next.dispatchBehindBytes(); + } + + @Override + public long flush() { + return next.flush(); + } + + @Override + public boolean resetWriteOffset(long phyOffset) { + return next.resetWriteOffset(phyOffset); + } + + @Override + public long getConfirmOffset() { + return next.getConfirmOffset(); + } + + @Override + public void setConfirmOffset(long phyOffset) { + next.setConfirmOffset(phyOffset); + } + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStoreFactory.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStoreFactory.java new file mode 100644 index 0000000000..84f5be76ce --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStoreFactory.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.broker.plugin; + +import com.alibaba.rocketmq.store.MessageStore; + +import java.io.IOException; +import java.lang.reflect.Constructor; + +public final class MessageStoreFactory { + public final static MessageStore build(MessageStorePluginContext context, MessageStore messageStore) + throws IOException { + String plugin = context.getBrokerConfig().getMessageStorePlugIn(); + if (plugin != null && plugin.trim().length() != 0) { + String[] pluginClasses = plugin.split(","); + for (int i = pluginClasses.length - 1; i >= 0; --i) { + String pluginClass = pluginClasses[i]; + try { + @SuppressWarnings("unchecked") + Class clazz = (Class) Class.forName(pluginClass); + Constructor construct = clazz.getConstructor(MessageStorePluginContext.class, MessageStore.class); + AbstractPluginMessageStore pluginMessageStore = (AbstractPluginMessageStore) construct.newInstance(context, messageStore); + messageStore = pluginMessageStore; + } catch (Throwable e) { + throw new RuntimeException(String.format( + "Initialize plugin's class %s not found!", pluginClass), e); + } + } + } + return messageStore; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStorePluginContext.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStorePluginContext.java new file mode 100644 index 0000000000..15e8b07ddc --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/plugin/MessageStorePluginContext.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.broker.plugin; + +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.store.MessageArrivingListener; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; + +public class MessageStorePluginContext { + private MessageStoreConfig messageStoreConfig; + private BrokerStatsManager brokerStatsManager; + private MessageArrivingListener messageArrivingListener; + private BrokerConfig brokerConfig; + + public MessageStorePluginContext(MessageStoreConfig messageStoreConfig, + BrokerStatsManager brokerStatsManager, MessageArrivingListener messageArrivingListener, + BrokerConfig brokerConfig) { + super(); + this.messageStoreConfig = messageStoreConfig; + this.brokerStatsManager = brokerStatsManager; + this.messageArrivingListener = messageArrivingListener; + this.brokerConfig = brokerConfig; + } + + public MessageStoreConfig getMessageStoreConfig() { + return messageStoreConfig; + } + + public BrokerStatsManager getBrokerStatsManager() { + return brokerStatsManager; + } + + public MessageArrivingListener getMessageArrivingListener() { + return messageArrivingListener; + } + + public BrokerConfig getBrokerConfig() { + return brokerConfig; + } + + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AbstractSendMessageProcessor.java new file mode 100644 index 0000000000..95db52d2bd --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -0,0 +1,332 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageHook; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.DBMsgConstants; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import com.alibaba.rocketmq.common.protocol.header.SendMessageResponseHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.common.utils.ChannelUtil; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; +import java.util.Random; + + +/** + * @author shijia.wxr + */ +public abstract class AbstractSendMessageProcessor implements NettyRequestProcessor { + protected static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + protected final static int DLQ_NUMS_PER_GROUP = 1; + protected final BrokerController brokerController; + protected final Random random = new Random(System.currentTimeMillis()); + protected final SocketAddress storeHost; + private List sendMessageHookList; + + + public AbstractSendMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + this.storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), brokerController + .getNettyServerConfig().getListenPort()); + } + + protected SendMessageContext buildMsgContext(ChannelHandlerContext ctx, + SendMessageRequestHeader requestHeader) { + if (!this.hasSendMessageHook()) { + return null; + } + SendMessageContext mqtraceContext; + mqtraceContext = new SendMessageContext(); + mqtraceContext.setProducerGroup(requestHeader.getProducerGroup()); + mqtraceContext.setTopic(requestHeader.getTopic()); + mqtraceContext.setMsgProps(requestHeader.getProperties()); + mqtraceContext.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + mqtraceContext.setBrokerAddr(this.brokerController.getBrokerAddr()); + mqtraceContext.setBrokerRegionId(this.brokerController.getBrokerConfig().getRegionId()); + mqtraceContext.setBornTimeStamp(requestHeader.getBornTimestamp()); + + Map properties = MessageDecoder.string2messageProperties(requestHeader.getProperties()); + String uniqueKey = properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + properties.put(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); + properties.put(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); + requestHeader.setProperties(MessageDecoder.messageProperties2String(properties)); + + + if (uniqueKey == null) { + uniqueKey = ""; + } + mqtraceContext.setMsgUniqueKey(uniqueKey); + return mqtraceContext; + } + + public boolean hasSendMessageHook() { + return sendMessageHookList != null && !this.sendMessageHookList.isEmpty(); + } + + protected MessageExtBrokerInner buildInnerMsg(final ChannelHandlerContext ctx, + final SendMessageRequestHeader requestHeader, final byte[] body, TopicConfig topicConfig) { + int queueIdInt = requestHeader.getQueueId(); + if (queueIdInt < 0) { + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); + } + int sysFlag = requestHeader.getSysFlag(); + + if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) { + sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG; + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(requestHeader.getTopic()); + msgInner.setBody(body); + msgInner.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msgInner, + MessageDecoder.string2messageProperties(requestHeader.getProperties())); + msgInner.setPropertiesString(requestHeader.getProperties()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(), + msgInner.getTags())); + + msgInner.setQueueId(queueIdInt); + msgInner.setSysFlag(sysFlag); + msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader + .getReconsumeTimes()); + return msgInner; + } + + public SocketAddress getStoreHost() { + return storeHost; + } + + protected RemotingCommand msgContentCheck(final ChannelHandlerContext ctx, + final SendMessageRequestHeader requestHeader, RemotingCommand request, + final RemotingCommand response) { + if (requestHeader.getTopic().length() > Byte.MAX_VALUE) { + log.warn("putMessage message topic length too long " + requestHeader.getTopic().length()); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + return response; + } + if (requestHeader.getProperties() != null && requestHeader.getProperties().length() > Short.MAX_VALUE) { + log.warn("putMessage message properties length too long " + + requestHeader.getProperties().length()); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + return response; + } + if (request.getBody().length > DBMsgConstants.MAX_BODY_SIZE) { + log.warn(" topic {} msg body size {} from {}", requestHeader.getTopic(), + request.getBody().length, ChannelUtil.getRemoteIp(ctx.channel())); + response.setRemark("msg body must be less 64KB"); + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + return response; + } + return response; + } + + protected RemotingCommand msgCheck(final ChannelHandlerContext ctx, + final SendMessageRequestHeader requestHeader, final RemotingCommand response) { + if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission()) + && this.brokerController.getTopicConfigManager().isOrderTopic(requestHeader.getTopic())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending message is forbidden"); + return response; + } + if (!this.brokerController.getTopicConfigManager().isTopicCanSendMessage(requestHeader.getTopic())) { + String errorMsg = + "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; + log.warn(errorMsg); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorMsg); + return response; + } + + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + int topicSysFlag = 0; + if (requestHeader.isUnitMode()) { + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } else { + topicSysFlag = TopicSysFlag.buildSysFlag(true, false); + } + } + + log.warn("the topic " + requestHeader.getTopic() + " not exist, producer: " + + ctx.channel().remoteAddress()); + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageMethod(// + requestHeader.getTopic(), // + requestHeader.getDefaultTopic(), // + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.getDefaultTopicQueueNums(), topicSysFlag); + + if (null == topicConfig) { + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + topicConfig = + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + requestHeader.getTopic(), 1, PermName.PERM_WRITE | PermName.PERM_READ, + topicSysFlag); + } + } + + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + requestHeader.getTopic() + "] not exist, apply first please!" + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + } + + int queueIdInt = requestHeader.getQueueId(); + int idValid = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums()); + if (queueIdInt >= idValid) { + String errorInfo = String.format("request queueId[%d] is illagal, %s Producer: %s", + queueIdInt, + topicConfig.toString(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + log.warn(errorInfo); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorInfo); + + return response; + } + return response; + } + + public void registerSendMessageHook(List sendMessageHookList) { + this.sendMessageHookList = sendMessageHookList; + } + + protected void doResponse(ChannelHandlerContext ctx, RemotingCommand request, + final RemotingCommand response) { + if (!request.isOnewayRPC()) { + try { + ctx.writeAndFlush(response); + } catch (Throwable e) { + log.error("SendMessageProcessor process request over, but response failed", e); + log.error(request.toString()); + log.error(response.toString()); + } + } + } + + public void executeSendMessageHookBefore(final ChannelHandlerContext ctx, final RemotingCommand request, + SendMessageContext context) { + if (hasSendMessageHook()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + final SendMessageRequestHeader requestHeader = parseRequestHeader(request); + + if (null != requestHeader) { + context.setProducerGroup(requestHeader.getProducerGroup()); + context.setTopic(requestHeader.getTopic()); + context.setBodyLength(request.getBody().length); + context.setMsgProps(requestHeader.getProperties()); + context.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + context.setBrokerAddr(this.brokerController.getBrokerAddr()); + context.setQueueId(requestHeader.getQueueId()); + } + + hook.sendMessageBefore(context); + requestHeader.setProperties(context.getMsgProps()); + } catch (Throwable e) { + } + } + } + } + + protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) + throws RemotingCommandException { + + SendMessageRequestHeaderV2 requestHeaderV2 = null; + SendMessageRequestHeader requestHeader = null; + switch (request.getCode()) { + case RequestCode.SEND_MESSAGE_V2: + requestHeaderV2 = + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + case RequestCode.SEND_MESSAGE: + if (null == requestHeaderV2) { + requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + } else { + requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); + } + default: + break; + } + return requestHeader; + } + + public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) { + if (hasSendMessageHook()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + if (response != null) { + final SendMessageResponseHeader responseHeader = + (SendMessageResponseHeader) response.readCustomHeader(); + context.setMsgId(responseHeader.getMsgId()); + context.setQueueId(responseHeader.getQueueId()); + context.setQueueOffset(responseHeader.getQueueOffset()); + context.setCode(response.getCode()); + context.setErrorMsg(response.getRemark()); + } + hook.sendMessageAfter(context); + } catch (Throwable e) { + + } + } + } + } + + @Override + public boolean rejectRequest() { + return false; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java new file mode 100644 index 0000000000..b45a86690d --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -0,0 +1,1212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.admin.TopicOffset; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageId; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.header.*; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.stats.StatsItem; +import com.alibaba.rocketmq.common.stats.StatsSnapshot; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + * @author manhong.yqd + */ +public class AdminBrokerProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + + public AdminBrokerProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.UPDATE_AND_CREATE_TOPIC: + return this.updateAndCreateTopic(ctx, request); + case RequestCode.DELETE_TOPIC_IN_BROKER: + return this.deleteTopic(ctx, request); + case RequestCode.GET_ALL_TOPIC_CONFIG: + return this.getAllTopicConfig(ctx, request); + case RequestCode.UPDATE_BROKER_CONFIG: + return this.updateBrokerConfig(ctx, request); + case RequestCode.GET_BROKER_CONFIG: + return this.getBrokerConfig(ctx, request); + case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP: + return this.searchOffsetByTimestamp(ctx, request); + case RequestCode.GET_MAX_OFFSET: + return this.getMaxOffset(ctx, request); + case RequestCode.GET_MIN_OFFSET: + return this.getMinOffset(ctx, request); + case RequestCode.GET_EARLIEST_MSG_STORETIME: + return this.getEarliestMsgStoretime(ctx, request); + case RequestCode.GET_BROKER_RUNTIME_INFO: + return this.getBrokerRuntimeInfo(ctx, request); + case RequestCode.LOCK_BATCH_MQ: + return this.lockBatchMQ(ctx, request); + case RequestCode.UNLOCK_BATCH_MQ: + return this.unlockBatchMQ(ctx, request); + case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP: + return this.updateAndCreateSubscriptionGroup(ctx, request); + case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG: + return this.getAllSubscriptionGroup(ctx, request); + case RequestCode.DELETE_SUBSCRIPTIONGROUP: + return this.deleteSubscriptionGroup(ctx, request); + case RequestCode.GET_TOPIC_STATS_INFO: + return this.getTopicStatsInfo(ctx, request); + case RequestCode.GET_CONSUMER_CONNECTION_LIST: + return this.getConsumerConnectionList(ctx, request); + case RequestCode.GET_PRODUCER_CONNECTION_LIST: + return this.getProducerConnectionList(ctx, request); + case RequestCode.GET_CONSUME_STATS: + return this.getConsumeStats(ctx, request); + case RequestCode.GET_ALL_CONSUMER_OFFSET: + return this.getAllConsumerOffset(ctx, request); + case RequestCode.GET_ALL_DELAY_OFFSET: + return this.getAllDelayOffset(ctx, request); + case RequestCode.INVOKE_BROKER_TO_RESET_OFFSET: + return this.resetOffset(ctx, request); + case RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS: + return this.getConsumerStatus(ctx, request); + case RequestCode.QUERY_TOPIC_CONSUME_BY_WHO: + return this.queryTopicConsumeByWho(ctx, request); + case RequestCode.REGISTER_FILTER_SERVER: + return this.registerFilterServer(ctx, request); + case RequestCode.QUERY_CONSUME_TIME_SPAN: + return this.queryConsumeTimeSpan(ctx, request); + case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER: + return this.getSystemTopicListFromBroker(ctx, request); + case RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE: + return this.cleanExpiredConsumeQueue(); + case RequestCode.CLEAN_UNUSED_TOPIC: + return this.cleanUnusedTopic(); + case RequestCode.GET_CONSUMER_RUNNING_INFO: + return this.getConsumerRunningInfo(ctx, request); + case RequestCode.QUERY_CORRECTION_OFFSET: + return this.queryCorrectionOffset(ctx, request); + case RequestCode.CONSUME_MESSAGE_DIRECTLY: + return this.consumeMessageDirectly(ctx, request); + case RequestCode.CLONE_GROUP_OFFSET: + return this.cloneGroupOffset(ctx, request); + case RequestCode.VIEW_BROKER_STATS_DATA: + return ViewBrokerStatsData(ctx, request); + case RequestCode.GET_BROKER_CONSUME_STATS: + return fetchAllConsumeStatsInBroker(ctx, request); + default: + break; + } + + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final CreateTopicRequestHeader requestHeader = + (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); + log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + + if (requestHeader.getTopic().equals(this.brokerController.getBrokerConfig().getBrokerClusterName())) { + String errorMsg = "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; + log.warn(errorMsg); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorMsg); + return response; + } + + try { + response.setCode(ResponseCode.SUCCESS); + response.setOpaque(request.getOpaque()); + response.markResponseType(); + response.setRemark(null); + ctx.writeAndFlush(response); + } catch (Exception e) { + } + + TopicConfig topicConfig = new TopicConfig(requestHeader.getTopic()); + topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); + topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); + topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); + topicConfig.setPerm(requestHeader.getPerm()); + topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); + + this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); + this.brokerController.registerBrokerAll(false, true); + return null; + } + + private RemotingCommand deleteTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DeleteTopicRequestHeader requestHeader = + (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); + + log.info("deleteTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); + this.brokerController.getMessageStore() + .cleanUnusedTopic(this.brokerController.getTopicConfigManager().getTopicConfigTable().keySet()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class); + // final GetAllTopicConfigResponseHeader responseHeader = + // (GetAllTopicConfigResponseHeader) response.readCustomHeader(); + + String content = this.brokerController.getTopicConfigManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } else { + log.error("No topic in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No topic in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + log.info("updateBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + byte[] body = request.getBody(); + if (body != null) { + try { + String bodyStr = new String(body, MixAll.DEFAULT_CHARSET); + Properties properties = MixAll.string2Properties(bodyStr); + if (properties != null) { + log.info("updateBrokerConfig, new config: " + properties + " client: " + ctx.channel().remoteAddress()); + this.brokerController.getConfiguration().update(properties); + if (properties.containsKey("brokerPermission")) { + this.brokerController.registerBrokerAll(false, false); + this.brokerController.getTopicConfigManager().getDataVersion().nextVersion(); + } + } else { + log.error("string2Properties error"); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("string2Properties error"); + return response; + } + } catch (UnsupportedEncodingException e) { + log.error("", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class); + final GetBrokerConfigResponseHeader responseHeader = (GetBrokerConfigResponseHeader) response.readCustomHeader(); + + String content = this.brokerController.getConfiguration().getAllConfigsFormatString(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + responseHeader.setVersion(this.brokerController.getConfiguration().getDataVersionJson()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand searchOffsetByTimestamp(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + final SearchOffsetRequestHeader requestHeader = + (SearchOffsetRequestHeader) request.decodeCommandCustomHeader(SearchOffsetRequestHeader.class); + + long offset = this.brokerController.getMessageStore().getOffsetInQueueByTime(requestHeader.getTopic(), requestHeader.getQueueId(), + requestHeader.getTimestamp()); + + responseHeader.setOffset(offset); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); + final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); + final GetMaxOffsetRequestHeader requestHeader = + (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); + + long offset = this.brokerController.getMessageStore().getMaxOffsetInQuque(requestHeader.getTopic(), requestHeader.getQueueId()); + + responseHeader.setOffset(offset); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getMinOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class); + final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader(); + final GetMinOffsetRequestHeader requestHeader = + (GetMinOffsetRequestHeader) request.decodeCommandCustomHeader(GetMinOffsetRequestHeader.class); + + long offset = this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(), requestHeader.getQueueId()); + + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class); + final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader(); + final GetEarliestMsgStoretimeRequestHeader requestHeader = + (GetEarliestMsgStoretimeRequestHeader) request.decodeCommandCustomHeader(GetEarliestMsgStoretimeRequestHeader.class); + + long timestamp = + this.brokerController.getMessageStore().getEarliestMessageTime(requestHeader.getTopic(), requestHeader.getQueueId()); + + responseHeader.setTimestamp(timestamp); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + HashMap runtimeInfo = this.prepareRuntimeInfo(); + KVTable kvTable = new KVTable(); + kvTable.setTable(runtimeInfo); + + byte[] body = kvTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand lockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + LockBatchRequestBody requestBody = LockBatchRequestBody.decode(request.getBody(), LockBatchRequestBody.class); + + Set lockOKMQSet = this.brokerController.getRebalanceLockManager().tryLockBatch(// + requestBody.getConsumerGroup(), // + requestBody.getMqSet(), // + requestBody.getClientId()); + + LockBatchResponseBody responseBody = new LockBatchResponseBody(); + responseBody.setLockOKMQSet(lockOKMQSet); + + response.setBody(responseBody.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand unlockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + UnlockBatchRequestBody requestBody = UnlockBatchRequestBody.decode(request.getBody(), UnlockBatchRequestBody.class); + + this.brokerController.getRebalanceLockManager().unlockBatch(// + requestBody.getConsumerGroup(), // + requestBody.getMqSet(), // + requestBody.getClientId()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + log.info("updateAndCreateSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + SubscriptionGroupConfig config = RemotingSerializable.decode(request.getBody(), SubscriptionGroupConfig.class); + if (config != null) { + this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getAllSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + String content = this.brokerController.getSubscriptionGroupManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } else { + log.error("No subscription group in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No subscription group in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DeleteSubscriptionGroupRequestHeader requestHeader = + (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); + + log.info("deleteSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + this.brokerController.getSubscriptionGroupManager().deleteSubscriptionGroupConfig(requestHeader.getGroupName()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetTopicStatsInfoRequestHeader requestHeader = + (GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); + + final String topic = requestHeader.getTopic(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + topic + "] not exist"); + return response; + } + + TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + + TopicOffset topicOffset = new TopicOffset(); + long min = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, i); + if (min < 0) + min = 0; + + long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + if (max < 0) + max = 0; + + long timestamp = 0; + if (max > 0) { + timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); + } + + topicOffset.setMinOffset(min); + topicOffset.setMaxOffset(max); + topicOffset.setLastUpdateTimestamp(timestamp); + + topicStatsTable.getOffsetTable().put(mq, topicOffset); + } + + byte[] body = topicStatsTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getConsumerConnectionList(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerConnectionListRequestHeader requestHeader = + (GetConsumerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetConsumerConnectionListRequestHeader.class); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup()); + if (consumerGroupInfo != null) { + ConsumerConnection bodydata = new ConsumerConnection(); + bodydata.setConsumeFromWhere(consumerGroupInfo.getConsumeFromWhere()); + bodydata.setConsumeType(consumerGroupInfo.getConsumeType()); + bodydata.setMessageModel(consumerGroupInfo.getMessageModel()); + bodydata.getSubscriptionTable().putAll(consumerGroupInfo.getSubscriptionTable()); + + Iterator> it = consumerGroupInfo.getChannelInfoTable().entrySet().iterator(); + while (it.hasNext()) { + ClientChannelInfo info = it.next().getValue(); + Connection connection = new Connection(); + connection.setClientId(info.getClientId()); + connection.setLanguage(info.getLanguage()); + connection.setVersion(info.getVersion()); + connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel())); + + bodydata.getConnectionSet().add(connection); + } + + byte[] body = bodydata.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + response.setCode(ResponseCode.CONSUMER_NOT_ONLINE); + response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] not online"); + return response; + } + + private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetProducerConnectionListRequestHeader requestHeader = + (GetProducerConnectionListRequestHeader) request.decodeCommandCustomHeader(GetProducerConnectionListRequestHeader.class); + + ProducerConnection bodydata = new ProducerConnection(); + HashMap channelInfoHashMap = + this.brokerController.getProducerManager().getGroupChannelTable().get(requestHeader.getProducerGroup()); + if (channelInfoHashMap != null) { + Iterator> it = channelInfoHashMap.entrySet().iterator(); + while (it.hasNext()) { + ClientChannelInfo info = it.next().getValue(); + Connection connection = new Connection(); + connection.setClientId(info.getClientId()); + connection.setLanguage(info.getLanguage()); + connection.setVersion(info.getVersion()); + connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel())); + + bodydata.getConnectionSet().add(connection); + } + + byte[] body = bodydata.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the producer group[" + requestHeader.getProducerGroup() + "] not exist"); + return response; + } + + private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumeStatsRequestHeader requestHeader = + (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); + + ConsumeStats consumeStats = new ConsumeStats(); + + Set topics = new HashSet(); + if (UtilAll.isBlank(requestHeader.getTopic())) { + topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup()); + } else { + topics.add(requestHeader.getTopic()); + } + + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.warn("consumeStats, topic config not exist, {}", topic); + continue; + } + + /** + + */ + { + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); + + if (null == findSubscriptionData // + && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) { + log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", requestHeader.getConsumerGroup(), topic); + continue; + } + } + + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + + OffsetWrapper offsetWrapper = new OffsetWrapper(); + + long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + if (brokerOffset < 0) + brokerOffset = 0; + + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(// + requestHeader.getConsumerGroup(), // + topic, // + i); + if (consumerOffset < 0) + consumerOffset = 0; + + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + + + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } + } + + consumeStats.getOffsetTable().put(mq, offsetWrapper); + } + + double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic); + + consumeTps += consumeStats.getConsumeTps(); + consumeStats.setConsumeTps(consumeTps); + } + + byte[] body = consumeStats.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getAllConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + String content = this.brokerController.getConsumerOffsetManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("get all consumer offset from master error.", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } else { + log.error("No consumer offset in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No consumer offset in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + private RemotingCommand getAllDelayOffset(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + String content = ((DefaultMessageStore) this.brokerController.getMessageStore()).getScheduleMessageService().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("get all delay offset from master error.", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } else { + log.error("No delay offset in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No delay offset in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + public RemotingCommand resetOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final ResetOffsetRequestHeader requestHeader = + (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class); + log.info("[reset-offset] reset offset started by {}. topic={}, group={}, timestamp={}, isForce={}", + new Object[]{RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup(), + requestHeader.getTimestamp(), requestHeader.isForce()}); + boolean isC = false; + LanguageCode language = request.getLanguage(); + switch (language) { + case CPP: + isC = true; + break; + } + return this.brokerController.getBroker2Client().resetOffset(requestHeader.getTopic(), requestHeader.getGroup(), + requestHeader.getTimestamp(), requestHeader.isForce(), isC); + } + + public RemotingCommand getConsumerStatus(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final GetConsumerStatusRequestHeader requestHeader = + (GetConsumerStatusRequestHeader) request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class); + + log.info("[get-consumer-status] get consumer status by {}. topic={}, group={}", + new Object[]{RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup()}); + + return this.brokerController.getBroker2Client().getConsumeStatus(requestHeader.getTopic(), requestHeader.getGroup(), + requestHeader.getClientAddr()); + } + + private RemotingCommand queryTopicConsumeByWho(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryTopicConsumeByWhoRequestHeader requestHeader = + (QueryTopicConsumeByWhoRequestHeader) request.decodeCommandCustomHeader(QueryTopicConsumeByWhoRequestHeader.class); + + + HashSet groups = this.brokerController.getConsumerManager().queryTopicConsumeByWho(requestHeader.getTopic()); + + Set groupInOffset = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(requestHeader.getTopic()); + if (groupInOffset != null && !groupInOffset.isEmpty()) { + groups.addAll(groupInOffset); + } + + GroupList groupList = new GroupList(); + groupList.setGroupList(groups); + byte[] body = groupList.encode(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand registerFilterServer(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterFilterServerResponseHeader.class); + final RegisterFilterServerResponseHeader responseHeader = (RegisterFilterServerResponseHeader) response.readCustomHeader(); + final RegisterFilterServerRequestHeader requestHeader = + (RegisterFilterServerRequestHeader) request.decodeCommandCustomHeader(RegisterFilterServerRequestHeader.class); + + this.brokerController.getFilterServerManager().registerFilterServer(ctx.channel(), requestHeader.getFilterServerAddr()); + + responseHeader.setBrokerId(this.brokerController.getBrokerConfig().getBrokerId()); + responseHeader.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryConsumeTimeSpanRequestHeader requestHeader = + (QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); + + final String topic = requestHeader.getTopic(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + topic + "] not exist"); + return response; + } + + List timeSpanSet = new ArrayList(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + QueueTimeSpan timeSpan = new QueueTimeSpan(); + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + timeSpan.setMessageQueue(mq); + + long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i); + timeSpan.setMinTimeStamp(minTime); + + long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); + timeSpan.setMaxTimeStamp(maxTime); + + long consumeTime; + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getGroup(), topic, i); + if (consumerOffset > 0) { + consumeTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset - 1); + } else { + consumeTime = minTime; + } + timeSpan.setConsumeTimeStamp(consumeTime); + + long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(requestHeader.getTopic(), i); + if (consumerOffset < maxBrokerOffset) { + long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset); + timeSpan.setDelayTime(System.currentTimeMillis() - nextTime); + } + timeSpanSet.add(timeSpan); + } + + QueryConsumeTimeSpanBody queryConsumeTimeSpanBody = new QueryConsumeTimeSpanBody(); + queryConsumeTimeSpanBody.setConsumeTimeSpanSet(timeSpanSet); + response.setBody(queryConsumeTimeSpanBody.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getSystemTopicListFromBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + Set topics = this.brokerController.getTopicConfigManager().getSystemTopic(); + TopicList topicList = new TopicList(); + topicList.setTopicList(topics); + response.setBody(topicList.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand cleanExpiredConsumeQueue() { + log.warn("invoke cleanExpiredConsumeQueue start."); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + brokerController.getMessageStore().cleanExpiredConsumerQueue(); + log.warn("invoke cleanExpiredConsumeQueue end."); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand cleanUnusedTopic() { + log.warn("invoke cleanUnusedTopic start."); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + brokerController.getMessageStore().cleanUnusedTopic(brokerController.getTopicConfigManager().getTopicConfigTable().keySet()); + log.warn("invoke cleanUnusedTopic end."); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + /** + + */ + private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final GetConsumerRunningInfoRequestHeader requestHeader = + (GetConsumerRunningInfoRequestHeader) request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class); + + return this.callConsumer(RequestCode.GET_CONSUMER_RUNNING_INFO, request, requestHeader.getConsumerGroup(), + requestHeader.getClientId()); + } + + private RemotingCommand queryCorrectionOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryCorrectionOffsetHeader requestHeader = + (QueryCorrectionOffsetHeader) request.decodeCommandCustomHeader(QueryCorrectionOffsetHeader.class); + + Map correctionOffset = this.brokerController.getConsumerOffsetManager() + .queryMinOffsetInAllGroup(requestHeader.getTopic(), requestHeader.getFilterGroups()); + + Map compareOffset = + this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getTopic(), requestHeader.getCompareGroup()); + + if (compareOffset != null && !compareOffset.isEmpty()) { + for (Map.Entry entry : compareOffset.entrySet()) { + Integer queueId = entry.getKey(); + correctionOffset.put(queueId, + correctionOffset.get(queueId) > entry.getValue() ? Long.MAX_VALUE : correctionOffset.get(queueId)); + } + } + + QueryCorrectionOffsetBody body = new QueryCorrectionOffsetBody(); + body.setCorrectionOffsets(correctionOffset); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final ConsumeMessageDirectlyResultRequestHeader requestHeader = (ConsumeMessageDirectlyResultRequestHeader) request + .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class); + + request.getExtFields().put("brokerName", this.brokerController.getBrokerConfig().getBrokerName()); + SelectMappedBufferResult selectMappedBufferResult = null; + try { + MessageId messageId = MessageDecoder.decodeMessageId(requestHeader.getMsgId()); + selectMappedBufferResult = this.brokerController.getMessageStore().selectOneMessageByOffset(messageId.getOffset()); + + byte[] body = new byte[selectMappedBufferResult.getSize()]; + selectMappedBufferResult.getByteBuffer().get(body); + request.setBody(body); + } catch (UnknownHostException e) { + } finally { + if (selectMappedBufferResult != null) { + selectMappedBufferResult.release(); + } + } + + return this.callConsumer(RequestCode.CONSUME_MESSAGE_DIRECTLY, request, requestHeader.getConsumerGroup(), + requestHeader.getClientId()); + } + + private RemotingCommand cloneGroupOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + CloneGroupOffsetRequestHeader requestHeader = + (CloneGroupOffsetRequestHeader) request.decodeCommandCustomHeader(CloneGroupOffsetRequestHeader.class); + + Set topics; + if (UtilAll.isBlank(requestHeader.getTopic())) { + topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getSrcGroup()); + } else { + topics = new HashSet(); + topics.add(requestHeader.getTopic()); + } + + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.warn("[cloneGroupOffset], topic config not exist, {}", topic); + continue; + } + + /** + + */ + if (!requestHeader.isOffline()) { + + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getSrcGroup(), topic); + if (this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getSrcGroup()) > 0 + && findSubscriptionData == null) { + log.warn("[cloneGroupOffset], the consumer group[{}], topic[{}] not exist", requestHeader.getSrcGroup(), topic); + continue; + } + } + + this.brokerController.getConsumerOffsetManager().cloneOffset(requestHeader.getSrcGroup(), requestHeader.getDestGroup(), + requestHeader.getTopic()); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final ViewBrokerStatsDataRequestHeader requestHeader = + (ViewBrokerStatsDataRequestHeader) request.decodeCommandCustomHeader(ViewBrokerStatsDataRequestHeader.class); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DefaultMessageStore messageStore = (DefaultMessageStore) this.brokerController.getMessageStore(); + + StatsItem statsItem = messageStore.getBrokerStatsManager().getStatsItem(requestHeader.getStatsName(), requestHeader.getStatsKey()); + if (null == statsItem) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The stats <%s> <%s> not exist", requestHeader.getStatsName(), requestHeader.getStatsKey())); + return response; + } + + BrokerStatsData brokerStatsData = new BrokerStatsData(); + + { + BrokerStatsItem it = new BrokerStatsItem(); + StatsSnapshot ss = statsItem.getStatsDataInMinute(); + it.setSum(ss.getSum()); + it.setTps(ss.getTps()); + it.setAvgpt(ss.getAvgpt()); + brokerStatsData.setStatsMinute(it); + } + + + { + BrokerStatsItem it = new BrokerStatsItem(); + StatsSnapshot ss = statsItem.getStatsDataInHour(); + it.setSum(ss.getSum()); + it.setTps(ss.getTps()); + it.setAvgpt(ss.getAvgpt()); + brokerStatsData.setStatsHour(it); + } + + + { + BrokerStatsItem it = new BrokerStatsItem(); + StatsSnapshot ss = statsItem.getStatsDataInDay(); + it.setSum(ss.getSum()); + it.setTps(ss.getTps()); + it.setAvgpt(ss.getAvgpt()); + brokerStatsData.setStatsDay(it); + } + + response.setBody(brokerStatsData.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + GetConsumeStatsInBrokerHeader requestHeader = + (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class); + boolean isOrder = requestHeader.isOrder(); + ConcurrentHashMap subscriptionGroups = + brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable(); + + List>> brokerConsumeStatsList = + new ArrayList>>(); + + long totalDiff = 0L; + + for (String group : subscriptionGroups.keySet()) { + Map> subscripTopicConsumeMap = new HashMap>(); + Set topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group); + List consumeStatsList = new ArrayList(); + for (String topic : topics) { + ConsumeStats consumeStats = new ConsumeStats(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.warn("consumeStats, topic config not exist, {}", topic); + continue; + } + + if (isOrder && !topicConfig.isOrder()) { + continue; + } + /** + + */ + { + SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic); + + if (null == findSubscriptionData // + && this.brokerController.getConsumerManager().findSubscriptionDataCount(group) > 0) { + log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", group, topic); + continue; + } + } + + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + OffsetWrapper offsetWrapper = new OffsetWrapper(); + long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + if (brokerOffset < 0) + brokerOffset = 0; + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(// + group, // + topic, // + i); + if (consumerOffset < 0) + consumerOffset = 0; + + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + + + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } + } + consumeStats.getOffsetTable().put(mq, offsetWrapper); + } + double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(group, topic); + consumeTps += consumeStats.getConsumeTps(); + consumeStats.setConsumeTps(consumeTps); + totalDiff += consumeStats.computeTotalDiff(); + consumeStatsList.add(consumeStats); + } + subscripTopicConsumeMap.put(group, consumeStatsList); + brokerConsumeStatsList.add(subscripTopicConsumeMap); + } + ConsumeStatsList consumeStats = new ConsumeStatsList(); + consumeStats.setBrokerAddr(brokerController.getBrokerAddr()); + consumeStats.setConsumeStatsList(brokerConsumeStatsList); + consumeStats.setTotalDiff(totalDiff); + response.setBody(consumeStats.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private HashMap prepareRuntimeInfo() { + HashMap runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo(); + runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)); + runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION)); + + runtimeInfo.put("msgPutTotalYesterdayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalYesterdayMorning())); + runtimeInfo.put("msgPutTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayMorning())); + runtimeInfo.put("msgPutTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayNow())); + + runtimeInfo.put("msgGetTotalYesterdayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalYesterdayMorning())); + runtimeInfo.put("msgGetTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayMorning())); + runtimeInfo.put("msgGetTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayNow())); + + runtimeInfo.put("sendThreadPoolQueueSize", String.valueOf(this.brokerController.getSendThreadPoolQueue().size())); + + runtimeInfo.put("sendThreadPoolQueueCapacity", + String.valueOf(this.brokerController.getBrokerConfig().getSendThreadPoolQueueCapacity())); + + runtimeInfo.put("pullThreadPoolQueueSize", String.valueOf(this.brokerController.getPullThreadPoolQueue().size())); + runtimeInfo.put("pullThreadPoolQueueCapacity", + String.valueOf(this.brokerController.getBrokerConfig().getPullThreadPoolQueueCapacity())); + + runtimeInfo.put("dispatchBehindBytes", String.valueOf(this.brokerController.getMessageStore().dispatchBehindBytes())); + runtimeInfo.put("pageCacheLockTimeMills", String.valueOf(this.brokerController.getMessageStore().lockTimeMills())); + + runtimeInfo.put("sendThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4SendThreadPoolQueue())); + runtimeInfo.put("pullThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4PullThreadPoolQueue())); + runtimeInfo.put("earliestMessageTimeStamp", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime())); + runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp())); + if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) { + DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore(); + runtimeInfo.put("remainTransientStoreBufferNumbs", String.valueOf(defaultMessageStore.remainTransientStoreBufferNumbs())); + if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + runtimeInfo.put("remainHowManyDataToCommit", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToCommit(), false)); + } + runtimeInfo.put("remainHowManyDataToFlush", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToFlush(), false)); + } + + java.io.File commitLogDir = new java.io.File(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + if (commitLogDir.exists()) { + runtimeInfo.put("commitLogDirCapacity", String.format("Total : %s, Free : %s.", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getFreeSpace(), false))); + } + + return runtimeInfo; + } + + private RemotingCommand callConsumer(// + final int requestCode, // + final RemotingCommand request, // + final String consumerGroup, // + final String clientId) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + ClientChannelInfo clientChannelInfo = this.brokerController.getConsumerManager().findChannel(consumerGroup, clientId); + + if (null == clientChannelInfo) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer <%s> <%s> not online", consumerGroup, clientId)); + return response; + } + + if (clientChannelInfo.getVersion() < MQVersion.Version.V3_1_8_SNAPSHOT.ordinal()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer <%s> Version <%s> too low to finish, please upgrade it to V3_1_8_SNAPSHOT", // + clientId, // + MQVersion.getVersionDesc(clientChannelInfo.getVersion()))); + return response; + } + + try { + RemotingCommand newRequest = RemotingCommand.createRequestCommand(requestCode, null); + newRequest.setExtFields(request.getExtFields()); + newRequest.setBody(request.getBody()); + + RemotingCommand consumerResponse = + this.brokerController.getBroker2Client().callClient(clientChannelInfo.getChannel(), newRequest); + return consumerResponse; + } catch (RemotingTimeoutException e) { + response.setCode(ResponseCode.CONSUME_MSG_TIMEOUT); + response + .setRemark(String.format("consumer <%s> <%s> Timeout: %s", consumerGroup, clientId, RemotingHelper.exceptionSimpleDesc(e))); + return response; + } catch (Exception e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark( + String.format("invoke consumer <%s> <%s> Exception: %s", consumerGroup, clientId, RemotingHelper.exceptionSimpleDesc(e))); + return response; + } + } + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java new file mode 100644 index 0000000000..254e63cfa7 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.UnregisterClientRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UnregisterClientResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumerData; +import com.alibaba.rocketmq.common.protocol.heartbeat.HeartbeatData; +import com.alibaba.rocketmq.common.protocol.heartbeat.ProducerData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class ClientManageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + + public ClientManageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.HEART_BEAT: + return this.heartBeat(ctx, request); + case RequestCode.UNREGISTER_CLIENT: + return this.unregisterClient(ctx, request); + default: + break; + } + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + public RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request) { + RemotingCommand response = RemotingCommand.createResponseCommand(null); + HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + ctx.channel(), + heartbeatData.getClientID(), + request.getLanguage(), + request.getVersion() + ); + + for (ConsumerData data : heartbeatData.getConsumerDataSet()) { + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig( + data.getGroupName()); + boolean isNotifyConsumerIdsChangedEnable = true; + if (null != subscriptionGroupConfig) { + isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable(); + int topicSysFlag = 0; + if (data.isUnitMode()) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } + String newTopic = MixAll.getRetryTopic(data.getGroupName()); + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + newTopic, + subscriptionGroupConfig.getRetryQueueNums(), + PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag); + } + + boolean changed = this.brokerController.getConsumerManager().registerConsumer( + data.getGroupName(), + clientChannelInfo, + data.getConsumeType(), + data.getMessageModel(), + data.getConsumeFromWhere(), + data.getSubscriptionDataSet(), + isNotifyConsumerIdsChangedEnable + ); + + if (changed) { + log.info("registerConsumer info changed {} {}", + data.toString(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + ); + } + } + + for (ProducerData data : heartbeatData.getProducerDataSet()) { + this.brokerController.getProducerManager().registerProducer(data.getGroupName(), + clientChannelInfo); + } + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(UnregisterClientResponseHeader.class); + final UnregisterClientRequestHeader requestHeader = + (UnregisterClientRequestHeader) request + .decodeCommandCustomHeader(UnregisterClientRequestHeader.class); + + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + ctx.channel(), + requestHeader.getClientID(), + request.getLanguage(), + request.getVersion()); + { + final String group = requestHeader.getProducerGroup(); + if (group != null) { + this.brokerController.getProducerManager().unregisterProducer(group, clientChannelInfo); + } + } + + { + final String group = requestHeader.getConsumerGroup(); + if (group != null) { + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(group); + boolean isNotifyConsumerIdsChangedEnable = true; + if (null != subscriptionGroupConfig) { + isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable(); + } + this.brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, isNotifyConsumerIdsChangedEnable); + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ConsumerManageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ConsumerManageProcessor.java new file mode 100644 index 0000000000..d7f91989df --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ConsumerManageProcessor.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.*; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class ConsumerManageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private final BrokerController brokerController; + + + public ConsumerManageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.GET_CONSUMER_LIST_BY_GROUP: + return this.getConsumerListByGroup(ctx, request); + case RequestCode.UPDATE_CONSUMER_OFFSET: + return this.updateConsumerOffset(ctx, request); + case RequestCode.QUERY_CONSUMER_OFFSET: + return this.queryConsumerOffset(ctx, request); + default: + break; + } + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + + public RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class); + final GetConsumerListByGroupRequestHeader requestHeader = + (GetConsumerListByGroupRequestHeader) request + .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo( + requestHeader.getConsumerGroup()); + if (consumerGroupInfo != null) { + List clientIds = consumerGroupInfo.getAllClientId(); + if (!clientIds.isEmpty()) { + GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody(); + body.setConsumerIdList(clientIds); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } else { + log.warn("getAllClientId failed, {} {}", requestHeader.getConsumerGroup(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + } + } else { + log.warn("getConsumerGroupInfo failed, {} {}", requestHeader.getConsumerGroup(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("no consumer for this group, " + requestHeader.getConsumerGroup()); + return response; + } + + private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class); + final UpdateConsumerOffsetRequestHeader requestHeader = + (UpdateConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); + this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class); + final QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.readCustomHeader(); + final QueryConsumerOffsetRequestHeader requestHeader = + (QueryConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class); + + long offset = + this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId()); + + + if (offset >= 0) { + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } else { + long minOffset = + this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(), + requestHeader.getQueueId()); + if (minOffset <= 0 + && !this.brokerController.getMessageStore().checkInDiskByConsumeOffset( + requestHeader.getTopic(), requestHeader.getQueueId(), 0)) { + responseHeader.setOffset(0L); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } else { + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("Not found, V3_0_6_SNAPSHOT maybe this group consumer boot first"); + } + } + + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java new file mode 100644 index 0000000000..ed10f1bce2 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.MessageStore; +import com.alibaba.rocketmq.store.PutMessageResult; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class EndTransactionProcessor implements NettyRequestProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private final BrokerController brokerController; + + public EndTransactionProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final EndTransactionRequestHeader requestHeader = + (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); + + + if (requestHeader.getFromTransactionCheck()) { + switch (requestHeader.getCommitOrRollback()) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: { + LOGGER.warn("check producer[{}] transaction state, but it's pending status." + + "RequestHeader: {} Remark: {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.toString(), + request.getRemark()); + return null; + } + + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: { + LOGGER.warn("check producer[{}] transaction state, the producer commit the message." + + "RequestHeader: {} Remark: {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.toString(), + request.getRemark()); + + break; + } + + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { + LOGGER.warn("check producer[{}] transaction state, the producer rollback the message." + + "RequestHeader: {} Remark: {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.toString(), + request.getRemark()); + break; + } + default: + return null; + } + } else { + switch (requestHeader.getCommitOrRollback()) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: { + LOGGER.warn("the producer[{}] end transaction in sending message, and it's pending status." + + "RequestHeader: {} Remark: {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.toString(), + request.getRemark()); + return null; + } + + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: { + break; + } + + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: { + LOGGER.warn("the producer[{}] end transaction in sending message, rollback the message." + + "RequestHeader: {} Remark: {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.toString(), + request.getRemark()); + break; + } + default: + return null; + } + } + + final MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getCommitLogOffset()); + if (msgExt != null) { + final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + if (!pgroupRead.equals(requestHeader.getProducerGroup())) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the producer group wrong"); + return response; + } + + if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the transaction state table offset wrong"); + return response; + } + + if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the commit log offset wrong"); + return response; + } + + MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt); + msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback())); + + msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); + msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); + msgInner.setStoreTimestamp(msgExt.getStoreTimestamp()); + if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) { + msgInner.setBody(null); + } + + final MessageStore messageStore = this.brokerController.getMessageStore(); + final PutMessageResult putMessageResult = messageStore.putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + break; + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("create maped file failed."); + break; + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark("the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("service not available now."); + break; + case OS_PAGECACHE_BUSY: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("OS page cache busy, please try another machine"); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + + return response; + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("find prepared transaction message failed"); + return response; + } + + return response; + } + + @Override + public boolean rejectRequest() { + return false; + } + + private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + + TopicFilterType topicFilterType = + (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG + : TopicFilterType.SINGLE_TAG; + long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + msgInner.setTagsCode(tagsCodeValue); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + + msgInner.setWaitStoreMsgOK(false); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + + msgInner.setTopic(msgExt.getTopic()); + msgInner.setQueueId(msgExt.getQueueId()); + + return msgInner; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java new file mode 100644 index 0000000000..a92ead06f1 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class ForwardRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private final BrokerController brokerController; + + + public ForwardRequestProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java new file mode 100644 index 0000000000..0152b93a81 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java @@ -0,0 +1,540 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.broker.longpolling.PullRequest; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.pagecache.ManyMessageTransfer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.GetMessageResult; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.PutMessageResult; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import io.netty.channel.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class PullMessageProcessor implements NettyRequestProcessor { + private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + private List consumeMessageHookList; + + public PullMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + return this.processRequest(ctx.channel(), request, true); + } + + @Override + public boolean rejectRequest() { + return false; + } + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend) + throws RemotingCommandException { + RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); + final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader(); + final PullMessageRequestHeader requestHeader = + (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class); + + + response.setOpaque(request.getOpaque()); + + if (LOG.isDebugEnabled()) { + LOG.debug("receive PullMessage request command, " + request); + } + + + if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] pulling message is forbidden"); + return response; + } + + + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup()); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark("subscription group not exist, " + requestHeader.getConsumerGroup() + " " + + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)); + return response; + } + + + if (!subscriptionGroupConfig.isConsumeEnable()) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup()); + return response; + } + + final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag()); + final boolean hasCommitOffsetFlag = PullSysFlag.hasCommitOffsetFlag(requestHeader.getSysFlag()); + final boolean hasSubscriptionFlag = PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag()); + + final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0; + + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + LOG.error("the topic " + requestHeader.getTopic() + " not exist, consumer: " + RemotingHelper.parseChannelRemoteAddr(channel)); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark( + "topic[" + requestHeader.getTopic() + "] not exist, apply first please!" + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + + + if (!PermName.isReadable(topicConfig.getPerm())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden"); + return response; + } + + + if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) { + String errorInfo = "queueId[" + requestHeader.getQueueId() + "] is illagal,Topic :" + requestHeader.getTopic() + + " topicConfig.readQueueNums: " + topicConfig.getReadQueueNums() + " consumer: " + channel.remoteAddress(); + LOG.warn(errorInfo); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorInfo); + return response; + } + + + SubscriptionData subscriptionData = null; + if (hasSubscriptionFlag) { + try { + subscriptionData = FilterAPI.buildSubscriptionData(requestHeader.getConsumerGroup(), requestHeader.getTopic(), + requestHeader.getSubscription()); + } catch (Exception e) { + LOG.warn("parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(), // + requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED); + response.setRemark("parse the consumer's subscription failed"); + return response; + } + } else { + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup()); + if (null == consumerGroupInfo) { + LOG.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); + response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); + return response; + } + + if (!subscriptionGroupConfig.isConsumeBroadcastEnable() // + && consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] can not consume by broadcast way"); + return response; + } + + subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic()); + if (null == subscriptionData) { + LOG.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); + response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); + return response; + } + + + if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) { + LOG.warn("the broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(), + subscriptionData.getSubString()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST); + response.setRemark("the consumer's subscription not latest"); + return response; + } + } + + final GetMessageResult getMessageResult = + this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(), + requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), subscriptionData); + if (getMessageResult != null) { + response.setRemark(getMessageResult.getStatus().name()); + responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset()); + responseHeader.setMinOffset(getMessageResult.getMinOffset()); + responseHeader.setMaxOffset(getMessageResult.getMaxOffset()); + + + if (getMessageResult.isSuggestPullingFromSlave()) { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly()); + } else { + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + } + + switch (this.brokerController.getMessageStoreConfig().getBrokerRole()) { + case ASYNC_MASTER: + case SYNC_MASTER: + break; + case SLAVE: + if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()) { + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + } + break; + } + + if (this.brokerController.getBrokerConfig().isSlaveReadEnable()) { + // consume too slow ,redirect to another machine + if (getMessageResult.isSuggestPullingFromSlave()) { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly()); + } + // consume ok + else { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId()); + } + } else { + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + } + + switch (getMessageResult.getStatus()) { + case FOUND: + response.setCode(ResponseCode.SUCCESS); + break; + case MESSAGE_WAS_REMOVING: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + case NO_MATCHED_LOGIC_QUEUE: + case NO_MESSAGE_IN_QUEUE: + if (0 != requestHeader.getQueueOffset()) { + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + + // XXX: warn and notify me + LOG.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}", // + requestHeader.getQueueOffset(), // + getMessageResult.getNextBeginOffset(), // + requestHeader.getTopic(), // + requestHeader.getQueueId(), // + requestHeader.getConsumerGroup()// + ); + } else { + response.setCode(ResponseCode.PULL_NOT_FOUND); + } + break; + case NO_MATCHED_MESSAGE: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + case OFFSET_FOUND_NULL: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_OVERFLOW_BADLY: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + // XXX: warn and notify me + LOG.info("the request offset: " + requestHeader.getQueueOffset() + " over flow badly, broker max offset: " + + getMessageResult.getMaxOffset() + ", consumer: " + channel.remoteAddress()); + break; + case OFFSET_OVERFLOW_ONE: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_TOO_SMALL: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + LOG.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}", + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(), + getMessageResult.getMinOffset(), channel.remoteAddress()); + break; + default: + assert false; + break; + } + + if (this.hasConsumeMessageHook()) { + ConsumeMessageContext context = new ConsumeMessageContext(); + context.setConsumerGroup(requestHeader.getConsumerGroup()); + context.setTopic(requestHeader.getTopic()); + context.setQueueId(requestHeader.getQueueId()); + + String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); + + switch (response.getCode()) { + case ResponseCode.SUCCESS: + + context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_SUCCESS); + context.setCommercialRcvTimes(getMessageResult.getMsgCount4Commercial()); + context.setCommercialRcvSize(getMessageResult.getBufferTotalSize()); + context.setCommercialOwner(owner); + + break; + case ResponseCode.PULL_NOT_FOUND: + if (!brokerAllowSuspend) { + + + context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS); + context.setCommercialRcvTimes(1); + context.setCommercialOwner(owner); + + } + break; + case ResponseCode.PULL_RETRY_IMMEDIATELY: + case ResponseCode.PULL_OFFSET_MOVED: + context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS); + context.setCommercialRcvTimes(1); + context.setCommercialOwner(owner); + break; + default: + assert false; + break; + } + + this.executeConsumeMessageHookBefore(context); + } + + switch (response.getCode()) { + case ResponseCode.SUCCESS: + + this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), + getMessageResult.getMessageCount()); + + this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(), + getMessageResult.getBufferTotalSize()); + + this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) { + final long beginTimeMills = this.brokerController.getMessageStore().now(); + final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId()); + this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), + (int) (this.brokerController.getMessageStore().now() - beginTimeMills)); + response.setBody(r); + } else { + try { + FileRegion fileRegion = + new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult); + channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + getMessageResult.release(); + if (!future.isSuccess()) { + LOG.error("transfer many message by pagecache failed, " + channel.remoteAddress(), future.cause()); + } + } + }); + } catch (Throwable e) { + LOG.error("transfer many message by pagecache exception", e); + getMessageResult.release(); + } + + response = null; + } + break; + case ResponseCode.PULL_NOT_FOUND: + + if (brokerAllowSuspend && hasSuspendFlag) { + long pollingTimeMills = suspendTimeoutMillisLong; + if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) { + pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills(); + } + + String topic = requestHeader.getTopic(); + long offset = requestHeader.getQueueOffset(); + int queueId = requestHeader.getQueueId(); + PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills, + this.brokerController.getMessageStore().now(), offset, subscriptionData); + this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest); + response = null; + break; + } + + + case ResponseCode.PULL_RETRY_IMMEDIATELY: + break; + case ResponseCode.PULL_OFFSET_MOVED: + if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE + || this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(requestHeader.getTopic()); + mq.setQueueId(requestHeader.getQueueId()); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + + OffsetMovedEvent event = new OffsetMovedEvent(); + event.setConsumerGroup(requestHeader.getConsumerGroup()); + event.setMessageQueue(mq); + event.setOffsetRequest(requestHeader.getQueueOffset()); + event.setOffsetNew(getMessageResult.getNextBeginOffset()); + this.generateOffsetMovedEvent(event); + LOG.warn( + "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}", + requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(), + responseHeader.getSuggestWhichBrokerId()); + } else { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId()); + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + LOG.warn("PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}", + requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(), + responseHeader.getSuggestWhichBrokerId()); + } + + break; + default: + assert false; + } + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store getMessage return null"); + } + + + boolean storeOffsetEnable = brokerAllowSuspend; + storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag; + storeOffsetEnable = storeOffsetEnable + && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; + if (storeOffsetEnable) { + this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel), + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); + } + return response; + } + + + public boolean hasConsumeMessageHook() { + return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); + } + + public void executeConsumeMessageHookBefore(final ConsumeMessageContext context) { + if (hasConsumeMessageHook()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + } + } + } + } + + private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic, final int queueId) { + final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize()); + + long storeTimestamp = 0; + try { + List messageBufferList = getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + + byteBuffer.put(bb); + storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + } + } finally { + getMessageResult.release(); + } + + this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp); + return byteBuffer.array(); + } + + private void generateOffsetMovedEvent(final OffsetMovedEvent event) { + try { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(MixAll.OFFSET_MOVED_EVENT); + msgInner.setTags(event.getConsumerGroup()); + msgInner.setDelayTimeLevel(0); + msgInner.setKeys(event.getConsumerGroup()); + msgInner.setBody(event.encode()); + msgInner.setFlag(0); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(TopicFilterType.SINGLE_TAG, msgInner.getTags())); + + msgInner.setQueueId(0); + msgInner.setSysFlag(0); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(RemotingUtil.string2SocketAddress(this.brokerController.getBrokerAddr())); + msgInner.setStoreHost(msgInner.getBornHost()); + + msgInner.setReconsumeTimes(0); + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + } catch (Exception e) { + LOG.warn(String.format("generateOffsetMovedEvent Exception, %s", event.toString()), e); + } + } + + public void excuteRequestWhenWakeup(final Channel channel, final RemotingCommand request) throws RemotingCommandException { + Runnable run = new Runnable() { + @Override + public void run() { + try { + final RemotingCommand response = PullMessageProcessor.this.processRequest(channel, request, false); + + if (response != null) { + response.setOpaque(request.getOpaque()); + response.markResponseType(); + try { + channel.writeAndFlush(response).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (!future.isSuccess()) { + LOG.error("processRequestWrapper response to " + future.channel().remoteAddress() + " failed", + future.cause()); + LOG.error(request.toString()); + LOG.error(response.toString()); + } + } + }); + } catch (Throwable e) { + LOG.error("processRequestWrapper process request over, but response failed", e); + LOG.error(request.toString()); + LOG.error(response.toString()); + } + } + } catch (RemotingCommandException e1) { + LOG.error("excuteRequestWhenWakeup run", e1); + } + } + }; + + this.brokerController.getPullMessageExecutor().submit(run); + } + + public void registerConsumeMessageHook(List sendMessageHookList) { + this.consumeMessageHookList = sendMessageHookList; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java new file mode 100644 index 0000000000..738d11f784 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.pagecache.OneMessageTransfer; +import com.alibaba.rocketmq.broker.pagecache.QueryMessageTransfer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.ViewMessageRequestHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.QueryMessageResult; +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.FileRegion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class QueryMessageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private final BrokerController brokerController; + + + public QueryMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.QUERY_MESSAGE: + return this.queryMessage(ctx, request); + case RequestCode.VIEW_MESSAGE_BY_ID: + return this.viewMessageById(ctx, request); + default: + break; + } + + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + + public RemotingCommand queryMessage(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(QueryMessageResponseHeader.class); + final QueryMessageResponseHeader responseHeader = + (QueryMessageResponseHeader) response.readCustomHeader(); + final QueryMessageRequestHeader requestHeader = + (QueryMessageRequestHeader) request + .decodeCommandCustomHeader(QueryMessageRequestHeader.class); + + + response.setOpaque(request.getOpaque()); + + + String isUniqueKey = request.getExtFields().get(MixAll.UNIQUE_MSG_QUERY_FLAG); + if (isUniqueKey != null && isUniqueKey.equals("true")) { + requestHeader.setMaxNum(this.brokerController.getMessageStoreConfig().getDefaultQueryMaxNum()); + } + + final QueryMessageResult queryMessageResult = + this.brokerController.getMessageStore().queryMessage(requestHeader.getTopic(), + requestHeader.getKey(), requestHeader.getMaxNum(), requestHeader.getBeginTimestamp(), + requestHeader.getEndTimestamp()); + assert queryMessageResult != null; + + responseHeader.setIndexLastUpdatePhyoffset(queryMessageResult.getIndexLastUpdatePhyoffset()); + responseHeader.setIndexLastUpdateTimestamp(queryMessageResult.getIndexLastUpdateTimestamp()); + + + if (queryMessageResult.getBufferTotalSize() > 0) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + try { + FileRegion fileRegion = + new QueryMessageTransfer(response.encodeHeader(queryMessageResult + .getBufferTotalSize()), queryMessageResult); + ctx.channel().writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + queryMessageResult.release(); + if (!future.isSuccess()) { + log.error("transfer query message by pagecache failed, ", future.cause()); + } + } + }); + } catch (Throwable e) { + log.error("", e); + queryMessageResult.release(); + } + + return null; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("can not find message, maybe time range not correct"); + return response; + } + + + public RemotingCommand viewMessageById(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ViewMessageRequestHeader requestHeader = + (ViewMessageRequestHeader) request.decodeCommandCustomHeader(ViewMessageRequestHeader.class); + + + response.setOpaque(request.getOpaque()); + + final SelectMappedBufferResult selectMappedBufferResult = + this.brokerController.getMessageStore().selectOneMessageByOffset(requestHeader.getOffset()); + if (selectMappedBufferResult != null) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + try { + FileRegion fileRegion = + new OneMessageTransfer(response.encodeHeader(selectMappedBufferResult.getSize()), + selectMappedBufferResult); + ctx.channel().writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + selectMappedBufferResult.release(); + if (!future.isSuccess()) { + log.error("transfer one message by pagecache failed, ", future.cause()); + } + } + }); + } catch (Throwable e) { + log.error("", e); + selectMappedBufferResult.release(); + } + + return null; + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("can not find message by the offset, " + requestHeader.getOffset()); + } + + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java new file mode 100644 index 0000000000..414b3f43fb --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java @@ -0,0 +1,496 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageContext; +import com.alibaba.rocketmq.common.*; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageResponseHeader; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.PutMessageResult; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import io.netty.channel.ChannelHandlerContext; + +import java.net.SocketAddress; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { + + private List consumeMessageHookList; + + public SendMessageProcessor(final BrokerController brokerController) { + super(brokerController); + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + SendMessageContext mqtraceContext; + switch (request.getCode()) { + case RequestCode.CONSUMER_SEND_MSG_BACK: + return this.consumerSendMsgBack(ctx, request); + default: + SendMessageRequestHeader requestHeader = parseRequestHeader(request); + if (requestHeader == null) { + return null; + } + + mqtraceContext = buildMsgContext(ctx, requestHeader); + this.executeSendMessageHookBefore(ctx, request, mqtraceContext); + final RemotingCommand response = this.sendMessage(ctx, request, mqtraceContext, requestHeader); + + this.executeSendMessageHookAfter(response, mqtraceContext); + return response; + } + } + + @Override + public boolean rejectRequest() { + return this.brokerController.getMessageStore().isOSPageCacheBusy() || + this.brokerController.getMessageStore().isTransientStorePoolDeficient(); + } + + private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, final RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ConsumerSendMsgBackRequestHeader requestHeader = + (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); + + if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) { + + ConsumeMessageContext context = new ConsumeMessageContext(); + context.setConsumerGroup(requestHeader.getGroup()); + context.setTopic(requestHeader.getOriginTopic()); + context.setCommercialRcvStats(BrokerStatsManager.StatsType.SEND_BACK); + context.setCommercialRcvTimes(1); + context.setCommercialOwner(request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER)); + + this.executeConsumeMessageHookAfter(context); + } + + + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup()); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark("subscription group not exist, " + requestHeader.getGroup() + " " + + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)); + return response; + } + + + if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending message is forbidden"); + return response; + } + + + if (subscriptionGroupConfig.getRetryQueueNums() <= 0) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + String newTopic = MixAll.getRetryTopic(requestHeader.getGroup()); + int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % subscriptionGroupConfig.getRetryQueueNums(); + + + int topicSysFlag = 0; + if (requestHeader.isUnitMode()) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } + + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(// + newTopic, // + subscriptionGroupConfig.getRetryQueueNums(), // + PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag); + if (null == topicConfig) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("topic[" + newTopic + "] not exist"); + return response; + } + + + if (!PermName.isWriteable(topicConfig.getPerm())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark(String.format("the topic[%s] sending message is forbidden", newTopic)); + return response; + } + + MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getOffset()); + if (null == msgExt) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("look message by offset failed, " + requestHeader.getOffset()); + return response; + } + + + final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (null == retryTopic) { + MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic()); + } + msgExt.setWaitStoreMsgOK(false); + + + int delayLevel = requestHeader.getDelayLevel(); + + + int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes(); + if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) { + maxReconsumeTimes = requestHeader.getMaxReconsumeTimes(); + } + + + if (msgExt.getReconsumeTimes() >= maxReconsumeTimes// + || delayLevel < 0) { + newTopic = MixAll.getDLQTopic(requestHeader.getGroup()); + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP; + + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, // + DLQ_NUMS_PER_GROUP, // + PermName.PERM_WRITE, 0 + ); + if (null == topicConfig) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("topic[" + newTopic + "] not exist"); + return response; + } + } else { + if (0 == delayLevel) { + delayLevel = 3 + msgExt.getReconsumeTimes(); + } + + msgExt.setDelayTimeLevel(delayLevel); + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(newTopic); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags())); + + msgInner.setQueueId(queueIdInt); + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1); + + String originMsgId = MessageAccessor.getOriginMessageId(msgExt); + MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId); + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + case PUT_OK: + String backTopic = msgExt.getTopic(); + String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (correctTopic != null) { + backTopic = correctTopic; + } + + this.brokerController.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + default: + break; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(putMessageResult.getPutMessageStatus().name()); + return response; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("putMessageResult is null"); + return response; + } + + private RemotingCommand sendMessage(final ChannelHandlerContext ctx, // + final RemotingCommand request, // + final SendMessageContext sendMessageContext, // + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + + final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + + + response.setOpaque(request.getOpaque()); + + response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); + response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); + + if (log.isDebugEnabled()) { + log.debug("receive SendMessage request command, " + request); + } + + final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); + if (this.brokerController.getMessageStore().now() < startTimstamp) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp))); + return response; + } + + response.setCode(-1); + super.msgCheck(ctx, requestHeader, response); + if (response.getCode() != -1) { + return response; + } + + final byte[] body = request.getBody(); + + int queueIdInt = requestHeader.getQueueId(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + + if (queueIdInt < 0) { + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); + } + + int sysFlag = requestHeader.getSysFlag(); + + if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) { + sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG; + } + + String newTopic = requestHeader.getTopic(); + if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(groupName); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark( + "subscription group not exist, " + groupName + " " + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)); + return response; + } + + + int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes(); + if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) { + maxReconsumeTimes = requestHeader.getMaxReconsumeTimes(); + } + int reconsumeTimes = requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes(); + if (reconsumeTimes >= maxReconsumeTimes) { + newTopic = MixAll.getDLQTopic(groupName); + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP; + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, // + DLQ_NUMS_PER_GROUP, // + PermName.PERM_WRITE, 0 + ); + if (null == topicConfig) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("topic[" + newTopic + "] not exist"); + return response; + } + } + } + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(newTopic); + msgInner.setBody(body); + msgInner.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); + msgInner.setPropertiesString(requestHeader.getProperties()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(), msgInner.getTags())); + + msgInner.setQueueId(queueIdInt); + msgInner.setSysFlag(sysFlag); + msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + + if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { + String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark( + "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); + return response; + } + } + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + boolean sendOK = false; + + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + sendOK = true; + response.setCode(ResponseCode.SUCCESS); + break; + case FLUSH_DISK_TIMEOUT: + response.setCode(ResponseCode.FLUSH_DISK_TIMEOUT); + sendOK = true; + break; + case FLUSH_SLAVE_TIMEOUT: + response.setCode(ResponseCode.FLUSH_SLAVE_TIMEOUT); + sendOK = true; + break; + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + sendOK = true; + break; + + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("create mapped file failed, server is busy or broken."); + break; + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark( + "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark( + "service not available now, maybe disk full, " + diskUtil() + ", maybe your broker machine memory too small."); + break; + case OS_PAGECACHE_BUSY: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + + String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); + if (sendOK) { + + this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic()); + this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + this.brokerController.getBrokerStatsManager().incBrokerPutNums(); + + response.setRemark(null); + + responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + + + doResponse(ctx, request, response); + + + if (hasSendMessageHook()) { + sendMessageContext.setMsgId(responseHeader.getMsgId()); + sendMessageContext.setQueueId(responseHeader.getQueueId()); + sendMessageContext.setQueueOffset(responseHeader.getQueueOffset()); + + int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + return null; + } else { + if (hasSendMessageHook()) { + int wroteSize = request.getBody().length; + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + } + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + + return response; + } + + public boolean hasConsumeMessageHook() { + return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); + } + + public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) { + if (hasConsumeMessageHook()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + } + } + } + } + + public SocketAddress getStoreHost() { + return storeHost; + } + + private String diskUtil() { + String storePathPhysic = this.brokerController.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + + String storePathLogis = + StorePathConfigHelper.getStorePathConsumeQueue(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + double logisRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogis); + + String storePathIndex = + StorePathConfigHelper.getStorePathIndex(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + double indexRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathIndex); + + return String.format("CL: %5.2f CQ: %5.2f INDEX: %5.2f", physicRatio, logisRatio, indexRatio); + } + + public void registerConsumeMessageHook(List consumeMessageHookList) { + this.consumeMessageHookList = consumeMessageHookList; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java new file mode 100644 index 0000000000..8b3aefe429 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.slave; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.subscription.SubscriptionGroupManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.body.SubscriptionGroupWrapper; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + + +/** + * @author shijia.wxr + * @author manhong.yqd + */ +public class SlaveSynchronize { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final BrokerController brokerController; + private volatile String masterAddr = null; + + + public SlaveSynchronize(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } + + + public void syncAll() { + this.syncTopicConfig(); + this.syncConsumerOffset(); + this.syncDelayOffset(); + this.syncSubscriptionGroupConfig(); + } + + + private void syncTopicConfig() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + TopicConfigSerializeWrapper topicWrapper = + this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak); + if (!this.brokerController.getTopicConfigManager().getDataVersion() + .equals(topicWrapper.getDataVersion())) { + + this.brokerController.getTopicConfigManager().getDataVersion() + .assignNewOne(topicWrapper.getDataVersion()); + this.brokerController.getTopicConfigManager().getTopicConfigTable().clear(); + this.brokerController.getTopicConfigManager().getTopicConfigTable() + .putAll(topicWrapper.getTopicConfigTable()); + this.brokerController.getTopicConfigManager().persist(); + + log.info("update slave topic config from master, {}", masterAddrBak); + } + } catch (Exception e) { + log.error("syncTopicConfig Exception, " + masterAddrBak, e); + } + } + } + + + private void syncConsumerOffset() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + ConsumerOffsetSerializeWrapper offsetWrapper = + this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak); + this.brokerController.getConsumerOffsetManager().getOffsetTable() + .putAll(offsetWrapper.getOffsetTable()); + this.brokerController.getConsumerOffsetManager().persist(); + log.info("update slave consumer offset from master, {}", masterAddrBak); + } catch (Exception e) { + log.error("syncConsumerOffset Exception, " + masterAddrBak, e); + } + } + } + + + private void syncDelayOffset() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + String delayOffset = + this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak); + if (delayOffset != null) { + + String fileName = + StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController + .getMessageStoreConfig().getStorePathRootDir()); + try { + MixAll.string2File(delayOffset, fileName); + } catch (IOException e) { + log.error("persist file Exception, " + fileName, e); + } + } + log.info("update slave delay offset from master, {}", masterAddrBak); + } catch (Exception e) { + log.error("syncDelayOffset Exception, " + masterAddrBak, e); + } + } + } + + + private void syncSubscriptionGroupConfig() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + SubscriptionGroupWrapper subscriptionWrapper = + this.brokerController.getBrokerOuterAPI() + .getAllSubscriptionGroupConfig(masterAddrBak); + + if (!this.brokerController.getSubscriptionGroupManager().getDataVersion() + .equals(subscriptionWrapper.getDataVersion())) { + SubscriptionGroupManager subscriptionGroupManager = + this.brokerController.getSubscriptionGroupManager(); + subscriptionGroupManager.getDataVersion().assignNewOne( + subscriptionWrapper.getDataVersion()); + subscriptionGroupManager.getSubscriptionGroupTable().clear(); + subscriptionGroupManager.getSubscriptionGroupTable().putAll( + subscriptionWrapper.getSubscriptionGroupTable()); + subscriptionGroupManager.persist(); + log.info("update slave Subscription Group from master, {}", masterAddrBak); + } + } catch (Exception e) { + log.error("syncSubscriptionGroup Exception, " + masterAddrBak, e); + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java new file mode 100644 index 0000000000..d954a469be --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.subscription; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class SubscriptionGroupManager extends ConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private final ConcurrentHashMap subscriptionGroupTable = + new ConcurrentHashMap(1024); + private final DataVersion dataVersion = new DataVersion(); + private transient BrokerController brokerController; + + + public SubscriptionGroupManager() { + this.init(); + } + + private void init() { + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.TOOLS_CONSUMER_GROUP); + this.subscriptionGroupTable.put(MixAll.TOOLS_CONSUMER_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.FILTERSRV_CONSUMER_GROUP); + this.subscriptionGroupTable.put(MixAll.FILTERSRV_CONSUMER_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.SELF_TEST_CONSUMER_GROUP); + this.subscriptionGroupTable.put(MixAll.SELF_TEST_CONSUMER_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.ONS_HTTP_PROXY_GROUP); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + this.subscriptionGroupTable.put(MixAll.ONS_HTTP_PROXY_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PULL_GROUP); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_PULL_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_PERMISSION_GROUP); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_PERMISSION_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.CID_ONSAPI_OWNER_GROUP); + subscriptionGroupConfig.setConsumeBroadcastEnable(true); + this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_OWNER_GROUP, subscriptionGroupConfig); + } + } + + + public SubscriptionGroupManager(BrokerController brokerController) { + this.brokerController = brokerController; + this.init(); + } + + + public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) { + SubscriptionGroupConfig old = this.subscriptionGroupTable.put(config.getGroupName(), config); + if (old != null) { + log.info("update subscription group config, old: " + old + " new: " + config); + } else { + log.info("create new subscription group, " + config); + } + + this.dataVersion.nextVersion(); + + this.persist(); + } + + public void disableConsume(final String groupName) { + SubscriptionGroupConfig old = this.subscriptionGroupTable.get(groupName); + if (old != null) { + old.setConsumeEnable(false); + this.dataVersion.nextVersion(); + } + } + + + public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) { + SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.get(group); + if (null == subscriptionGroupConfig) { + if (brokerController.getBrokerConfig().isAutoCreateSubscriptionGroup() || MixAll.isSysConsumerGroup(group)) { + subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(group); + SubscriptionGroupConfig preConfig = this.subscriptionGroupTable.putIfAbsent(group, subscriptionGroupConfig); + if (null == preConfig) { + log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString()); + } + this.dataVersion.nextVersion(); + this.persist(); + } + } + + return subscriptionGroupConfig; + } + + + @Override + public String encode() { + return this.encode(false); + } + + @Override + public String configFilePath() { + //return BrokerPathConfigHelper.getSubscriptionGroupPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir()); + return BrokerPathConfigHelper.getSubscriptionGroupPath(System.getProperty("user.home") + File.separator + "store"); + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class); + if (obj != null) { + this.subscriptionGroupTable.putAll(obj.subscriptionGroupTable); + this.dataVersion.assignNewOne(obj.dataVersion); + this.printLoadDataWhenFirstBoot(obj); + } + } + } + + public String encode(final boolean prettyFormat) { + return RemotingSerializable.toJson(this, prettyFormat); + } + + private void printLoadDataWhenFirstBoot(final SubscriptionGroupManager sgm) { + Iterator> it = sgm.getSubscriptionGroupTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("load exist subscription group, {}", next.getValue().toString()); + } + } + + public ConcurrentHashMap getSubscriptionGroupTable() { + return subscriptionGroupTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void deleteSubscriptionGroupConfig(final String groupName) { + SubscriptionGroupConfig old = this.subscriptionGroupTable.remove(groupName); + if (old != null) { + log.info("delete subscription group OK, subscription group: " + old); + this.dataVersion.nextVersion(); + this.persist(); + } else { + log.warn("delete subscription group failed, subscription group: " + old + " not exist"); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java new file mode 100644 index 0000000000..94d7e9fd62 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java @@ -0,0 +1,440 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.topic; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class TopicConfigManager extends ConfigManager { + private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final long LOCK_TIMEOUT_MILLIS = 3000; + private transient final Lock lockTopicConfigTable = new ReentrantLock(); + + private final ConcurrentHashMap topicConfigTable = + new ConcurrentHashMap(1024); + private final DataVersion dataVersion = new DataVersion(); + private final Set systemTopicList = new HashSet(); + private transient BrokerController brokerController; + + + public TopicConfigManager() { + } + + + public TopicConfigManager(BrokerController brokerController) { + this.brokerController = brokerController; + { + // MixAll.SELF_TEST_TOPIC + String topic = MixAll.SELF_TEST_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // MixAll.DEFAULT_TOPIC + if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { + String topic = MixAll.DEFAULT_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig() + .getDefaultTopicQueueNums()); + topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig() + .getDefaultTopicQueueNums()); + int perm = PermName.PERM_INHERIT | PermName.PERM_READ | PermName.PERM_WRITE; + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } + { + // MixAll.BENCHMARK_TOPIC + String topic = MixAll.BENCHMARK_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1024); + topicConfig.setWriteQueueNums(1024); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + + String topic = this.brokerController.getBrokerConfig().getBrokerClusterName(); + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + int perm = PermName.PERM_INHERIT; + if (this.brokerController.getBrokerConfig().isClusterTopicEnable()) { + perm |= PermName.PERM_READ | PermName.PERM_WRITE; + } + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + + String topic = this.brokerController.getBrokerConfig().getBrokerName(); + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + int perm = PermName.PERM_INHERIT; + if (this.brokerController.getBrokerConfig().isBrokerTopicEnable()) { + perm |= PermName.PERM_READ | PermName.PERM_WRITE; + } + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // MixAll.OFFSET_MOVED_EVENT + String topic = MixAll.OFFSET_MOVED_EVENT; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } + + + public boolean isSystemTopic(final String topic) { + return this.systemTopicList.contains(topic); + } + + + public Set getSystemTopic() { + return this.systemTopicList; + } + + + public boolean isTopicCanSendMessage(final String topic) { + return !topic.equals(MixAll.DEFAULT_TOPIC); + } + + + public TopicConfig selectTopicConfig(final String topic) { + return this.topicConfigTable.get(topic); + } + + + public TopicConfig createTopicInSendMessageMethod(final String topic, final String defaultTopic, + final String remoteAddress, final int clientDefaultTopicQueueNums, final int topicSysFlag) { + TopicConfig topicConfig = null; + boolean createNew = false; + + try { + if (this.lockTopicConfigTable.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic); + if (defaultTopicConfig != null) { + if (defaultTopic.equals(MixAll.DEFAULT_TOPIC)) { + if (!this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { + defaultTopicConfig.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); + } + } + + if (PermName.isInherited(defaultTopicConfig.getPerm())) { + topicConfig = new TopicConfig(topic); + + int queueNums = + clientDefaultTopicQueueNums > defaultTopicConfig.getWriteQueueNums() ? defaultTopicConfig + .getWriteQueueNums() : clientDefaultTopicQueueNums; + + if (queueNums < 0) { + queueNums = 0; + } + + topicConfig.setReadQueueNums(queueNums); + topicConfig.setWriteQueueNums(queueNums); + int perm = defaultTopicConfig.getPerm(); + perm &= ~PermName.PERM_INHERIT; + topicConfig.setPerm(perm); + topicConfig.setTopicSysFlag(topicSysFlag); + topicConfig.setTopicFilterType(defaultTopicConfig.getTopicFilterType()); + } else { + LOG.warn("create new topic failed, because the default topic[" + defaultTopic + + "] no perm, " + defaultTopicConfig.getPerm() + " producer: " + + remoteAddress); + } + } else { + LOG.warn("create new topic failed, because the default topic[" + defaultTopic + + "] not exist." + " producer: " + remoteAddress); + } + + if (topicConfig != null) { + LOG.info("create new topic by default topic[" + defaultTopic + "], " + topicConfig + + " producer: " + remoteAddress); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + createNew = true; + + this.persist(); + } + } finally { + this.lockTopicConfigTable.unlock(); + } + } + } catch (InterruptedException e) { + LOG.error("createTopicInSendMessageMethod exception", e); + } + + if (createNew) { + this.brokerController.registerBrokerAll(false, true); + } + + return topicConfig; + } + + public TopicConfig createTopicInSendMessageBackMethod( + final String topic, + final int clientDefaultTopicQueueNums, + final int perm, + final int topicSysFlag) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + boolean createNew = false; + + try { + if (this.lockTopicConfigTable.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(clientDefaultTopicQueueNums); + topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums); + topicConfig.setPerm(perm); + topicConfig.setTopicSysFlag(topicSysFlag); + + LOG.info("create new topic {}", topicConfig); + this.topicConfigTable.put(topic, topicConfig); + createNew = true; + this.dataVersion.nextVersion(); + this.persist(); + } finally { + this.lockTopicConfigTable.unlock(); + } + } + } catch (InterruptedException e) { + LOG.error("createTopicInSendMessageBackMethod exception", e); + } + + if (createNew) { + this.brokerController.registerBrokerAll(false, true); + } + + return topicConfig; + } + + public void updateTopicUnitFlag(final String topic, final boolean unit) { + + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) { + int oldTopicSysFlag = topicConfig.getTopicSysFlag(); + if (unit) { + topicConfig.setTopicSysFlag(TopicSysFlag.setUnitFlag(oldTopicSysFlag)); + } else { + topicConfig.setTopicSysFlag(TopicSysFlag.clearUnitFlag(oldTopicSysFlag)); + } + + LOG.info("update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag", oldTopicSysFlag, + topicConfig.getTopicSysFlag()); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + this.persist(); + this.brokerController.registerBrokerAll(false, true); + } + } + + public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) { + int oldTopicSysFlag = topicConfig.getTopicSysFlag(); + if (hasUnitSub) { + topicConfig.setTopicSysFlag(TopicSysFlag.setUnitSubFlag(oldTopicSysFlag)); + } + + LOG.info("update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag", oldTopicSysFlag, + topicConfig.getTopicSysFlag()); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + this.persist(); + this.brokerController.registerBrokerAll(false, true); + } + } + + public void updateTopicConfig(final TopicConfig topicConfig) { + TopicConfig old = this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + if (old != null) { + LOG.info("update topic config, old: " + old + " new: " + topicConfig); + } else { + LOG.info("create new topic, " + topicConfig); + } + + this.dataVersion.nextVersion(); + + this.persist(); + } + + + public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) { + + if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) { + boolean isChange = false; + Set orderTopics = orderKVTableFromNs.getTable().keySet(); + for (String topic : orderTopics) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null && !topicConfig.isOrder()) { + topicConfig.setOrder(true); + isChange = true; + LOG.info("update order topic config, topic={}, order={}", topic, true); + } + } + + for (Map.Entry entry : this.topicConfigTable.entrySet()) { + String topic = entry.getKey(); + if (!orderTopics.contains(topic)) { + TopicConfig topicConfig = entry.getValue(); + if (topicConfig.isOrder()) { + topicConfig.setOrder(false); + isChange = true; + LOG.info("update order topic config, topic={}, order={}", topic, false); + } + } + } + + if (isChange) { + this.dataVersion.nextVersion(); + this.persist(); + } + } + } + + public boolean isOrderTopic(final String topic) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig == null) { + return false; + } else { + return topicConfig.isOrder(); + } + } + + public void deleteTopicConfig(final String topic) { + TopicConfig old = this.topicConfigTable.remove(topic); + if (old != null) { + LOG.info("delete topic config OK, topic: " + old); + this.dataVersion.nextVersion(); + this.persist(); + } else { + LOG.warn("delete topic config failed, topic: " + topic + " not exist"); + } + } + + public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + topicConfigSerializeWrapper.setDataVersion(this.dataVersion); + return topicConfigSerializeWrapper; + } + + @Override + public String encode() { + return encode(false); + } + + @Override + public String configFilePath() { +// return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig() +// .getStorePathRootDir()); + return BrokerPathConfigHelper.getTopicConfigPath(System.getProperty("user.home") + File.separator + "store"); + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = + TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class); + if (topicConfigSerializeWrapper != null) { + this.topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable()); + this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion()); + this.printLoadDataWhenFirstBoot(topicConfigSerializeWrapper); + } + } + } + + public String encode(final boolean prettyFormat) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + topicConfigSerializeWrapper.setDataVersion(this.dataVersion); + return topicConfigSerializeWrapper.toJson(prettyFormat); + } + + private void printLoadDataWhenFirstBoot(final TopicConfigSerializeWrapper tcs) { + Iterator> it = tcs.getTopicConfigTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + LOG.info("load exist local topic, {}", next.getValue().toString()); + } + } + + public DataVersion getDataVersion() { + return dataVersion; + } + + public ConcurrentHashMap getTopicConfigTable() { + return topicConfigTable; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java new file mode 100644 index 0000000000..4328cf8f1c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.transaction; + +public class TransactionRecord { + // Commit Log Offset + private long offset; + private String producerGroup; + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java new file mode 100644 index 0000000000..9d977abd7c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.transaction; + +import java.util.List; + + +public interface TransactionStore { + public boolean open(); + + + public void close(); + + + public boolean put(final List trs); + + + public void remove(final List pks); + + + public List traverse(final long pk, final int nums); + + + public long totalRecords(); + + + public long minPK(); + + + public long maxPK(); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java new file mode 100644 index 0000000000..47de33ba0c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.transaction.jdbc; + +import com.alibaba.rocketmq.broker.transaction.TransactionRecord; +import com.alibaba.rocketmq.broker.transaction.TransactionStore; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.sql.*; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicLong; + + +public class JDBCTransactionStore implements TransactionStore { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); + private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig; + private Connection connection; + private AtomicLong totalRecordsValue = new AtomicLong(0); + + public JDBCTransactionStore(JDBCTransactionStoreConfig jdbcTransactionStoreConfig) { + this.jdbcTransactionStoreConfig = jdbcTransactionStoreConfig; + } + + @Override + public boolean open() { + if (this.loadDriver()) { + Properties props = new Properties(); + props.put("user", jdbcTransactionStoreConfig.getJdbcUser()); + props.put("password", jdbcTransactionStoreConfig.getJdbcPassword()); + + try { + this.connection = + DriverManager.getConnection(this.jdbcTransactionStoreConfig.getJdbcURL(), props); + + this.connection.setAutoCommit(false); + + + if (!this.computeTotalRecords()) { + return this.createDB(); + } + + return true; + } catch (SQLException e) { + log.info("Create JDBC Connection Exeption", e); + } + } + + return false; + } + + private boolean loadDriver() { + try { + Class.forName(this.jdbcTransactionStoreConfig.getJdbcDriverClass()).newInstance(); + log.info("Loaded the appropriate driver, {}", + this.jdbcTransactionStoreConfig.getJdbcDriverClass()); + return true; + } catch (Exception e) { + log.info("Loaded the appropriate driver Exception", e); + } + + return false; + } + + private boolean computeTotalRecords() { + Statement statement = null; + ResultSet resultSet = null; + try { + statement = this.connection.createStatement(); + + resultSet = statement.executeQuery("select count(offset) as total from t_transaction"); + if (!resultSet.next()) { + log.warn("computeTotalRecords ResultSet is empty"); + return false; + } + + this.totalRecordsValue.set(resultSet.getLong(1)); + } catch (Exception e) { + log.warn("computeTotalRecords Exception", e); + return false; + } finally { + if (null != statement) { + try { + statement.close(); + } catch (SQLException e) { + } + } + + if (null != resultSet) { + try { + resultSet.close(); + } catch (SQLException e) { + } + } + } + + return true; + } + + private boolean createDB() { + Statement statement = null; + try { + statement = this.connection.createStatement(); + + String sql = this.createTableSql(); + log.info("createDB SQL:\n {}", sql); + statement.execute(sql); + this.connection.commit(); + return true; + } catch (Exception e) { + log.warn("createDB Exception", e); + return false; + } finally { + if (null != statement) { + try { + statement.close(); + } catch (SQLException e) { + log.warn("Close statement exception", e); + } + } + } + } + + private String createTableSql() { + URL resource = JDBCTransactionStore.class.getClassLoader().getResource("transaction.sql"); + String fileContent = MixAll.file2String(resource); + return fileContent; + } + + @Override + public void close() { + try { + if (this.connection != null) { + this.connection.close(); + } + } catch (SQLException e) { + } + } + + @Override + public boolean put(List trs) { + PreparedStatement statement = null; + try { + this.connection.setAutoCommit(false); + statement = this.connection.prepareStatement("insert into t_transaction values (?, ?)"); + for (TransactionRecord tr : trs) { + statement.setLong(1, tr.getOffset()); + statement.setString(2, tr.getProducerGroup()); + statement.addBatch(); + } + int[] executeBatch = statement.executeBatch(); + this.connection.commit(); + this.totalRecordsValue.addAndGet(updatedRows(executeBatch)); + return true; + } catch (Exception e) { + log.warn("createDB Exception", e); + return false; + } finally { + if (null != statement) { + try { + statement.close(); + } catch (SQLException e) { + log.warn("Close statement exception", e); + } + } + } + } + + private long updatedRows(int[] rows) { + long res = 0; + for (int i : rows) { + res += i; + } + + return res; + } + + @Override + public void remove(List pks) { + PreparedStatement statement = null; + try { + this.connection.setAutoCommit(false); + statement = this.connection.prepareStatement("DELETE FROM t_transaction WHERE offset = ?"); + for (long pk : pks) { + statement.setLong(1, pk); + statement.addBatch(); + } + int[] executeBatch = statement.executeBatch(); + this.connection.commit(); + } catch (Exception e) { + log.warn("createDB Exception", e); + } finally { + if (null != statement) { + try { + statement.close(); + } catch (SQLException e) { + } + } + } + } + + @Override + public List traverse(long pk, int nums) { + return null; + } + + @Override + public long totalRecords() { + return this.totalRecordsValue.get(); + } + + @Override + public long minPK() { + return 0; + } + + @Override + public long maxPK() { + return 0; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java new file mode 100644 index 0000000000..1244cfc524 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.broker.transaction.jdbc; + +public class JDBCTransactionStoreConfig { + private String jdbcDriverClass = "com.mysql.jdbc.Driver"; + private String jdbcURL = "jdbc:mysql://xxx.xxx.xxx.xxx:1000/xxx?useUnicode=true&characterEncoding=UTF-8"; + private String jdbcUser = "xxx"; + private String jdbcPassword = "xxx"; + + + public String getJdbcDriverClass() { + return jdbcDriverClass; + } + + + public void setJdbcDriverClass(String jdbcDriverClass) { + this.jdbcDriverClass = jdbcDriverClass; + } + + + public String getJdbcURL() { + return jdbcURL; + } + + + public void setJdbcURL(String jdbcURL) { + this.jdbcURL = jdbcURL; + } + + + public String getJdbcUser() { + return jdbcUser; + } + + + public void setJdbcUser(String jdbcUser) { + this.jdbcUser = jdbcUser; + } + + + public String getJdbcPassword() { + return jdbcPassword; + } + + + public void setJdbcPassword(String jdbcPassword) { + this.jdbcPassword = jdbcPassword; + } +} diff --git a/rocketmq-broker/src/main/resources/transaction.sql b/rocketmq-broker/src/main/resources/transaction.sql new file mode 100644 index 0000000000..aaefe4335f --- /dev/null +++ b/rocketmq-broker/src/main/resources/transaction.sql @@ -0,0 +1,4 @@ +CREATE TABLE t_transaction( + offset NUMERIC(20) PRIMARY KEY, + producerGroup VARCHAR(64) +) diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java new file mode 100644 index 0000000000..34ebfa5c5d --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SendMessageTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.api; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.client.hook.SendMessageContext; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientAPIImpl; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.Test; + + +/** + * @author shijia.wxr + */ +public class SendMessageTest { + @Test + public void test_sendMessage() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + + brokerController.start(); + + MQClientAPIImpl client = new MQClientAPIImpl(new NettyClientConfig(), null, null, null); + client.start(); + + for (int i = 0; i < 100; i++) { + String topic = "UnitTestTopic_" + i % 3; + Message msg = new Message(topic, "TAG1 TAG2", "100200300", ("Hello, Nice world\t" + i).getBytes()); + msg.setDelayTimeLevel(i % 3 + 1); + + try { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup("abc"); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(MixAll.DEFAULT_TOPIC); + requestHeader.setDefaultTopicQueueNums(4); + requestHeader.setQueueId(i % 4); + requestHeader.setSysFlag(0); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + + SendResult result = client.sendMessage("127.0.0.1:10911", "brokerName", msg, requestHeader, 1000 * 5, + CommunicationMode.SYNC, new SendMessageContext(), null); + System.out.println(i + "\t" + result); + } catch (Exception e) { + e.printStackTrace(); + } + } + + client.shutdown(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java new file mode 100644 index 0000000000..55844eb8e5 --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ConsumerOffsetManagerTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.offset; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.Test; + +import java.util.Random; + + +/** + * @author shijia.wxr + */ +public class ConsumerOffsetManagerTest { + @Test + public void test_flushConsumerOffset() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + brokerController.start(); + + ConsumerOffsetManager consumerOffsetManager = new ConsumerOffsetManager(brokerController); + + Random random = new Random(); + + for (int i = 0; i < 100; i++) { + String group = "DIANPU_GROUP_" + i; + for (int id = 0; id < 16; id++) { + consumerOffsetManager.commitOffset(null, group, "TOPIC_A", id, + random.nextLong() % 1024 * 1024 * 1024); + consumerOffsetManager.commitOffset(null, group, "TOPIC_B", id, + random.nextLong() % 1024 * 1024 * 1024); + consumerOffsetManager.commitOffset(null, group, "TOPIC_C", id, + random.nextLong() % 1024 * 1024 * 1024); + } + } + + consumerOffsetManager.persist(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java new file mode 100644 index 0000000000..9edd02e008 --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: TopicConfigManagerTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.topic; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class TopicConfigManagerTest { + @Test + public void test_flushTopicConfig() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + brokerController.start(); + + TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController); + + TopicConfig topicConfig = + topicConfigManager.createTopicInSendMessageMethod("TestTopic_SEND", MixAll.DEFAULT_TOPIC, + null, 4, 0); + assertTrue(topicConfig != null); + + System.out.println(topicConfig); + + for (int i = 0; i < 10; i++) { + String topic = "UNITTEST-" + i; + topicConfig = + topicConfigManager + .createTopicInSendMessageMethod(topic, MixAll.DEFAULT_TOPIC, null, 4, 0); + assertTrue(topicConfig != null); + } + + topicConfigManager.persist(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-client/deploy.bat b/rocketmq-client/deploy.bat new file mode 100644 index 0000000000..f7780708eb --- /dev/null +++ b/rocketmq-client/deploy.bat @@ -0,0 +1 @@ +mvn -Dmaven.test.skip=true deploy -Pclient-shade \ No newline at end of file diff --git a/rocketmq-client/install.bat b/rocketmq-client/install.bat new file mode 100644 index 0000000000..87bf456d41 --- /dev/null +++ b/rocketmq-client/install.bat @@ -0,0 +1,2 @@ +mvn -Dmaven.test.skip=true clean package install -Pclient-shade -U + diff --git a/rocketmq-client/pom.xml b/rocketmq-client/pom.xml new file mode 100644 index 0000000000..63a611423b --- /dev/null +++ b/rocketmq-client/pom.xml @@ -0,0 +1,97 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + + 4.0.0 + jar + rocketmq-client + rocketmq-client ${project.version} + + + + client-shade + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + + package + + shade + + + + + false + false + true + false + true + + + com.alibaba:fastjson + io.netty:netty-all + com.alibaba.rocketmq:rocketmq-client + com.alibaba.rocketmq:rocketmq-common + com.alibaba.rocketmq:rocketmq-remoting + + + + + io.netty + com.alibaba.rocketmq.shade.io.netty + + + com.alibaba.fastjson + com.alibaba.rocketmq.shade.com.alibaba.fastjson + + + + + + + + + + + + + junit + junit + test + + + ${project.groupId} + rocketmq-common + + + org.slf4j + slf4j-api + + + diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java new file mode 100644 index 0000000000..4d80564b0e --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java @@ -0,0 +1,202 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +/** + * Client Common configuration + * + * @author shijia.wxr + * @author vongosling + */ +public class ClientConfig { + public static final String SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY = "com.rocketmq.sendMessageWithVIPChannel"; + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV)); + private String clientIP = RemotingUtil.getLocalAddress(); + private String instanceName = System.getProperty("rocketmq.client.name", "DEFAULT"); + private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors(); + /** + * Pulling topic information interval from the named server + */ + private int pollNameServerInteval = 1000 * 30; + /** + * Heartbeat interval in microseconds with message broker + */ + private int heartbeatBrokerInterval = 1000 * 30; + /** + * Offset persistent interval for consumer + */ + private int persistConsumerOffsetInterval = 1000 * 5; + private boolean unitMode = false; + private String unitName; + private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "true")); + + public String buildMQClientId() { + StringBuilder sb = new StringBuilder(); + sb.append(this.getClientIP()); + + sb.append("@"); + sb.append(this.getInstanceName()); + if (!UtilAll.isBlank(this.unitName)) { + sb.append("@"); + sb.append(this.unitName); + } + + return sb.toString(); + } + + public String getClientIP() { + return clientIP; + } + + public void setClientIP(String clientIP) { + this.clientIP = clientIP; + } + + public String getInstanceName() { + return instanceName; + } + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } + + public void changeInstanceNameToPID() { + if (this.instanceName.equals("DEFAULT")) { + this.instanceName = String.valueOf(UtilAll.getPid()); + } + } + + public void resetClientConfig(final ClientConfig cc) { + this.namesrvAddr = cc.namesrvAddr; + this.clientIP = cc.clientIP; + this.instanceName = cc.instanceName; + this.clientCallbackExecutorThreads = cc.clientCallbackExecutorThreads; + this.pollNameServerInteval = cc.pollNameServerInteval; + this.heartbeatBrokerInterval = cc.heartbeatBrokerInterval; + this.persistConsumerOffsetInterval = cc.persistConsumerOffsetInterval; + this.unitMode = cc.unitMode; + this.unitName = cc.unitName; + this.vipChannelEnabled = cc.vipChannelEnabled; + } + + public ClientConfig cloneClientConfig() { + ClientConfig cc = new ClientConfig(); + cc.namesrvAddr = namesrvAddr; + cc.clientIP = clientIP; + cc.instanceName = instanceName; + cc.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + cc.pollNameServerInteval = pollNameServerInteval; + cc.heartbeatBrokerInterval = heartbeatBrokerInterval; + cc.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + cc.unitMode = unitMode; + cc.unitName = unitName; + cc.vipChannelEnabled = vipChannelEnabled; + return cc; + } + + public String getNamesrvAddr() { + return namesrvAddr; + } + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + public int getClientCallbackExecutorThreads() { + return clientCallbackExecutorThreads; + } + + + public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) { + this.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + } + + + public int getPollNameServerInteval() { + return pollNameServerInteval; + } + + + public void setPollNameServerInteval(int pollNameServerInteval) { + this.pollNameServerInteval = pollNameServerInteval; + } + + + public int getHeartbeatBrokerInterval() { + return heartbeatBrokerInterval; + } + + + public void setHeartbeatBrokerInterval(int heartbeatBrokerInterval) { + this.heartbeatBrokerInterval = heartbeatBrokerInterval; + } + + + public int getPersistConsumerOffsetInterval() { + return persistConsumerOffsetInterval; + } + + + public void setPersistConsumerOffsetInterval(int persistConsumerOffsetInterval) { + this.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + } + + + public String getUnitName() { + return unitName; + } + + + public void setUnitName(String unitName) { + this.unitName = unitName; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + + public boolean isVipChannelEnabled() { + return vipChannelEnabled; + } + + + public void setVipChannelEnabled(final boolean vipChannelEnabled) { + this.vipChannelEnabled = vipChannelEnabled; + } + + + @Override + public String toString() { + return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName + + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInteval=" + pollNameServerInteval + + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + + persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + + vipChannelEnabled + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java new file mode 100644 index 0000000000..4e202e978b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * Base interface for MQ management + * + * @author shijia.wxr + */ +public interface MQAdmin { + /** + * Creates an topic + * + * @param key + * accesskey + * @param newTopic + * topic name + * @param queueNum + * topic's queue number + * + * @throws MQClientException + */ + void createTopic(final String key, final String newTopic, final int queueNum) + throws MQClientException; + + + /** + * Creates an topic + * + * @param key + * accesskey + * @param newTopic + * topic name + * @param queueNum + * topic's queue number + * @param topicSysFlag + * topic system flag + * + * @throws MQClientException + */ + void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException; + + + /** + * Gets the message queue offset according to some time in milliseconds
+ * be cautious to call because of more IO overhead + * + * @param mq + * Instance of MessageQueue + * @param timestamp + * from when in milliseconds. + * + * @return offset + * + * @throws MQClientException + */ + long searchOffset(final MessageQueue mq, final long timestamp) throws MQClientException; + + + /** + * Gets the max offset + * + * @param mq + * Instance of MessageQueue + * + * @return the max offset + * + * @throws MQClientException + */ + long maxOffset(final MessageQueue mq) throws MQClientException; + + + /** + * Gets the minimum offset + * + * @param mq + * Instance of MessageQueue + * + * @return the minimum offset + * + * @throws MQClientException + */ + long minOffset(final MessageQueue mq) throws MQClientException; + + + /** + * Gets the earliest stored message time + * + * @param mq + * Instance of MessageQueue + * + * @return the time in microseconds + * + * @throws MQClientException + */ + long earliestMsgStoreTime(final MessageQueue mq) throws MQClientException; + + + /** + * Query message according tto message id + * + * @param offsetMsgId + * message id + * + * @return message + * + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + MessageExt viewMessage(final String offsetMsgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * Query messages + * + * @param topic + * message topic + * @param key + * message key index word + * @param maxNum + * max message number + * @param begin + * from when + * @param end + * to when + * + * @return Instance of QueryResult + * + * @throws MQClientException + * @throws InterruptedException + */ + QueryResult queryMessage(final String topic, final String key, final int maxNum, final long begin, + final long end) throws MQClientException, InterruptedException; + + /** + + * @param topic + * @param msgId + * @return The {@code MessageExt} of given msgId + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + MessageExt viewMessage(String topic, String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + +} \ No newline at end of file diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java new file mode 100644 index 0000000000..5934b496a8 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import org.slf4j.Logger; + +import java.util.Set; +import java.util.TreeSet; + + +/** + * @author shijia.wxr + */ +public class MQHelper { + public static void resetOffsetByTimestamp( + final MessageModel messageModel, + final String consumerGroup, + final String topic, + final long timestamp) throws Exception { + resetOffsetByTimestamp(messageModel, "DEFAULT", consumerGroup, topic, timestamp); + } + + /** + * Reset consumer topic offset according to time + * + * @param messageModel + * which model + * @param instanceName + * which instance + * @param consumerGroup + * consumer group + * @param topic + * topic + * @param timestamp + * time + * + * @throws Exception + */ + public static void resetOffsetByTimestamp( + final MessageModel messageModel, + final String instanceName, + final String consumerGroup, + final String topic, + final long timestamp) throws Exception { + final Logger log = ClientLogger.getLog(); + + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup); + consumer.setInstanceName(instanceName); + consumer.setMessageModel(messageModel); + consumer.start(); + + Set mqs = null; + try { + mqs = consumer.fetchSubscribeMessageQueues(topic); + if (mqs != null && !mqs.isEmpty()) { + TreeSet mqsNew = new TreeSet(mqs); + for (MessageQueue mq : mqsNew) { + long offset = consumer.searchOffset(mq, timestamp); + if (offset >= 0) { + consumer.updateConsumeOffset(mq, offset); + log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}", + consumerGroup, offset, mq); + } + } + } + } catch (Exception e) { + log.warn("resetOffsetByTimestamp Exception", e); + throw e; + } finally { + if (mqs != null) { + consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs); + } + consumer.shutdown(); + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java new file mode 100644 index 0000000000..43c81063fe --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class QueryResult { + private final long indexLastUpdateTimestamp; + private final List messageList; + + + public QueryResult(long indexLastUpdateTimestamp, List messageList) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + this.messageList = messageList; + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public List getMessageList() { + return messageList; + } + + + @Override + public String toString() { + return "QueryResult [indexLastUpdateTimestamp=" + indexLastUpdateTimestamp + ", messageList=" + + messageList + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java new file mode 100644 index 0000000000..203aae08bc --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.protocol.ResponseCode; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Common Validator + * + * @author manhong.yqd + */ +public class Validators { + public static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; + public static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); + public static final int CHARACTER_MAX_LENGTH = 255; + + /** + * @param origin + * @param patternStr + * + * @return The resulting {@code String} + */ + public static String getGroupWithRegularExpression(String origin, String patternStr) { + Pattern pattern = Pattern.compile(patternStr); + Matcher matcher = pattern.matcher(origin); + while (matcher.find()) { + return matcher.group(0); + } + return null; + } + + /** + * Validate group + * + * @param group + * + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkGroup(String group) throws MQClientException { + if (UtilAll.isBlank(group)) { + throw new MQClientException("the specified group is blank", null); + } + if (!regularExpressionMatcher(group, PATTERN)) { + throw new MQClientException(String.format( + "the specified group[%s] contains illegal characters, allowing only %s", group, + VALID_PATTERN_STR), null); + } + if (group.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified group is longer than group max length 255.", null); + } + } + + /** + * @param origin + * @param pattern + * + * @return true if, and only if, the entire origin sequence + * matches this matcher's pattern + */ + public static boolean regularExpressionMatcher(String origin, Pattern pattern) { + if (pattern == null) { + return true; + } + Matcher matcher = pattern.matcher(origin); + return matcher.matches(); + } + + /** + * Validate message + * + * @param msg + * @param defaultMQProducer + * + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer) + throws MQClientException { + if (null == msg) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message is null"); + } + // topic + Validators.checkTopic(msg.getTopic()); + // body + if (null == msg.getBody()) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body is null"); + } + + if (0 == msg.getBody().length) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body length is zero"); + } + + if (msg.getBody().length > defaultMQProducer.getMaxMessageSize()) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, + "the message body size over max value, MAX: " + defaultMQProducer.getMaxMessageSize()); + } + } + + /** + * Validate topic + * + * @param topic + * + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkTopic(String topic) throws MQClientException { + if (UtilAll.isBlank(topic)) { + throw new MQClientException("the specified topic is blank", null); + } + + if (!regularExpressionMatcher(topic, PATTERN)) { + throw new MQClientException(String.format( + "the specified topic[%s] contains illegal characters, allowing only %s", topic, + VALID_PATTERN_STR), null); + } + + if (topic.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified topic is longer than topic max length 255.", null); + } + + //whether the same with system reserved keyword + if (topic.equals(MixAll.DEFAULT_TOPIC)) { + throw new MQClientException( + String.format("the topic[%s] is conflict with default topic.", topic), null); + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java new file mode 100644 index 0000000000..071a872a66 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.admin; + +/** + * @author shijia.wxr + */ +public interface MQAdminExtInner { + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ClientErrorCode.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ClientErrorCode.java new file mode 100644 index 0000000000..88d0eeaca4 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ClientErrorCode.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.common; + +public class ClientErrorCode { + public static final int CONNECT_BROKER_EXCEPTION = 10001; + public static final int ACCESS_BROKER_TIMEOUT = 10002; + public static final int BROKER_NOT_EXIST_EXCEPTION = 10003; + public static final int NO_NAME_SERVER_EXCEPTION = 10004; + public static final int NOT_FOUND_TOPIC_EXCEPTION = 10005; +} \ No newline at end of file diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ThreadLocalIndex.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ThreadLocalIndex.java new file mode 100644 index 0000000000..63fda5d41b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/common/ThreadLocalIndex.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.common; + +import java.util.Random; + +public class ThreadLocalIndex { + private final ThreadLocal threadLocalIndex = new ThreadLocal(); + private final Random random = new Random(); + public ThreadLocalIndex(int value) { + + } + + public int getAndIncrement() { + Integer index = this.threadLocalIndex.get(); + if (null == index) { + index = Math.abs(random.nextInt()); + if (index < 0) index = 0; + this.threadLocalIndex.set(index); + } + + index = Math.abs(index + 1); + if (index < 0) + index = 0; + + this.threadLocalIndex.set(index); + return index; + } + + @Override + public String toString() { + return "ThreadLocalIndex{" + + "threadLocalIndex=" + threadLocalIndex.get() + + '}'; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java new file mode 100644 index 0000000000..4d70167c8c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; + + +/** + * Strategy Algorithm for message allocating between consumers + * + * @author shijia.wxr + * @author vongosling + */ +public interface AllocateMessageQueueStrategy { + + /** + * Allocating by consumer id + * + * @param consumerGroup + * current consumer group + * @param currentCID + * current consumer id + * @param mqAll + * message queue set in current topic + * @param cidAll + * consumer set in current consumer group + * + * @return The allocate result of given strategy + */ + List allocate( + final String consumerGroup, + final String currentCID, + final List mqAll, + final List cidAll + ); + + + /** + * Algorithm name + * + * @return The strategy name + */ + String getName(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java new file mode 100644 index 0000000000..96040aed0a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -0,0 +1,381 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.HashSet; +import java.util.Set; + + +/** + * Default pulling consumer + * + * @author shijia.wxr + */ +public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer { + protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; + + /** + * Do the same thing for the same Group, the application must be set,and + * guarantee Globally unique + */ + private String consumerGroup; + /** + * Long polling mode, the Consumer connection max suspend time, it is not + * recommended to modify + */ + private long brokerSuspendMaxTimeMillis = 1000 * 20; + /** + * Long polling mode, the Consumer connection timeout(must greater than + * brokerSuspendMaxTimeMillis), it is not recommended to modify + */ + private long consumerTimeoutMillisWhenSuspend = 1000 * 30; + /** + * The socket timeout in milliseconds + */ + private long consumerPullTimeoutMillis = 1000 * 10; + /** + * Consumption pattern,default is clustering + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * Message queue listener + */ + private MessageQueueListener messageQueueListener; + /** + * Offset Storage + */ + private OffsetStore offsetStore; + /** + * Topic set you want to register + */ + private Set registerTopics = new HashSet(); + /** + * Queue allocation algorithm + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); + /** + * Whether the unit of subscription group + */ + private boolean unitMode = false; + + private int maxReconsumeTimes = 16; + + + public DefaultMQPullConsumer() { + this(MixAll.DEFAULT_CONSUMER_GROUP, null); + } + + + public DefaultMQPullConsumer(final String consumerGroup, RPCHook rpcHook) { + this.consumerGroup = consumerGroup; + defaultMQPullConsumerImpl = new DefaultMQPullConsumerImpl(this, rpcHook); + } + + + public DefaultMQPullConsumer(final String consumerGroup) { + this(consumerGroup, null); + } + + + public DefaultMQPullConsumer(RPCHook rpcHook) { + this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.defaultMQPullConsumerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQPullConsumerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String offsetMsgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.defaultMQPullConsumerImpl.viewMessage(offsetMsgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQPullConsumerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public long getBrokerSuspendMaxTimeMillis() { + return brokerSuspendMaxTimeMillis; + } + + + public void setBrokerSuspendMaxTimeMillis(long brokerSuspendMaxTimeMillis) { + this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public long getConsumerPullTimeoutMillis() { + return consumerPullTimeoutMillis; + } + + + public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) { + this.consumerPullTimeoutMillis = consumerPullTimeoutMillis; + } + + + public long getConsumerTimeoutMillisWhenSuspend() { + return consumerTimeoutMillisWhenSuspend; + } + + + public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { + this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public MessageQueueListener getMessageQueueListener() { + return messageQueueListener; + } + + + public void setMessageQueueListener(MessageQueueListener messageQueueListener) { + this.messageQueueListener = messageQueueListener; + } + + + public Set getRegisterTopics() { + return registerTopics; + } + + + public void setRegisterTopics(Set registerTopics) { + this.registerTopics = registerTopics; + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, null); + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, brokerName); + } + + @Override + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchSubscribeMessageQueues(topic); + } + + @Override + public void start() throws MQClientException { + this.defaultMQPullConsumerImpl.start(); + } + + @Override + public void shutdown() { + this.defaultMQPullConsumerImpl.shutdown(); + } + + @Override + public void registerMessageQueueListener(String topic, MessageQueueListener listener) { + synchronized (this.registerTopics) { + this.registerTopics.add(topic); + if (listener != null) { + this.messageQueueListener = listener; + } + } + } + + @Override + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums); + } + + @Override + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, timeout); + } + + @Override + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, pullCallback); + } + + @Override + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, pullCallback, timeout); + } + + @Override + public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pullBlockIfNotFound(mq, subExpression, offset, maxNums); + } + + @Override + public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pullBlockIfNotFound(mq, subExpression, offset, maxNums, pullCallback); + } + + @Override + public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException { + this.defaultMQPullConsumerImpl.updateConsumeOffset(mq, offset); + } + + @Override + public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchConsumeOffset(mq, fromStore); + } + + @Override + public Set fetchMessageQueuesInBalance(String topic) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchMessageQueuesInBalance(topic); + } + + @Override + public MessageExt viewMessage(String topic, String uniqKey) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + MessageDecoder.decodeMessageId(uniqKey); + return this.viewMessage(uniqKey); + } catch (Exception e) { + } + return this.defaultMQPullConsumerImpl.queryMessageByUniqKey(topic, uniqKey); + } + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, brokerName, consumerGroup); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + public DefaultMQPullConsumerImpl getDefaultMQPullConsumerImpl() { + return defaultMQPullConsumerImpl; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public int getMaxReconsumeTimes() { + return maxReconsumeTimes; + } + + + public void setMaxReconsumeTimes(final int maxReconsumeTimes) { + this.maxReconsumeTimes = maxReconsumeTimes; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java new file mode 100644 index 0000000000..f37e982fe0 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -0,0 +1,519 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + + +/** + * Wrapped push consumer.in fact,it works as remarkable as the pull consumer + * + * @author shijia.wxr + */ +public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer { + protected final transient DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + /** + * Do the same thing for the same Group, the application must be set,and + * guarantee Globally unique + */ + private String consumerGroup; + /** + * Consumption pattern,default is clustering + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * Consumption offset + */ + private ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + /** + * Backtracking consumption time with second precision.time format is + * 20131223171201
+ * Implying Seventeen twelve and 01 seconds on December 23, 2013 year
+ * Default backtracking consumption time Half an hour ago + */ + private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30)); + /** + * Queue allocation algorithm + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy; + + /** + * Subscription relationship + */ + private Map subscription = new HashMap(); + /** + * Message listener + */ + private MessageListener messageListener; + /** + * Offset Storage + */ + private OffsetStore offsetStore; + /** + * Minimum consumer thread number + */ + private int consumeThreadMin = 20; + /** + * Max consumer thread number + */ + private int consumeThreadMax = 64; + + /** + * Threshold for dynamic adjustment of the number of thread pool + */ + private long adjustThreadPoolNumsThreshold = 100000; + + /** + * Concurrently max span offset.it has no effect on sequential consumption + */ + private int consumeConcurrentlyMaxSpan = 2000; + /** + * Flow control threshold + */ + private int pullThresholdForQueue = 1000; + /** + * Message pull Interval + */ + private long pullInterval = 0; + /** + * Batch consumption size + */ + private int consumeMessageBatchMaxSize = 1; + /** + * Batch pull size + */ + private int pullBatchSize = 32; + + /** + * Whether update subscription relationship when every pull + */ + private boolean postSubscriptionWhenPull = false; + + /** + * Whether the unit of subscription group + */ + private boolean unitMode = false; + + private int maxReconsumeTimes = -1; + private long suspendCurrentQueueTimeMillis = 1000; + private long consumeTimeout = 15; + + + public DefaultMQPushConsumer() { + this(MixAll.DEFAULT_CONSUMER_GROUP, null, new AllocateMessageQueueAveragely()); + } + + + public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.consumerGroup = consumerGroup; + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); + } + + + public DefaultMQPushConsumer(RPCHook rpcHook) { + this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook, new AllocateMessageQueueAveragely()); + } + + + public DefaultMQPushConsumer(final String consumerGroup) { + this(consumerGroup, null, new AllocateMessageQueueAveragely()); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.defaultMQPushConsumerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQPushConsumerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.defaultMQPushConsumerImpl.viewMessage(offsetMsgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQPushConsumerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + @Override + public MessageExt viewMessage(String topic, String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + MessageDecoder.decodeMessageId(msgId); + return this.viewMessage(msgId); + } catch (Exception e) { + } + return this.defaultMQPushConsumerImpl.queryMessageByUniqKey(topic, msgId); + } + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public int getConsumeConcurrentlyMaxSpan() { + return consumeConcurrentlyMaxSpan; + } + + + public void setConsumeConcurrentlyMaxSpan(int consumeConcurrentlyMaxSpan) { + this.consumeConcurrentlyMaxSpan = consumeConcurrentlyMaxSpan; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } + + + public int getConsumeMessageBatchMaxSize() { + return consumeMessageBatchMaxSize; + } + + + public void setConsumeMessageBatchMaxSize(int consumeMessageBatchMaxSize) { + this.consumeMessageBatchMaxSize = consumeMessageBatchMaxSize; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public int getConsumeThreadMax() { + return consumeThreadMax; + } + + + public void setConsumeThreadMax(int consumeThreadMax) { + this.consumeThreadMax = consumeThreadMax; + } + + + public int getConsumeThreadMin() { + return consumeThreadMin; + } + + + public void setConsumeThreadMin(int consumeThreadMin) { + this.consumeThreadMin = consumeThreadMin; + } + + + public DefaultMQPushConsumerImpl getDefaultMQPushConsumerImpl() { + return defaultMQPushConsumerImpl; + } + + + public MessageListener getMessageListener() { + return messageListener; + } + + + public void setMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public int getPullBatchSize() { + return pullBatchSize; + } + + + public void setPullBatchSize(int pullBatchSize) { + this.pullBatchSize = pullBatchSize; + } + + + public long getPullInterval() { + return pullInterval; + } + + + public void setPullInterval(long pullInterval) { + this.pullInterval = pullInterval; + } + + + public int getPullThresholdForQueue() { + return pullThresholdForQueue; + } + + + public void setPullThresholdForQueue(int pullThresholdForQueue) { + this.pullThresholdForQueue = pullThresholdForQueue; + } + + + public Map getSubscription() { + return subscription; + } + + + public void setSubscription(Map subscription) { + this.subscription = subscription; + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, null); + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, brokerName); + } + + + @Override + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + return this.defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(topic); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQPushConsumerImpl.start(); + } + + + @Override + public void shutdown() { + this.defaultMQPushConsumerImpl.shutdown(); + } + + + @Override + @Deprecated + public void registerMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + this.defaultMQPushConsumerImpl.registerMessageListener(messageListener); + } + + + @Override + public void registerMessageListener(MessageListenerConcurrently messageListener) { + this.messageListener = messageListener; + this.defaultMQPushConsumerImpl.registerMessageListener(messageListener); + } + + + @Override + public void registerMessageListener(MessageListenerOrderly messageListener) { + this.messageListener = messageListener; + this.defaultMQPushConsumerImpl.registerMessageListener(messageListener); + } + + + @Override + public void subscribe(String topic, String subExpression) throws MQClientException { + this.defaultMQPushConsumerImpl.subscribe(topic, subExpression); + } + + + @Override + public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { + this.defaultMQPushConsumerImpl.subscribe(topic, fullClassName, filterClassSource); + } + + + @Override + public void unsubscribe(String topic) { + this.defaultMQPushConsumerImpl.unsubscribe(topic); + } + + + @Override + public void updateCorePoolSize(int corePoolSize) { + this.defaultMQPushConsumerImpl.updateCorePoolSize(corePoolSize); + } + + + @Override + public void suspend() { + this.defaultMQPushConsumerImpl.suspend(); + } + + + @Override + public void resume() { + this.defaultMQPushConsumerImpl.resume(); + } + + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + public String getConsumeTimestamp() { + return consumeTimestamp; + } + + + public void setConsumeTimestamp(String consumeTimestamp) { + this.consumeTimestamp = consumeTimestamp; + } + + + public boolean isPostSubscriptionWhenPull() { + return postSubscriptionWhenPull; + } + + + public void setPostSubscriptionWhenPull(boolean postSubscriptionWhenPull) { + this.postSubscriptionWhenPull = postSubscriptionWhenPull; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public long getAdjustThreadPoolNumsThreshold() { + return adjustThreadPoolNumsThreshold; + } + + + public void setAdjustThreadPoolNumsThreshold(long adjustThreadPoolNumsThreshold) { + this.adjustThreadPoolNumsThreshold = adjustThreadPoolNumsThreshold; + } + + + public int getMaxReconsumeTimes() { + return maxReconsumeTimes; + } + + + public void setMaxReconsumeTimes(final int maxReconsumeTimes) { + this.maxReconsumeTimes = maxReconsumeTimes; + } + + + public long getSuspendCurrentQueueTimeMillis() { + return suspendCurrentQueueTimeMillis; + } + + + public void setSuspendCurrentQueueTimeMillis(final long suspendCurrentQueueTimeMillis) { + this.suspendCurrentQueueTimeMillis = suspendCurrentQueueTimeMillis; + } + + + public long getConsumeTimeout() { + return consumeTimeout; + } + + public void setConsumeTimeout(final long consumeTimeout) { + this.consumeTimeout = consumeTimeout; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java new file mode 100644 index 0000000000..2a46b656a5 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.Set; + + +/** + * Message queue consumer interface + * + * @author shijia.wxr + */ +public interface MQConsumer extends MQAdmin { + /** + * If consuming failure,message will be send back to the brokers,and delay consuming some time + * + * @param msg + * @param delayLevel + * + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + @Deprecated + void sendMessageBack(final MessageExt msg, final int delayLevel) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + /** + * If consuming failure,message will be send back to the broker,and delay consuming some time + * + * @param msg + * @param delayLevel + * @param brokerName + * + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + void sendMessageBack(final MessageExt msg, final int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * Fetch message queues from consumer cache according to the topic + * + * @param topic + * message topic + * + * @return queue set + * + * @throws MQClientException + */ + Set fetchSubscribeMessageQueues(final String topic) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java new file mode 100644 index 0000000000..1125d09290 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.Set; + + +/** + * Pulling consumer interface + * + * @author shijia.wxr + */ +public interface MQPullConsumer extends MQConsumer { + /** + * Start the consumer + * + * @throws MQClientException + */ + void start() throws MQClientException; + + + /** + * Shutdown the consumer + */ + void shutdown(); + + + /** + * Register the message queue listener + * + * @param topic + * @param listener + */ + void registerMessageQueueListener(final String topic, final MessageQueueListener listener); + + + /** + * Pulling the messages,not blocking + * + * @param mq + * from which message queue + * @param subExpression + * subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
+ * if null or * expression,meaning subscribe all + * @param offset + * from where to pull + * @param maxNums + * max pulling numbers + * + * @return The resulting {@code PullRequest} + * + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + */ + PullResult pull(final MessageQueue mq, final String subExpression, final long offset, + final int maxNums) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + + /** + * Pulling the messages in the specified timeout + * + * @param mq + * @param subExpression + * @param offset + * @param maxNums + * @param timeout + * + * @return The resulting {@code PullRequest} + * + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + */ + PullResult pull(final MessageQueue mq, final String subExpression, final long offset, + final int maxNums, final long timeout) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException; + + + /** + * Pulling the messages in a async. way + * + * @param mq + * @param subExpression + * @param offset + * @param maxNums + * @param pullCallback + * + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums, + final PullCallback pullCallback) throws MQClientException, RemotingException, + InterruptedException; + + /** + * Pulling the messages in a async. way + * + * @param mq + * @param subExpression + * @param offset + * @param maxNums + * @param pullCallback + * @param timeout + * + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums, + final PullCallback pullCallback, long timeout) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * Pulling the messages,if no message arrival,blocking some time + * + * @param mq + * @param subExpression + * @param offset + * @param maxNums + * + * @return The resulting {@code PullRequest} + * + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + */ + PullResult pullBlockIfNotFound(final MessageQueue mq, final String subExpression, + final long offset, final int maxNums) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException; + + + /** + * Pulling the messages through callback function,if no message arrival,blocking. + * + * @param mq + * @param subExpression + * @param offset + * @param maxNums + * @param pullCallback + * + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + void pullBlockIfNotFound(final MessageQueue mq, final String subExpression, final long offset, + final int maxNums, final PullCallback pullCallback) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * Update the offset + * + * @param mq + * @param offset + * + * @throws MQClientException + */ + void updateConsumeOffset(final MessageQueue mq, final long offset) throws MQClientException; + + + /** + * Fetch the offset + * + * @param mq + * @param fromStore + * + * @return The fetched offset of given queue + * + * @throws MQClientException + */ + long fetchConsumeOffset(final MessageQueue mq, final boolean fromStore) throws MQClientException; + + + /** + * Fetch the message queues according to the topic + * + * @param topic + * message topic + * + * @return message queue set + * + * @throws MQClientException + */ + Set fetchMessageQueuesInBalance(final String topic) throws MQClientException; + + /** + * If consuming failure,message will be send back to the broker,and delay consuming in some time later.
+ * Mind! message can only be consumed in the same group. + * + * @param msg + * @param delayLevel + * @param brokerName + * @param consumerGroup + * + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java new file mode 100644 index 0000000000..d68b5593b8 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import org.slf4j.Logger; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + + +/** + * Schedule service for pull consumer + * + * @author shijia.wxr + */ +public class MQPullConsumerScheduleService { + private final Logger log = ClientLogger.getLog(); + private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl(); + private final ConcurrentHashMap taskTable = + new ConcurrentHashMap(); + private DefaultMQPullConsumer defaultMQPullConsumer; + private int pullThreadNums = 20; + private ConcurrentHashMap callbackTable = + new ConcurrentHashMap(); + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + + public MQPullConsumerScheduleService(final String consumerGroup) { + this.defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); + this.defaultMQPullConsumer.setMessageModel(MessageModel.CLUSTERING); + } + + public void putTask(String topic, Set mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + + for (MessageQueue mq : mqNewSet) { + if (!this.taskTable.containsKey(mq)) { + PullTaskImpl command = new PullTaskImpl(mq); + this.taskTable.put(mq, command); + this.scheduledThreadPoolExecutor.schedule(command, 0, TimeUnit.MILLISECONDS); + + } + } + } + + public void start() throws MQClientException { + final String group = this.defaultMQPullConsumer.getConsumerGroup(); + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.pullThreadNums, + new ThreadFactoryImpl("PullMsgThread-" + group) + ); + + this.defaultMQPullConsumer.setMessageQueueListener(this.messageQueueListener); + + this.defaultMQPullConsumer.start(); + + log.info("MQPullConsumerScheduleService start OK, {} {}", + this.defaultMQPullConsumer.getConsumerGroup(), this.callbackTable); + } + + public void registerPullTaskCallback(final String topic, final PullTaskCallback callback) { + this.callbackTable.put(topic, callback); + this.defaultMQPullConsumer.registerMessageQueueListener(topic, null); + } + + public void shutdown() { + if (this.scheduledThreadPoolExecutor != null) { + this.scheduledThreadPoolExecutor.shutdown(); + } + + if (this.defaultMQPullConsumer != null) { + this.defaultMQPullConsumer.shutdown(); + } + } + + public ConcurrentHashMap getCallbackTable() { + return callbackTable; + } + + public void setCallbackTable(ConcurrentHashMap callbackTable) { + this.callbackTable = callbackTable; + } + + public int getPullThreadNums() { + return pullThreadNums; + } + + public void setPullThreadNums(int pullThreadNums) { + this.pullThreadNums = pullThreadNums; + } + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + public void setDefaultMQPullConsumer(DefaultMQPullConsumer defaultMQPullConsumer) { + this.defaultMQPullConsumer = defaultMQPullConsumer; + } + + public MessageModel getMessageModel() { + return this.defaultMQPullConsumer.getMessageModel(); + } + + public void setMessageModel(MessageModel messageModel) { + this.defaultMQPullConsumer.setMessageModel(messageModel); + } + + class MessageQueueListenerImpl implements MessageQueueListener { + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageModel messageModel = + MQPullConsumerScheduleService.this.defaultMQPullConsumer.getMessageModel(); + switch (messageModel) { + case BROADCASTING: + MQPullConsumerScheduleService.this.putTask(topic, mqAll); + break; + case CLUSTERING: + MQPullConsumerScheduleService.this.putTask(topic, mqDivided); + break; + default: + break; + } + } + } + + class PullTaskImpl implements Runnable { + private final MessageQueue messageQueue; + private volatile boolean cancelled = false; + + + public PullTaskImpl(final MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + @Override + public void run() { + String topic = this.messageQueue.getTopic(); + if (!this.isCancelled()) { + PullTaskCallback pullTaskCallback = + MQPullConsumerScheduleService.this.callbackTable.get(topic); + if (pullTaskCallback != null) { + final PullTaskContext context = new PullTaskContext(); + context.setPullConsumer(MQPullConsumerScheduleService.this.defaultMQPullConsumer); + try { + pullTaskCallback.doPullTask(this.messageQueue, context); + } catch (Throwable e) { + context.setPullNextDelayTimeMillis(1000); + log.error("doPullTask Exception", e); + } + + if (!this.isCancelled()) { + MQPullConsumerScheduleService.this.scheduledThreadPoolExecutor.schedule(this, + context.getPullNextDelayTimeMillis(), TimeUnit.MILLISECONDS); + } else { + log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); + } + } else { + log.warn("Pull Task Callback not exist , {}", topic); + } + } else { + log.warn("The Pull Task is cancelled, {}", messageQueue); + } + } + + + public boolean isCancelled() { + return cancelled; + } + + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java new file mode 100644 index 0000000000..e47739d814 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.exception.MQClientException; + + +/** + * Push consumer + * + * @author shijia.wxr + */ +public interface MQPushConsumer extends MQConsumer { + /** + * Start the consumer + * + * @throws MQClientException + */ + void start() throws MQClientException; + + + /** + * Shutdown the consumer + */ + void shutdown(); + + + /** + * Register the message listener + * + * @param messageListener + */ + @Deprecated + void registerMessageListener(MessageListener messageListener); + + + void registerMessageListener(final MessageListenerConcurrently messageListener); + + + void registerMessageListener(final MessageListenerOrderly messageListener); + + + /** + * Subscribe some topic + * + * @param topic + * @param subExpression + * subscription expression.it only support or operation such as + * "tag1 || tag2 || tag3"
+ * if null or * expression,meaning subscribe all + * + * @throws MQClientException + */ + void subscribe(final String topic, final String subExpression) throws MQClientException; + + + /** + * Subscribe some topic + * + * @param topic + * @param fullClassName + * full class name,must extend + * com.alibaba.rocketmq.common.filter. MessageFilter + * @param filterClassSource + * class source code,used UTF-8 file encoding,must be responsible + * for your code safety + * + * @throws MQClientException + */ + void subscribe(final String topic, final String fullClassName, final String filterClassSource) throws MQClientException; + + + /** + * Unsubscribe consumption some topic + * + * @param topic + * message topic + */ + void unsubscribe(final String topic); + + + /** + * Update the consumer thread pool size Dynamically + * + * @param corePoolSize + */ + void updateCorePoolSize(int corePoolSize); + + + /** + * Suspend the consumption + */ + void suspend(); + + + /** + * Resume the consumption + */ + void resume(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java new file mode 100644 index 0000000000..bb25a3a4b2 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.Set; + + +/** + * A MessageQueueListener is implemented by the application and may be specified when a message queue changed + * + * @author shijia.wxr + * @author vongosling + */ +public interface MessageQueueListener { + /** + * @param topic + * message topic + * @param mqAll + * all queues in this message topic + * @param mqDivided + * collection of queues,assigned to the current consumer + */ + void messageQueueChanged(final String topic, final Set mqAll, + final Set mqDivided); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java new file mode 100644 index 0000000000..545cff263f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +/** + * Async message pulling interface + * + * @author shijia.wxr + */ +public interface PullCallback { + public void onSuccess(final PullResult pullResult); + + public void onException(final Throwable e); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java new file mode 100644 index 0000000000..b4852438c0 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class PullResult { + private final PullStatus pullStatus; + private final long nextBeginOffset; + private final long minOffset; + private final long maxOffset; + private List msgFoundList; + + + public PullResult(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, + List msgFoundList) { + super(); + this.pullStatus = pullStatus; + this.nextBeginOffset = nextBeginOffset; + this.minOffset = minOffset; + this.maxOffset = maxOffset; + this.msgFoundList = msgFoundList; + } + + + public PullStatus getPullStatus() { + return pullStatus; + } + + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + + public long getMinOffset() { + return minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public List getMsgFoundList() { + return msgFoundList; + } + + + public void setMsgFoundList(List msgFoundList) { + this.msgFoundList = msgFoundList; + } + + + @Override + public String toString() { + return "PullResult [pullStatus=" + pullStatus + ", nextBeginOffset=" + nextBeginOffset + + ", minOffset=" + minOffset + ", maxOffset=" + maxOffset + ", msgFoundList=" + + (msgFoundList == null ? 0 : msgFoundList.size()) + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java new file mode 100644 index 0000000000..35166f3a7d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +/** + * @author shijia.wxr + */ +public enum PullStatus { + /** + * Founded + */ + FOUND, + /** + * No new message can be pull + */ + NO_NEW_MSG, + /** + * Filtering results can not match + */ + NO_MATCHED_MSG, + /** + * Illegal offset,may be too big or too small + */ + OFFSET_ILLEGAL +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java new file mode 100644 index 0000000000..19d5bfc42c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +public interface PullTaskCallback { + public void doPullTask(final MessageQueue mq, final PullTaskContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java new file mode 100644 index 0000000000..72c57d6234 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +public class PullTaskContext { + + private int pullNextDelayTimeMillis = 200; + + private MQPullConsumer pullConsumer; + + + public int getPullNextDelayTimeMillis() { + return pullNextDelayTimeMillis; + } + + + public void setPullNextDelayTimeMillis(int pullNextDelayTimeMillis) { + this.pullNextDelayTimeMillis = pullNextDelayTimeMillis; + } + + + public MQPullConsumer getPullConsumer() { + return pullConsumer; + } + + + public void setPullConsumer(MQPullConsumer pullConsumer) { + this.pullConsumer = pullConsumer; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java new file mode 100644 index 0000000000..36fcf1967d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * Consumer concurrent consumption context + * + * @author shijia.wxr + */ +public class ConsumeConcurrentlyContext { + private final MessageQueue messageQueue; + /** + * Message consume retry strategy
+ * -1,no retry,put into DLQ directly
+ * 0,broker control retry frequency
+ * >0,client control retry frequency + */ + private int delayLevelWhenNextConsume = 0; + private int ackIndex = Integer.MAX_VALUE; + + public ConsumeConcurrentlyContext(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public int getDelayLevelWhenNextConsume() { + return delayLevelWhenNextConsume; + } + + + public void setDelayLevelWhenNextConsume(int delayLevelWhenNextConsume) { + this.delayLevelWhenNextConsume = delayLevelWhenNextConsume; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public int getAckIndex() { + return ackIndex; + } + + + public void setAckIndex(int ackIndex) { + this.ackIndex = ackIndex; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java new file mode 100644 index 0000000000..d0d3bf4a23 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * @author shijia.wxr + */ +public enum ConsumeConcurrentlyStatus { + /** + * Success consumption + */ + CONSUME_SUCCESS, + /** + * Failure consumption,later try to consume + */ + RECONSUME_LATER; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java new file mode 100644 index 0000000000..26a3892ee3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * Consumer Orderly consumption context + * + * @author shijia.wxr + */ +public class ConsumeOrderlyContext { + private final MessageQueue messageQueue; + private boolean autoCommit = true; + private long suspendCurrentQueueTimeMillis = -1; + + + public ConsumeOrderlyContext(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public boolean isAutoCommit() { + return autoCommit; + } + + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public long getSuspendCurrentQueueTimeMillis() { + return suspendCurrentQueueTimeMillis; + } + + + public void setSuspendCurrentQueueTimeMillis(long suspendCurrentQueueTimeMillis) { + this.suspendCurrentQueueTimeMillis = suspendCurrentQueueTimeMillis; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java new file mode 100644 index 0000000000..e490c5c54a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * @author shijia.wxr + */ +public enum ConsumeOrderlyStatus { + /** + * Success consumption + */ + SUCCESS, + /** + * Rollback consumption(only for binlog consumption) + */ + @Deprecated + ROLLBACK, + /** + * Commit offset(only for binlog consumption) + */ + @Deprecated + COMMIT, + /** + * Suspend current queue a moment + */ + SUSPEND_CURRENT_QUEUE_A_MOMENT; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeReturnType.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeReturnType.java new file mode 100644 index 0000000000..44f998e28c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeReturnType.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * Created by alvin on 16-11-30. + */ +public enum ConsumeReturnType { + /** + * consume return success + */ + SUCCESS, + /** + * consume timeout ,even if success + */ + TIME_OUT, + /** + * consume throw exception + */ + EXCEPTION, + /** + * consume return null + */ + RETURNNULL, + /** + * consume return failed + */ + FAILED +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java new file mode 100644 index 0000000000..f34946e114 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * A MessageListener object is used to receive asynchronously delivered messages. + * + * @author shijia.wxr + */ +public interface MessageListener { +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java new file mode 100644 index 0000000000..f0b0c61465 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +/** + * A MessageListenerConcurrently object is used to receive asynchronously delivered messages concurrently + * + * @author shijia.wxr + */ +public interface MessageListenerConcurrently extends MessageListener { + /** + * It is not recommend to throw exception,rather than returning ConsumeConcurrentlyStatus.RECONSUME_LATER if consumption failure + * + * @param msgs + * msgs.size() >= 1
+ * DefaultMQPushConsumer.consumeMessageBatchMaxSize=1,you can modify here + * @param context + * + * @return The consume status + */ + ConsumeConcurrentlyStatus consumeMessage(final List msgs, + final ConsumeConcurrentlyContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java new file mode 100644 index 0000000000..d30cdfab41 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +/** + * A MessageListenerConcurrently object is used to receive asynchronously delivered messages orderly.one queue,one thread + * + * @author shijia.wxr + */ +public interface MessageListenerOrderly extends MessageListener { + /** + * It is not recommend to throw exception,rather than returning ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT if consumption failure + * + * @param msgs + * msgs.size() >= 1
+ * DefaultMQPushConsumer.consumeMessageBatchMaxSize=1,you can modify here + * @param context + * + * @return The consume status + */ + ConsumeOrderlyStatus consumeMessage(final List msgs, + final ConsumeOrderlyContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java new file mode 100644 index 0000000000..413d64600f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; + + +/** + * Average Hashing queue algorithm + * + * @author manhong.yqd + */ +public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy { + private final Logger log = ClientLogger.getLog(); + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", + consumerGroup, + currentCID, + cidAll); + return result; + } + + int index = cidAll.indexOf(currentCID); + int mod = mqAll.size() % cidAll.size(); + int averageSize = + mqAll.size() <= cidAll.size() ? 1 : (mod > 0 && index < mod ? mqAll.size() / cidAll.size() + + 1 : mqAll.size() / cidAll.size()); + int startIndex = (mod > 0 && index < mod) ? index * averageSize : index * averageSize + mod; + int range = Math.min(averageSize, mqAll.size() - startIndex); + for (int i = 0; i < range; i++) { + result.add(mqAll.get((startIndex + i) % mqAll.size())); + } + return result; + } + + @Override + public String getName() { + return "AVG"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java new file mode 100644 index 0000000000..17f4611e52 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; + + +/** + * Cycle average Hashing queue algorithm + * + * @author manhong.yqd + */ +public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy { + private final Logger log = ClientLogger.getLog(); + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", + consumerGroup, + currentCID, + cidAll); + return result; + } + + int index = cidAll.indexOf(currentCID); + for (int i = index; i < mqAll.size(); i++) { + if (i % cidAll.size() == index) { + result.add(mqAll.get(i)); + } + } + return result; + } + + @Override + public String getName() { + return "AVG_BY_CIRCLE"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java new file mode 100644 index 0000000000..783678ca11 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class AllocateMessageQueueByConfig implements AllocateMessageQueueStrategy { + private List messageQueueList; + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + return this.messageQueueList; + } + + @Override + public String getName() { + return "CONFIG"; + } + + public List getMessageQueueList() { + return messageQueueList; + } + + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java new file mode 100644 index 0000000000..5464fe3fee --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + + +/** + * Computer room Hashing queue algorithm, such as Alipay logic room + */ +public class AllocateMessageQueueByMachineRoom implements AllocateMessageQueueStrategy { + private Set consumeridcs; + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + List result = new ArrayList(); + int currentIndex = cidAll.indexOf(currentCID); + if (currentIndex < 0) { + return result; + } + List premqAll = new ArrayList(); + for (MessageQueue mq : mqAll) { + String[] temp = mq.getBrokerName().split("@"); + if (temp.length == 2 && consumeridcs.contains(temp[0])) { + premqAll.add(mq); + } + } + // Todo cid + int mod = premqAll.size() / cidAll.size(); + int rem = premqAll.size() % cidAll.size(); + int startindex = mod * currentIndex; + int endindex = startindex + mod; + for (int i = startindex; i < endindex; i++) { + result.add(mqAll.get(i)); + } + if (rem > currentIndex) { + result.add(premqAll.get(currentIndex + mod * cidAll.size())); + } + return result; + } + + @Override + public String getName() { + return "MACHINE_ROOM"; + } + + public Set getConsumeridcs() { + return consumeridcs; + } + + + public void setConsumeridcs(Set consumeridcs) { + this.consumeridcs = consumeridcs; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java new file mode 100644 index 0000000000..39aec12d68 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Local storage implementation + * + * @author shijia.wxr + */ +public class LocalFileOffsetStore implements OffsetStore { + public final static String LOCAL_OFFSET_STORE_DIR = System.getProperty( + "rocketmq.client.localOffsetStoreDir", + System.getProperty("user.home") + File.separator + ".rocketmq_offsets"); + private final static Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private final String groupName; + private final String storePath; + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + + public LocalFileOffsetStore(MQClientInstance mQClientFactory, String groupName) { + this.mQClientFactory = mQClientFactory; + this.groupName = groupName; + this.storePath = LOCAL_OFFSET_STORE_DIR + File.separator + // + this.mQClientFactory.getClientId() + File.separator + // + this.groupName + File.separator + // + "offsets.json"; + } + + + @Override + public void load() throws MQClientException { + OffsetSerializeWrapper offsetSerializeWrapper = this.readLocalOffset(); + if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) { + offsetTable.putAll(offsetSerializeWrapper.getOffsetTable()); + + for (MessageQueue mq : offsetSerializeWrapper.getOffsetTable().keySet()) { + AtomicLong offset = offsetSerializeWrapper.getOffsetTable().get(mq); + log.info("load consumer's offset, {} {} {}", + this.groupName, + mq, + offset.get()); + } + } + } + + + @Override + public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) { + if (mq != null) { + AtomicLong offsetOld = this.offsetTable.get(mq); + if (null == offsetOld) { + offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset)); + } + + if (null != offsetOld) { + if (increaseOnly) { + MixAll.compareAndIncreaseOnly(offsetOld, offset); + } else { + offsetOld.set(offset); + } + } + } + } + + + @Override + public long readOffset(final MessageQueue mq, final ReadOffsetType type) { + if (mq != null) { + switch (type) { + case MEMORY_FIRST_THEN_STORE: + case READ_FROM_MEMORY: { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + return offset.get(); + } else if (ReadOffsetType.READ_FROM_MEMORY == type) { + return -1; + } + } + case READ_FROM_STORE: { + OffsetSerializeWrapper offsetSerializeWrapper; + try { + offsetSerializeWrapper = this.readLocalOffset(); + } catch (MQClientException e) { + return -1; + } + if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) { + AtomicLong offset = offsetSerializeWrapper.getOffsetTable().get(mq); + if (offset != null) { + this.updateOffset(mq, offset.get(), false); + return offset.get(); + } + } + } + default: + break; + } + } + + return -1; + } + + + @Override + public void persistAll(Set mqs) { + if (null == mqs || mqs.isEmpty()) + return; + + OffsetSerializeWrapper offsetSerializeWrapper = new OffsetSerializeWrapper(); + for (Map.Entry entry : this.offsetTable.entrySet()) { + if (mqs.contains(entry.getKey())) { + AtomicLong offset = entry.getValue(); + offsetSerializeWrapper.getOffsetTable().put(entry.getKey(), offset); + } + } + + String jsonString = offsetSerializeWrapper.toJson(true); + if (jsonString != null) { + try { + MixAll.string2File(jsonString, this.storePath); + } catch (IOException e) { + log.error("persistAll consumer offset Exception, " + this.storePath, e); + } + } + } + + + @Override + public void persist(MessageQueue mq) { + } + + @Override + public void removeOffset(MessageQueue mq) { + + } + + @Override + public void updateConsumeOffsetToBroker(final MessageQueue mq, final long offset, final boolean isOneway) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + + } + + @Override + public Map cloneOffsetTable(String topic) { + Map cloneOffsetTable = new HashMap(); + for (Map.Entry entry : this.offsetTable.entrySet()) { + MessageQueue mq = entry.getKey(); + if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) { + continue; + } + cloneOffsetTable.put(mq, entry.getValue().get()); + + } + return cloneOffsetTable; + } + + private OffsetSerializeWrapper readLocalOffset() throws MQClientException { + String content = MixAll.file2String(this.storePath); + if (null == content || content.length() == 0) { + return this.readLocalOffsetBak(); + } else { + OffsetSerializeWrapper offsetSerializeWrapper = null; + try { + offsetSerializeWrapper = + OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class); + } catch (Exception e) { + log.warn("readLocalOffset Exception, and try to correct", e); + return this.readLocalOffsetBak(); + } + + return offsetSerializeWrapper; + } + } + + private OffsetSerializeWrapper readLocalOffsetBak() throws MQClientException { + String content = MixAll.file2String(this.storePath + ".bak"); + if (content != null && content.length() > 0) { + OffsetSerializeWrapper offsetSerializeWrapper = null; + try { + offsetSerializeWrapper = + OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class); + } catch (Exception e) { + log.warn("readLocalOffset Exception", e); + throw new MQClientException("readLocalOffset Exception, maybe fastjson version too low" // + + FAQUrl.suggestTodo(FAQUrl.LOAD_JSON_EXCEPTION), // + e); + } + return offsetSerializeWrapper; + } + + return null; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java new file mode 100644 index 0000000000..4434b8653b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Wrapper class for offset serialization + * + * @author shijia.wxr + */ +public class OffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + public ConcurrentHashMap getOffsetTable() { + return offsetTable; + } + + public void setOffsetTable(ConcurrentHashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java new file mode 100644 index 0000000000..346beb1712 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.Map; +import java.util.Set; + + +/** + * Offset store interface + * + * @author shijia.wxr + */ +public interface OffsetStore { + /** + * Load + * + * @throws MQClientException + */ + void load() throws MQClientException; + + + /** + * Update the offset,store it in memory + * + * @param mq + * @param offset + * @param increaseOnly + */ + void updateOffset(final MessageQueue mq, final long offset, final boolean increaseOnly); + + /** + * Get offset from local storage + * + * @param mq + * @param type + * + * @return The fetched offset + */ + long readOffset(final MessageQueue mq, final ReadOffsetType type); + + /** + * Persist all offsets,may be in local storage or remote name server + * + * @param mqs + */ + void persistAll(final Set mqs); + + /** + * Persist the offset,may be in local storage or remote name server + * + * @param mq + */ + void persist(final MessageQueue mq); + + /** + * Remove offset + * + * @param mq + */ + void removeOffset(MessageQueue mq); + + /** + * @param topic + * + * @return The cloned offset table of given topic + */ + Map cloneOffsetTable(String topic); + + /** + * + * @param mq + * @param offset + * @param isOneway + */ + void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java new file mode 100644 index 0000000000..3691a62bae --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +public enum ReadOffsetType { + /** + * From memory + */ + READ_FROM_MEMORY, + /** + * From storage + */ + READ_FROM_STORE, + /** + * From memory,then from storage + */ + MEMORY_FIRST_THEN_STORE; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java new file mode 100644 index 0000000000..9df2be3b50 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import org.slf4j.Logger; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Remote storage implementation + * + * @author shijia.wxr + */ +public class RemoteBrokerOffsetStore implements OffsetStore { + private final static Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private final String groupName; + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + + public RemoteBrokerOffsetStore(MQClientInstance mQClientFactory, String groupName) { + this.mQClientFactory = mQClientFactory; + this.groupName = groupName; + } + + + @Override + public void load() { + } + + + @Override + public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) { + if (mq != null) { + AtomicLong offsetOld = this.offsetTable.get(mq); + if (null == offsetOld) { + offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset)); + } + + if (null != offsetOld) { + if (increaseOnly) { + MixAll.compareAndIncreaseOnly(offsetOld, offset); + } else { + offsetOld.set(offset); + } + } + } + } + + + @Override + public long readOffset(final MessageQueue mq, final ReadOffsetType type) { + if (mq != null) { + switch (type) { + case MEMORY_FIRST_THEN_STORE: + case READ_FROM_MEMORY: { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + return offset.get(); + } else if (ReadOffsetType.READ_FROM_MEMORY == type) { + return -1; + } + } + case READ_FROM_STORE: { + try { + long brokerOffset = this.fetchConsumeOffsetFromBroker(mq); + AtomicLong offset = new AtomicLong(brokerOffset); + this.updateOffset(mq, offset.get(), false); + return brokerOffset; + } + // No offset in broker + catch (MQBrokerException e) { + return -1; + } + //Other exceptions + catch (Exception e) { + log.warn("fetchConsumeOffsetFromBroker exception, " + mq, e); + return -2; + } + } + default: + break; + } + } + + return -1; + } + + + @Override + public void persistAll(Set mqs) { + if (null == mqs || mqs.isEmpty()) + return; + + final HashSet unusedMQ = new HashSet(); + if (mqs != null && !mqs.isEmpty()) { + for (Map.Entry entry : this.offsetTable.entrySet()) { + MessageQueue mq = entry.getKey(); + AtomicLong offset = entry.getValue(); + if (offset != null) { + if (mqs.contains(mq)) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + log.info("[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", + this.groupName, + this.mQClientFactory.getClientId(), + mq, + offset.get()); + } catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); + } + } else { + unusedMQ.add(mq); + } + } + } + } + + if (!unusedMQ.isEmpty()) { + for (MessageQueue mq : unusedMQ) { + this.offsetTable.remove(mq); + log.info("remove unused mq, {}, {}", mq, this.groupName); + } + } + } + + + @Override + public void persist(MessageQueue mq) { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + log.info("[persist] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", + this.groupName, + this.mQClientFactory.getClientId(), + mq, + offset.get()); + } catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); + } + } + } + + public void removeOffset(MessageQueue mq) { + if (mq != null) { + this.offsetTable.remove(mq); + log.info("remove unnecessary messageQueue offset. group={}, mq={}, offsetTableSize={}", this.groupName, mq, + offsetTable.size()); + } + } + + @Override + public Map cloneOffsetTable(String topic) { + Map cloneOffsetTable = new HashMap(); + for (Map.Entry entry : this.offsetTable.entrySet()) { + MessageQueue mq = entry.getKey(); + if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) { + continue; + } + cloneOffsetTable.put(mq, entry.getValue().get()); + } + return cloneOffsetTable; + } + + /** + * Update the Consumer Offset in one way, once the Master is off, updated to Slave, + * here need to be optimized. + */ + private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + updateConsumeOffsetToBroker(mq, offset, true); + } + + /** + * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, + * here need to be optimized. + */ + @Override + public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + if (null == findBrokerResult) { + // TODO Here may be heavily overhead for Name Server,need tuning + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + } + + if (findBrokerResult != null) { + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setConsumerGroup(this.groupName); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setCommitOffset(offset); + + if (isOneway) { + this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffsetOneway( + findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5); + } else { + this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffset( + findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5); + } + } else { + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + } + + private long fetchConsumeOffsetFromBroker(MessageQueue mq) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + if (null == findBrokerResult) { + // TODO Here may be heavily overhead for Name Server,need tuning + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + } + + if (findBrokerResult != null) { + QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader(); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setConsumerGroup(this.groupName); + requestHeader.setQueueId(mq.getQueueId()); + + return this.mQClientFactory.getMQClientAPIImpl().queryConsumerOffset( + findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5); + } else { + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java new file mode 100644 index 0000000000..7fc09f8188 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.exception; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; + + +/** + * @author shijia.wxr + */ +public class MQBrokerException extends Exception { + private static final long serialVersionUID = 5975020272601250368L; + private final int responseCode; + private final String errorMessage; + + + public MQBrokerException(int responseCode, String errorMessage) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage)); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + + public int getResponseCode() { + return responseCode; + } + + + public String getErrorMessage() { + return errorMessage; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java new file mode 100644 index 0000000000..f343a67c1f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.exception; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; + + +/** + * @author shijia.wxr + */ +public class MQClientException extends Exception { + private static final long serialVersionUID = -5758410930844185841L; + private int responseCode; + private String errorMessage; + + + public MQClientException(String errorMessage, Throwable cause) { + super(FAQUrl.attachDefaultURL(errorMessage), cause); + this.responseCode = -1; + this.errorMessage = errorMessage; + } + + + public MQClientException(int responseCode, String errorMessage) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage)); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public MQClientException setResponseCode(final int responseCode) { + this.responseCode = responseCode; + return this; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(final String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java new file mode 100644 index 0000000000..62918038d5 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * @author manhong.yqd + */ +public class CheckForbiddenContext { + private String nameSrvAddr; + private String group; + private Message message; + private MessageQueue mq; + private String brokerAddr; + private CommunicationMode communicationMode; + private SendResult sendResult; + private Exception exception; + private Object arg; + private boolean unitMode = false; + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public Message getMessage() { + return message; + } + + + public void setMessage(Message message) { + this.message = message; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public CommunicationMode getCommunicationMode() { + return communicationMode; + } + + + public void setCommunicationMode(CommunicationMode communicationMode) { + this.communicationMode = communicationMode; + } + + + public SendResult getSendResult() { + return sendResult; + } + + + public void setSendResult(SendResult sendResult) { + this.sendResult = sendResult; + } + + + public Exception getException() { + return exception; + } + + + public void setException(Exception exception) { + this.exception = exception; + } + + + public Object getArg() { + return arg; + } + + + public void setArg(Object arg) { + this.arg = arg; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public String getNameSrvAddr() { + return nameSrvAddr; + } + + + public void setNameSrvAddr(String nameSrvAddr) { + this.nameSrvAddr = nameSrvAddr; + } + + + @Override + public String toString() { + return "SendMessageContext [nameSrvAddr=" + nameSrvAddr + ", group=" + group + ", message=" + message + + ", mq=" + mq + ", brokerAddr=" + brokerAddr + ", communicationMode=" + communicationMode + + ", sendResult=" + sendResult + ", exception=" + exception + ", unitMode=" + unitMode + + ", arg=" + arg + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java new file mode 100644 index 0000000000..35a2740cc1 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.client.exception.MQClientException; + + +/** + * @author manhong.yqd + */ +public interface CheckForbiddenHook { + public String hookName(); + + + public void checkForbidden(final CheckForbiddenContext context) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java new file mode 100644 index 0000000000..0c0e7cdee7 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; +import java.util.Map; + + +public class ConsumeMessageContext { + private String consumerGroup; + private List msgList; + private MessageQueue mq; + private boolean success; + private String status; + private Object mqTraceContext; + private Map props; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public List getMsgList() { + return msgList; + } + + + public void setMsgList(List msgList) { + this.msgList = msgList; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public boolean isSuccess() { + return success; + } + + + public void setSuccess(boolean success) { + this.success = success; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Map getProps() { + return props; + } + + + public void setProps(Map props) { + this.props = props; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java new file mode 100644 index 0000000000..96b0e53c19 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +public interface ConsumeMessageHook { + String hookName(); + + void consumeMessageBefore(final ConsumeMessageContext context); + + void consumeMessageAfter(final ConsumeMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java new file mode 100644 index 0000000000..c47f09eaa2 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; + + +/** + * @author manhong.yqd + */ +public class FilterMessageContext { + private String consumerGroup; + private List msgList; + private MessageQueue mq; + private Object arg; + private boolean unitMode; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public List getMsgList() { + return msgList; + } + + + public void setMsgList(List msgList) { + this.msgList = msgList; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public Object getArg() { + return arg; + } + + + public void setArg(Object arg) { + this.arg = arg; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + @Override + public String toString() { + return "ConsumeMessageContext [consumerGroup=" + consumerGroup + ", msgList=" + msgList + ", mq=" + + mq + ", arg=" + arg + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java new file mode 100644 index 0000000000..1528ef982f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +/** + * @author manhong.yqd + */ +public interface FilterMessageHook { + public String hookName(); + + + public void filterMessage(final FilterMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java new file mode 100644 index 0000000000..9552456757 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.message.MessageType; + +import java.util.Map; + + +public class SendMessageContext { + private String producerGroup; + private Message message; + private MessageQueue mq; + private String brokerAddr; + private String bornHost; + private CommunicationMode communicationMode; + private SendResult sendResult; + private Exception exception; + private Object mqTraceContext; + private Map props; + private DefaultMQProducerImpl producer; + private MessageType msgType = MessageType.Normal_Msg; + + public MessageType getMsgType() { + return msgType; + } + + public void setMsgType(final MessageType msgType) { + this.msgType = msgType; + } + + public DefaultMQProducerImpl getProducer() { + return producer; + } + + public void setProducer(final DefaultMQProducerImpl producer) { + this.producer = producer; + } + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Message getMessage() { + return message; + } + + + public void setMessage(Message message) { + this.message = message; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public CommunicationMode getCommunicationMode() { + return communicationMode; + } + + + public void setCommunicationMode(CommunicationMode communicationMode) { + this.communicationMode = communicationMode; + } + + + public SendResult getSendResult() { + return sendResult; + } + + + public void setSendResult(SendResult sendResult) { + this.sendResult = sendResult; + } + + + public Exception getException() { + return exception; + } + + + public void setException(Exception exception) { + this.exception = exception; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Map getProps() { + return props; + } + + + public void setProps(Map props) { + this.props = props; + } + + + public String getBornHost() { + return bornHost; + } + + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java new file mode 100644 index 0000000000..22e1fb3626 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +public interface SendMessageHook { + String hookName(); + + void sendMessageBefore(final SendMessageContext context); + + void sendMessageAfter(final SendMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java new file mode 100644 index 0000000000..79a539eb62 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.impl.producer.MQProducerInner; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GetConsumerStatusBody; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBody; +import com.alibaba.rocketmq.common.protocol.header.*; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + + +/** + * @author shijia.wxr + */ +public class ClientRemotingProcessor implements NettyRequestProcessor { + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mqClientFactory; + + + public ClientRemotingProcessor(final MQClientInstance mqClientFactory) { + this.mqClientFactory = mqClientFactory; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.CHECK_TRANSACTION_STATE: + return this.checkTransactionState(ctx, request); + case RequestCode.NOTIFY_CONSUMER_IDS_CHANGED: + return this.notifyConsumerIdsChanged(ctx, request); + case RequestCode.RESET_CONSUMER_CLIENT_OFFSET: + return this.resetOffset(ctx, request); + case RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT: + return this.getConsumeStatus(ctx, request); + + case RequestCode.GET_CONSUMER_RUNNING_INFO: + return this.getConsumerRunningInfo(ctx, request); + + case RequestCode.CONSUME_MESSAGE_DIRECTLY: + return this.consumeMessageDirectly(ctx, request); + default: + break; + } + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + public RemotingCommand checkTransactionState(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final CheckTransactionStateRequestHeader requestHeader = + (CheckTransactionStateRequestHeader) request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class); + final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); + final MessageExt messageExt = MessageDecoder.decode(byteBuffer); + if (messageExt != null) { + final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + if (group != null) { + MQProducerInner producer = this.mqClientFactory.selectProducer(group); + if (producer != null) { + final String addr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + producer.checkTransactionState(addr, messageExt, requestHeader); + } else { + log.debug("checkTransactionState, pick producer by group[{}] failed", group); + } + } else { + log.warn("checkTransactionState, pick producer group failed"); + } + } else { + log.warn("checkTransactionState, decode message failed"); + } + + return null; + } + + public RemotingCommand notifyConsumerIdsChanged(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + try { + final NotifyConsumerIdsChangedRequestHeader requestHeader = + (NotifyConsumerIdsChangedRequestHeader) request.decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class); + log.info("receive broker's notification[{}], the consumer group: {} changed, rebalance immediately", + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + requestHeader.getConsumerGroup()); + this.mqClientFactory.rebalanceImmediately(); + } catch (Exception e) { + log.error("notifyConsumerIdsChanged exception", RemotingHelper.exceptionSimpleDesc(e)); + } + return null; + } + + public RemotingCommand resetOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final ResetOffsetRequestHeader requestHeader = + (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class); + log.info("invoke reset offset operation from broker. brokerAddr={}, topic={}, group={}, timestamp={}", + new Object[]{RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), requestHeader.getGroup(), + requestHeader.getTimestamp()}); + Map offsetTable = new HashMap(); + if (request.getBody() != null) { + ResetOffsetBody body = ResetOffsetBody.decode(request.getBody(), ResetOffsetBody.class); + offsetTable = body.getOffsetTable(); + } + this.mqClientFactory.resetOffset(requestHeader.getTopic(), requestHeader.getGroup(), offsetTable); + return null; + } + + @Deprecated + public RemotingCommand getConsumeStatus(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerStatusRequestHeader requestHeader = + (GetConsumerStatusRequestHeader) request.decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class); + + Map offsetTable = this.mqClientFactory.getConsumerStatus(requestHeader.getTopic(), requestHeader.getGroup()); + GetConsumerStatusBody body = new GetConsumerStatusBody(); + body.setMessageQueueTable(offsetTable); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + return response; + } + + private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerRunningInfoRequestHeader requestHeader = + (GetConsumerRunningInfoRequestHeader) request.decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class); + + ConsumerRunningInfo consumerRunningInfo = this.mqClientFactory.consumerRunningInfo(requestHeader.getConsumerGroup()); + if (null != consumerRunningInfo) { + if (requestHeader.isJstackEnable()) { + Map map = Thread.getAllStackTraces(); + String jstack = UtilAll.jstack(map); + consumerRunningInfo.setJstack(jstack); + } + + response.setCode(ResponseCode.SUCCESS); + response.setBody(consumerRunningInfo.encode()); + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer Group <%s> not exist in this consumer", requestHeader.getConsumerGroup())); + } + + return response; + } + + private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ConsumeMessageDirectlyResultRequestHeader requestHeader = + (ConsumeMessageDirectlyResultRequestHeader) request + .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class); + + final MessageExt msg = MessageDecoder.decode(ByteBuffer.wrap(request.getBody())); + + ConsumeMessageDirectlyResult result = + this.mqClientFactory.consumeMessageDirectly(msg, requestHeader.getConsumerGroup(), requestHeader.getBrokerName()); + + if (null != result) { + response.setCode(ResponseCode.SUCCESS); + response.setBody(result.encode()); + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer Group <%s> not exist in this consumer", requestHeader.getConsumerGroup())); + } + + return response; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java new file mode 100644 index 0000000000..bc2f95d01d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +/** + * @author shijia.wxr + */ +public enum CommunicationMode { + SYNC, + ASYNC, + ONEWAY, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java new file mode 100644 index 0000000000..22805cd1b6 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +/** + * @author shijia.wxr + */ +public class FindBrokerResult { + private final String brokerAddr; + private final boolean slave; + + + public FindBrokerResult(String brokerAddr, boolean slave) { + this.brokerAddr = brokerAddr; + this.slave = slave; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public boolean isSlave() { + return slave; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java new file mode 100644 index 0000000000..9f7e964d5b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java @@ -0,0 +1,418 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import org.slf4j.Logger; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class MQAdminImpl { + + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private long timeoutMillis = 6000; + + + public MQAdminImpl(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + public long getTimeoutMillis() { + return timeoutMillis; + } + + + public void setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + try { + TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(key, timeoutMillis); + List brokerDataList = topicRouteData.getBrokerDatas(); + if (brokerDataList != null && !brokerDataList.isEmpty()) { + Collections.sort(brokerDataList); + + boolean createOKAtLeastOnce = false; + MQClientException exception = null; + + StringBuilder orderTopicString = new StringBuilder(); + + for (BrokerData brokerData : brokerDataList) { + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr != null) { + TopicConfig topicConfig = new TopicConfig(newTopic); + topicConfig.setReadQueueNums(queueNum); + topicConfig.setWriteQueueNums(queueNum); + topicConfig.setTopicSysFlag(topicSysFlag); + + boolean createOK = false; + for (int i = 0; i < 5; i++) { + try { + this.mQClientFactory.getMQClientAPIImpl().createTopic(addr, key, topicConfig, timeoutMillis); + createOK = true; + createOKAtLeastOnce = true; + break; + } catch (Exception e) { + if (4 == i) { + exception = new MQClientException("create topic to broker exception", e); + } + } + } + + if (createOK) { + orderTopicString.append(brokerData.getBrokerName()); + orderTopicString.append(":"); + orderTopicString.append(queueNum); + orderTopicString.append(";"); + } + } + } + + if (exception != null && !createOKAtLeastOnce) { + throw exception; + } + } else { + throw new MQClientException("Not found broker, maybe key is wrong", null); + } + } catch (Exception e) { + throw new MQClientException("create new topic failed", e); + } + } + + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + try { + TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis); + if (topicRouteData != null) { + TopicPublishInfo topicPublishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + return topicPublishInfo.getMessageQueueList(); + } + } + } catch (Exception e) { + throw new MQClientException("Can not find Message Queue for this topic, " + topic, e); + } + + throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null); + } + + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + try { + TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis); + if (topicRouteData != null) { + Set mqList = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + if (!mqList.isEmpty()) { + return mqList; + } else { + throw new MQClientException("Can not find Message Queue for this topic, " + topic + " Namesrv return empty", null); + } + } + } catch (Exception e) { + throw new MQClientException( + "Can not find Message Queue for this topic, " + topic + FAQUrl.suggestTodo(FAQUrl.MQLIST_NOT_EXIST), // + e); + } + + throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null); + } + + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp, + timeoutMillis); + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long maxOffset(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long minOffset(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getEarliestMsgStoretime(brokerAddr, mq.getTopic(), mq.getQueueId(), + timeoutMillis); + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + + MessageId messageId = null; + try { + messageId = MessageDecoder.decodeMessageId(msgId); + } catch (Exception e) { + throw new MQClientException(ResponseCode.NO_MESSAGE, "query message by id finished, but no message."); + } + return this.mQClientFactory.getMQClientAPIImpl().viewMessage(RemotingUtil.socketAddress2String(messageId.getAddress()), + messageId.getOffset(), timeoutMillis); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, + InterruptedException { + return queryMessage(topic, key, maxNum, begin, end, false); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws InterruptedException, MQClientException { + + QueryResult qr = this.queryMessage(topic, uniqKey, 32, + MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE, true); + if (qr != null && qr.getMessageList() != null && qr.getMessageList().size() > 0) { + return qr.getMessageList().get(0); + } else { + return null; + } + } + + protected QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end, boolean isUniqKey) throws MQClientException, + InterruptedException { + TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + if (null == topicRouteData) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + } + + if (topicRouteData != null) { + List brokerAddrs = new LinkedList(); + for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + brokerAddrs.add(addr); + } + } + + if (!brokerAddrs.isEmpty()) { + final CountDownLatch countDownLatch = new CountDownLatch(brokerAddrs.size()); + final List queryResultList = new LinkedList(); + + for (String addr : brokerAddrs) { + try { + QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setKey(key); + requestHeader.setMaxNum(maxNum); + requestHeader.setBeginTimestamp(begin); + requestHeader.setEndTimestamp(end); + + this.mQClientFactory.getMQClientAPIImpl().queryMessage(addr, requestHeader, timeoutMillis * 3, + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + try { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryMessageResponseHeader responseHeader = null; + try { + responseHeader = + (QueryMessageResponseHeader) response + .decodeCommandCustomHeader(QueryMessageResponseHeader.class); + } catch (RemotingCommandException e) { + log.error("decodeCommandCustomHeader exception", e); + return; + } + + List wrappers = + MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); + + QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); + queryResultList.add(qr); + break; + } + default: + log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); + break; + } + } else { + log.warn("getResponseCommand return null"); + } + } finally { + countDownLatch.countDown(); + } + } + }, isUniqKey); + } catch (Exception e) { + log.warn("queryMessage exception", e); + } + + } + + boolean ok = countDownLatch.await(timeoutMillis * 4, TimeUnit.MILLISECONDS); + if (!ok) { + log.warn("queryMessage, maybe some broker failed"); + } + + long indexLastUpdateTimestamp = 0; + List messageList = new LinkedList(); + for (QueryResult qr : queryResultList) { + if (qr.getIndexLastUpdateTimestamp() > indexLastUpdateTimestamp) { + indexLastUpdateTimestamp = qr.getIndexLastUpdateTimestamp(); + } + + for (MessageExt msgExt : qr.getMessageList()) { + if (isUniqKey) { + if (msgExt.getMsgId().equals(key)) { + + if (messageList.size() > 0) { + + if (messageList.get(0).getStoreTimestamp() > msgExt.getStoreTimestamp()) { + + messageList.clear(); + messageList.add(msgExt); + } + + } else { + + messageList.add(msgExt); + } + } else { + log.warn("queryMessage by uniqKey, find message key not matched, maybe hash duplicate {}", msgExt.toString()); + } + } else { + String keys = msgExt.getKeys(); + if (keys != null) { + boolean matched = false; + String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR); + if (keyArray != null) { + for (String k : keyArray) { + if (key.equals(k)) { + matched = true; + break; + } + } + } + + if (matched) { + messageList.add(msgExt); + } else { + log.warn("queryMessage, find message key not matched, maybe hash duplicate {}", msgExt.toString()); + } + } + } + } + } + + if (!messageList.isEmpty()) { + return new QueryResult(indexLastUpdateTimestamp, messageList); + } else { + throw new MQClientException(ResponseCode.NO_MESSAGE, "query message by key finished, but no message."); + } + } + } + + throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "The topic[" + topic + "] not matched route info"); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java new file mode 100644 index 0000000000..3d5ba28286 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java @@ -0,0 +1,1996 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.SendMessageContext; +import com.alibaba.rocketmq.client.impl.consumer.PullResultExt; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.SendCallback; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.client.producer.SendStatus; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.namesrv.TopAddressing; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.header.*; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.*; +import com.alibaba.rocketmq.common.protocol.heartbeat.HeartbeatData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import org.slf4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class MQClientAPIImpl { + + private final static Logger log = ClientLogger.getLog(); + public static boolean sendSmartMsg = + Boolean.parseBoolean(System.getProperty("com.alibaba.rocketmq.client.sendSmartMsg", "true")); + + static { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + } + + private final RemotingClient remotingClient; + private final TopAddressing topAddressing; + private final ClientRemotingProcessor clientRemotingProcessor; + private String nameSrvAddr = null; + private ClientConfig clientConfig; + + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, final ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook, final ClientConfig clientConfig) { + this.clientConfig = clientConfig; + topAddressing = new TopAddressing(MixAll.WS_ADDR, clientConfig.getUnitName()); + this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); + this.clientRemotingProcessor = clientRemotingProcessor; + + this.remotingClient.registerRPCHook(rpcHook); + this.remotingClient.registerProcessor(RequestCode.CHECK_TRANSACTION_STATE, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_RUNNING_INFO, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.CONSUME_MESSAGE_DIRECTLY, this.clientRemotingProcessor, null); + } + + public List getNameServerAddressList() { + return this.remotingClient.getNameServerAddressList(); + } + + public RemotingClient getRemotingClient() { + return remotingClient; + } + + public String fetchNameServerAddr() { + try { + String addrs = this.topAddressing.fetchNSAddr(); + if (addrs != null) { + if (!addrs.equals(this.nameSrvAddr)) { + log.info("name server address changed, old=" + this.nameSrvAddr + ", new=" + addrs); + this.updateNameServerAddressList(addrs); + this.nameSrvAddr = addrs; + return nameSrvAddr; + } + } + } catch (Exception e) { + log.error("fetchNameServerAddr Exception", e); + } + return nameSrvAddr; + } + + public void updateNameServerAddressList(final String addrs) { + List lst = new ArrayList(); + String[] addrArray = addrs.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + lst.add(addr); + } + + this.remotingClient.updateNameServerAddressList(lst); + } + } + + public void start() { + this.remotingClient.start(); + } + + public void shutdown() { + this.remotingClient.shutdown(); + } + + public void createSubscriptionGroup(final String addr, final SubscriptionGroupConfig config, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); + + byte[] body = RemotingSerializable.encode(config); + request.setBody(body); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + + } + + public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); + requestHeader.setTopic(topicConfig.getTopicName()); + requestHeader.setDefaultTopic(defaultTopic); + requestHeader.setReadQueueNums(topicConfig.getReadQueueNums()); + requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums()); + requestHeader.setPerm(topicConfig.getPerm()); + requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name()); + requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag()); + requestHeader.setOrder(topicConfig.isOrder()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + public SendResult sendMessage(// + final String addr, // 1 + final String brokerName, // 2 + final Message msg, // 3 + final SendMessageRequestHeader requestHeader, // 4 + final long timeoutMillis, // 5 + final CommunicationMode communicationMode, // 6 + final SendMessageContext context, // 7 + final DefaultMQProducerImpl producer // 8 + ) throws RemotingException, MQBrokerException, InterruptedException { + return sendMessage(addr, brokerName, msg, requestHeader, timeoutMillis, communicationMode, null, null, null, 0, context, producer); + } + + public SendResult sendMessage(// + final String addr, // 1 + final String brokerName, // 2 + final Message msg, // 3 + final SendMessageRequestHeader requestHeader, // 4 + final long timeoutMillis, // 5 + final CommunicationMode communicationMode, // 6 + final SendCallback sendCallback, // 7 + final TopicPublishInfo topicPublishInfo, // 8 + final MQClientInstance instance, // 9 + final int retryTimesWhenSendFailed, // 10 + final SendMessageContext context, // 11 + final DefaultMQProducerImpl producer // 12 + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = null; + if (sendSmartMsg) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + } + + request.setBody(msg.getBody()); + + switch (communicationMode) { + case ONEWAY: + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + return null; + case ASYNC: + final AtomicInteger times = new AtomicInteger(); + this.sendMessageAsync(addr, brokerName, msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, context, producer); + return null; + case SYNC: + return this.sendMessageSync(addr, brokerName, msg, timeoutMillis, request); + default: + assert false; + break; + } + + return null; + } + + private SendResult sendMessageSync(// + final String addr, // + final String brokerName, // + final Message msg, // + final long timeoutMillis, // + final RemotingCommand request// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + return this.processSendResponse(brokerName, msg, response); + } + + private void sendMessageAsync(// + final String addr, // + final String brokerName, // + final Message msg, // + final long timeoutMillis, // + final RemotingCommand request, // + final SendCallback sendCallback, // + final TopicPublishInfo topicPublishInfo, // + final MQClientInstance instance, // + final int retryTimesWhenSendFailed, // + final AtomicInteger times, // + final SendMessageContext context, // + final DefaultMQProducerImpl producer // + ) throws InterruptedException, RemotingException { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + RemotingCommand response = responseFuture.getResponseCommand(); + if (null == sendCallback && response != null) { + + try { + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response); + if (context != null && sendResult != null) { + context.setSendResult(sendResult); + context.getProducer().executeSendMessageHookAfter(context); + } + } catch (Throwable e) { + // + } + + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); + return; + } + + if (response != null) { + try { + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response); + assert sendResult != null; + if (context != null) { + context.setSendResult(sendResult); + context.getProducer().executeSendMessageHookAfter(context); + } + + try { + sendCallback.onSuccess(sendResult); + } catch (Throwable e) { + } + + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); + } catch (Exception e) { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); + onExceptionImpl(brokerName, msg, 0L, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); + } + } else { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); + if (!responseFuture.isSendRequestOK()) { + MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); + onExceptionImpl(brokerName, msg, 0L, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } else if (responseFuture.isTimeout()) { + MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", + responseFuture.getCause()); + onExceptionImpl(brokerName, msg, 0L, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } else { + MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); + onExceptionImpl(brokerName, msg, 0L, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } + } + } + }); + } + + + private void onExceptionImpl(final String brokerName, // + final Message msg, // + final long timeoutMillis, // + final RemotingCommand request, // + final SendCallback sendCallback, // + final TopicPublishInfo topicPublishInfo, // + final MQClientInstance instance, // + final int timesTotal, // + final AtomicInteger curTimes, // + final Exception e, // + final SendMessageContext context, // + final boolean needRetry, // + final DefaultMQProducerImpl producer // 12 + ) { + int tmp = curTimes.incrementAndGet(); + if (needRetry && tmp <= timesTotal) { + MessageQueue tmpmq = producer.selectOneMessageQueue(topicPublishInfo, brokerName); + String addr = instance.findBrokerAddressInPublish(tmpmq.getBrokerName()); + log.info("async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}", tmp, msg.getTopic(), addr, + tmpmq.getBrokerName()); + try { + request.setOpaque(RemotingCommand.createNewRequestId()); + sendMessageAsync(addr, tmpmq.getBrokerName(), msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, + timesTotal, curTimes, context, producer); + } catch (InterruptedException e1) { + onExceptionImpl(tmpmq.getBrokerName(), msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1, + context, false, producer); + } catch (RemotingConnectException e1) { + producer.updateFaultItem(brokerName, 3000, true); + onExceptionImpl(tmpmq.getBrokerName(), msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1, + context, true, producer); + } catch (RemotingTooMuchRequestException e1) { + onExceptionImpl(tmpmq.getBrokerName(), msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1, + context, false, producer); + } catch (RemotingException e1) { + producer.updateFaultItem(brokerName, 3000, true); + onExceptionImpl(tmpmq.getBrokerName(), msg, timeoutMillis, request, sendCallback, topicPublishInfo, instance, timesTotal, curTimes, e1, + context, true, producer); + } + } else { + if (context != null) { + context.setException(e); + context.getProducer().executeSendMessageHookAfter(context); + } + try { + sendCallback.onException(e); + } catch (Exception e2) { + } + } + } + + + private SendResult processSendResponse(// + final String brokerName, // + final Message msg, // + final RemotingCommand response// + ) throws MQBrokerException, RemotingCommandException { + switch (response.getCode()) { + case ResponseCode.FLUSH_DISK_TIMEOUT: + case ResponseCode.FLUSH_SLAVE_TIMEOUT: + case ResponseCode.SLAVE_NOT_AVAILABLE: { + // TODO LOG + } + case ResponseCode.SUCCESS: { + SendStatus sendStatus = SendStatus.SEND_OK; + switch (response.getCode()) { + case ResponseCode.FLUSH_DISK_TIMEOUT: + sendStatus = SendStatus.FLUSH_DISK_TIMEOUT; + break; + case ResponseCode.FLUSH_SLAVE_TIMEOUT: + sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT; + break; + case ResponseCode.SLAVE_NOT_AVAILABLE: + sendStatus = SendStatus.SLAVE_NOT_AVAILABLE; + break; + case ResponseCode.SUCCESS: + sendStatus = SendStatus.SEND_OK; + break; + default: + assert false; + break; + } + + SendMessageResponseHeader responseHeader = + (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + + MessageQueue messageQueue = new MessageQueue(msg.getTopic(), brokerName, responseHeader.getQueueId()); + + SendResult sendResult = new SendResult(sendStatus, + MessageClientIDSetter.getUniqID(msg), + responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); + sendResult.setTransactionId(responseHeader.getTransactionId()); + String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); + String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); + if (regionId == null || regionId.isEmpty()) { + regionId = MixAll.DEFAULT_TRACE_REGION_ID; + } + if (traceOn != null && traceOn.equals("false")) { + sendResult.setTraceOn(false); + } else { + sendResult.setTraceOn(true); + } + sendResult.setRegionId(regionId); + return sendResult; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public PullResult pullMessage(// + final String addr, // + final PullMessageRequestHeader requestHeader, // + final long timeoutMillis, // + final CommunicationMode communicationMode, // + final PullCallback pullCallback// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); + + switch (communicationMode) { + case ONEWAY: + assert false; + return null; + case ASYNC: + this.pullMessageAsync(addr, request, timeoutMillis, pullCallback); + return null; + case SYNC: + return this.pullMessageSync(addr, request, timeoutMillis); + default: + assert false; + break; + } + + return null; + } + + + private void pullMessageAsync(// + final String addr, // 1 + final RemotingCommand request, // + final long timeoutMillis, // + final PullCallback pullCallback// + ) throws RemotingException, InterruptedException { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + try { + PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response); + assert pullResult != null; + pullCallback.onSuccess(pullResult); + } catch (Exception e) { + pullCallback.onException(e); + } + } else { + if (!responseFuture.isSendRequestOK()) { + pullCallback.onException(new MQClientException("send request failed", responseFuture.getCause())); + } else if (responseFuture.isTimeout()) { + pullCallback.onException(new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", + responseFuture.getCause())); + } else { + pullCallback.onException(new MQClientException("unknow reseaon", responseFuture.getCause())); + } + } + } + }); + } + + private PullResult pullMessageSync(// + final String addr, // 1 + final RemotingCommand request, // 2 + final long timeoutMillis// 3 + ) throws RemotingException, InterruptedException, MQBrokerException { + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + return this.processPullResponse(response); + } + + private PullResult processPullResponse(final RemotingCommand response) throws MQBrokerException, RemotingCommandException { + PullStatus pullStatus = PullStatus.NO_NEW_MSG; + switch (response.getCode()) { + case ResponseCode.SUCCESS: + pullStatus = PullStatus.FOUND; + break; + case ResponseCode.PULL_NOT_FOUND: + pullStatus = PullStatus.NO_NEW_MSG; + break; + case ResponseCode.PULL_RETRY_IMMEDIATELY: + pullStatus = PullStatus.NO_MATCHED_MSG; + break; + case ResponseCode.PULL_OFFSET_MOVED: + pullStatus = PullStatus.OFFSET_ILLEGAL; + break; + + default: + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + PullMessageResponseHeader responseHeader = + (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); + + return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(), + responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody()); + } + + public MessageExt viewMessage(final String addr, final long phyoffset, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + ViewMessageRequestHeader requestHeader = new ViewMessageRequestHeader(); + requestHeader.setOffset(phyoffset); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody()); + MessageExt messageExt = MessageDecoder.clientDecode(byteBuffer, true); + return messageExt; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public long searchOffset(final String addr, final String topic, final int queueId, final long timestamp, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + requestHeader.setTimestamp(timestamp); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + SearchOffsetResponseHeader responseHeader = + (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public long getMaxOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetMaxOffsetResponseHeader responseHeader = + (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public List getConsumerIdListByGroup(// + final String addr, // + final String consumerGroup, // + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + MQBrokerException, InterruptedException { + GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerListByGroupResponseBody body = + GetConsumerListByGroupResponseBody.decode(response.getBody(), GetConsumerListByGroupResponseBody.class); + return body.getConsumerIdList(); + } + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public long getMinOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetMinOffsetResponseHeader responseHeader = + (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public long getEarliestMsgStoretime(final String addr, final String topic, final int queueId, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + GetEarliestMsgStoretimeRequestHeader requestHeader = new GetEarliestMsgStoretimeRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_EARLIEST_MSG_STORETIME, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetEarliestMsgStoretimeResponseHeader responseHeader = + (GetEarliestMsgStoretimeResponseHeader) response.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class); + + return responseHeader.getTimestamp(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public long queryConsumerOffset(// + final String addr, // + final QueryConsumerOffsetRequestHeader requestHeader, // + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void updateConsumerOffset(// + final String addr, // + final UpdateConsumerOffsetRequestHeader requestHeader, // + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void updateConsumerOffsetOneway(// + final String addr, // + final UpdateConsumerOffsetRequestHeader requestHeader, // + final long timeoutMillis// + ) throws RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException, + InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader); + + this.remotingClient.invokeOneway(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); + } + + + public void sendHearbeat(// + final String addr, // + final HeartbeatData heartbeatData, // + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + + request.setBody(heartbeatData.encode()); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void unregisterClient(// + final String addr, // + final String clientID, // + final String producerGroup, // + final String consumerGroup, // + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + final UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader(); + requestHeader.setClientID(clientID); + requestHeader.setProducerGroup(producerGroup); + requestHeader.setConsumerGroup(consumerGroup); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void endTransactionOneway(// + final String addr, // + final EndTransactionRequestHeader requestHeader, // + final String remark, // + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader); + + request.setRemark(remark); + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + } + + + public void queryMessage( + final String addr, + final QueryMessageRequestHeader requestHeader, + final long timeoutMillis, + final InvokeCallback invokeCallback, + final Boolean isUnqiueKey + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader); + request.addExtField(MixAll.UNIQUE_MSG_QUERY_FLAG, isUnqiueKey.toString()); + this.remotingClient.invokeAsync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis, + invokeCallback); + } + + + public boolean registerClient(final String addr, final HeartbeatData heartbeat, final long timeoutMillis) + throws RemotingException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + + request.setBody(heartbeat.encode()); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + return response.getCode() == ResponseCode.SUCCESS; + } + + + public void consumerSendMessageBack( + final String addr, + final MessageExt msg, + final String consumerGroup, + final int delayLevel, + final long timeoutMillis, + final int maxConsumeRetryTimes + ) throws RemotingException, MQBrokerException, InterruptedException { + ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); + + requestHeader.setGroup(consumerGroup); + requestHeader.setOriginTopic(msg.getTopic()); + requestHeader.setOffset(msg.getCommitLogOffset()); + requestHeader.setDelayLevel(delayLevel); + requestHeader.setOriginMsgId(msg.getMsgId()); + requestHeader.setMaxReconsumeTimes(maxConsumeRetryTimes); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public Set lockBatchMQ(// + final String addr, // + final LockBatchRequestBody requestBody, // + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); + + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); + Set messageQueues = responseBody.getLockOKMQSet(); + return messageQueues; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void unlockBatchMQ(// + final String addr, // + final UnlockBatchRequestBody requestBody, // + final long timeoutMillis, // + final boolean oneway// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null); + + request.setBody(requestBody.encode()); + + if (oneway) { + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + } else { + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + + public TopicStatsTable getTopicStatsInfo(final String addr, final String topic, final long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + GetTopicStatsInfoRequestHeader requestHeader = new GetTopicStatsInfoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + TopicStatsTable topicStatsTable = TopicStatsTable.decode(response.getBody(), TopicStatsTable.class); + return topicStatsTable; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final long timeoutMillis) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, + MQBrokerException { + return getConsumeStats(addr, consumerGroup, null, timeoutMillis); + } + + + public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, final String topic, final long timeoutMillis) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, + MQBrokerException { + GetConsumeStatsRequestHeader requestHeader = new GetConsumeStatsRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ConsumeStats consumeStats = ConsumeStats.decode(response.getBody(), ConsumeStats.class); + return consumeStats; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public ProducerConnection getProducerConnectionList(final String addr, final String producerGroup, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException { + GetProducerConnectionListRequestHeader requestHeader = new GetProducerConnectionListRequestHeader(); + requestHeader.setProducerGroup(producerGroup); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_PRODUCER_CONNECTION_LIST, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return ProducerConnection.decode(response.getBody(), ProducerConnection.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public ConsumerConnection getConsumerConnectionList(final String addr, final String consumerGroup, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException { + GetConsumerConnectionListRequestHeader requestHeader = new GetConsumerConnectionListRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_CONNECTION_LIST, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ConsumerConnection consumerConnection = ConsumerConnection.decode(response.getBody(), ConsumerConnection.class); + return consumerConnection; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public KVTable getBrokerRuntimeInfo(final String addr, final long timeoutMillis) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_RUNTIME_INFO, null); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return KVTable.decode(response.getBody(), KVTable.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void updateBrokerConfig(final String addr, final Properties properties, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException, UnsupportedEncodingException { + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null); + + String str = MixAll.properties2String(properties); + if (str != null && str.length() > 0) { + request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET)); + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + + public Properties getBrokerConfig(final String addr, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException, UnsupportedEncodingException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CONFIG, null); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return MixAll.string2Properties(new String(response.getBody(), MixAll.DEFAULT_CHARSET)); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public ClusterInfo getBrokerClusterInfo(final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ClusterInfo responseBody = ClusterInfo.decode(response.getBody(), ClusterInfo.class); + return responseBody; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public TopicRouteData getDefaultTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.TOPIC_NOT_EXIST: { + // TODO LOG + break; + } + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return TopicRouteData.decode(body, TopicRouteData.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.TOPIC_NOT_EXIST: { + if (!topic.equals(MixAll.DEFAULT_TOPIC)) + log.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic); + break; + } + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return TopicRouteData.decode(body, TopicRouteData.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public TopicList getTopicListFromNameServer(final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName, final long timeoutMillis) throws RemotingCommandException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException { + WipeWritePermOfBrokerRequestHeader requestHeader = new WipeWritePermOfBrokerRequestHeader(); + requestHeader.setBrokerName(brokerName); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.WIPE_WRITE_PERM_OF_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + WipeWritePermOfBrokerResponseHeader responseHeader = + (WipeWritePermOfBrokerResponseHeader) response.decodeCommandCustomHeader(WipeWritePermOfBrokerResponseHeader.class); + return responseHeader.getWipeTopicCount(); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteTopicInBroker(final String addr, final String topic, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteTopicInNameServer(final String addr, final String topic, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteSubscriptionGroup(final String addr, final String groupName, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + DeleteSubscriptionGroupRequestHeader requestHeader = new DeleteSubscriptionGroupRequestHeader(); + requestHeader.setGroupName(groupName); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public String getKVConfigValue(final String namespace, final String key, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetKVConfigRequestHeader requestHeader = new GetKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetKVConfigResponseHeader responseHeader = + (GetKVConfigResponseHeader) response.decodeCommandCustomHeader(GetKVConfigResponseHeader.class); + return responseHeader.getValue(); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void putKVConfigValue(final String namespace, final String key, final String value, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + PutKVConfigRequestHeader requestHeader = new PutKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + requestHeader.setValue(value); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUT_KV_CONFIG, requestHeader); + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + RemotingCommand errResponse = null; + for (String namesrvAddr : nameServerAddressList) { + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + } + + + public void deleteKVConfigValue(final String namespace, final String key, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + DeleteKVConfigRequestHeader requestHeader = new DeleteKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG, requestHeader); + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + RemotingCommand errResponse = null; + for (String namesrvAddr : nameServerAddressList) { + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + } + + + public KVTable getKVListByNamespace(final String namespace, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetKVListByNamespaceRequestHeader requestHeader = new GetKVListByNamespaceRequestHeader(); + requestHeader.setNamespace(namespace); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KVLIST_BY_NAMESPACE, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return KVTable.decode(response.getBody(), KVTable.class); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public Map invokeBrokerToResetOffset(final String addr, final String topic, final String group, + final long timestamp, final boolean isForce, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + return invokeBrokerToResetOffset(addr, topic, group, timestamp, isForce, timeoutMillis, false); + } + + + public Map invokeBrokerToResetOffset(final String addr, final String topic, final String group, + final long timestamp, final boolean isForce, final long timeoutMillis, boolean isC) + throws RemotingException, MQClientException, InterruptedException { + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setTimestamp(timestamp); + requestHeader.setForce(isForce); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader); + if (isC) { + request.setLanguage(LanguageCode.CPP); + } + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + ResetOffsetBody body = ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class); + return body.getOffsetTable(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public Map> invokeBrokerToGetConsumerStatus(final String addr, final String topic, final String group, + final String clientAddr, final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setClientAddr(clientAddr); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerStatusBody body = GetConsumerStatusBody.decode(response.getBody(), GetConsumerStatusBody.class); + return body.getConsumerTable(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public GroupList queryTopicConsumeByWho(final String addr, final String topic, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException { + QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GroupList groupList = GroupList.decode(response.getBody(), GroupList.class); + return groupList; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public List queryConsumeTimeSpan(final String addr, final String topic, final String group, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, + MQBrokerException { + QueryConsumeTimeSpanRequestHeader requestHeader = new QueryConsumeTimeSpanRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_TIME_SPAN, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryConsumeTimeSpanBody consumeTimeSpanBody = GroupList.decode(response.getBody(), QueryConsumeTimeSpanBody.class); + return consumeTimeSpanBody.getConsumeTimeSpanSet(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public TopicList getTopicsByCluster(final String cluster, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetTopicsByClusterRequestHeader requestHeader = new GetTopicsByClusterRequestHeader(); + requestHeader.setCluster(cluster); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPICS_BY_CLUSTER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void registerMessageFilterClass(final String addr, // + final String consumerGroup, // + final String topic, // + final String className, // + final int classCRC, // + final byte[] classBody, // + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException { + RegisterMessageFilterClassRequestHeader requestHeader = new RegisterMessageFilterClassRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClassName(className); + requestHeader.setTopic(topic); + requestHeader.setClassCRC(classCRC); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REGISTER_MESSAGE_FILTER_CLASS, requestHeader); + request.setBody(classBody); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public TopicList getSystemTopicList(final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (topicList.getTopicList() != null && !topicList.getTopicList().isEmpty() + && !UtilAll.isBlank(topicList.getBrokerAddr())) { + TopicList tmp = getSystemTopicListFromBroker(topicList.getBrokerAddr(), timeoutMillis); + if (tmp.getTopicList() != null && !tmp.getTopicList().isEmpty()) { + topicList.getTopicList().addAll(tmp.getTopicList()); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public TopicList getSystemTopicListFromBroker(final String addr, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public boolean cleanExpiredConsumeQueue(final String addr, long timeoutMillis) throws MQClientException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null); + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return true; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public boolean cleanUnusedTopicByAddr(final String addr, long timeoutMillis) throws MQClientException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLEAN_UNUSED_TOPIC, null); + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return true; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + public ConsumerRunningInfo getConsumerRunningInfo(final String addr, String consumerGroup, String clientId, boolean jstack, + final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClientId(clientId); + requestHeader.setJstackEnable(jstack); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + ConsumerRunningInfo info = ConsumerRunningInfo.decode(body, ConsumerRunningInfo.class); + return info; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String addr, // + String consumerGroup, // + String clientId, // + String msgId, // + final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClientId(clientId); + requestHeader.setMsgId(msgId); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + ConsumeMessageDirectlyResult info = ConsumeMessageDirectlyResult.decode(body, ConsumeMessageDirectlyResult.class); + return info; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + public Map queryCorrectionOffset(final String addr, final String topic, final String group, Set filterGroup, + long timeoutMillis) throws MQClientException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException { + QueryCorrectionOffsetHeader requestHeader = new QueryCorrectionOffsetHeader(); + requestHeader.setCompareGroup(group); + requestHeader.setTopic(topic); + if (filterGroup != null) { + StringBuilder sb = new StringBuilder(); + String splitor = ""; + for (String s : filterGroup) { + sb.append(splitor).append(s); + splitor = ","; + } + requestHeader.setFilterGroups(sb.toString()); + } + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + QueryCorrectionOffsetBody body = QueryCorrectionOffsetBody.decode(response.getBody(), QueryCorrectionOffsetBody.class); + return body.getCorrectionOffsets(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + public TopicList getUnitTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_UNIT_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public TopicList getHasUnitSubTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public TopicList getHasUnitSubUnUnitTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void cloneGroupOffset(final String addr, final String srcGroup, final String destGroup, final String topic, + final boolean isOffline, final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + CloneGroupOffsetRequestHeader requestHeader = new CloneGroupOffsetRequestHeader(); + requestHeader.setSrcGroup(srcGroup); + requestHeader.setDestGroup(destGroup); + requestHeader.setTopic(topic); + requestHeader.setOffline(isOffline); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLONE_GROUP_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public BrokerStatsData viewBrokerStatsData(String brokerAddr, String statsName, String statsKey, long timeoutMillis) + throws MQClientException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException { + ViewBrokerStatsDataRequestHeader requestHeader = new ViewBrokerStatsDataRequestHeader(); + requestHeader.setStatsName(statsName); + requestHeader.setStatsKey(statsKey); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.VIEW_BROKER_STATS_DATA, requestHeader); + + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return BrokerStatsData.decode(body, BrokerStatsData.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public Set getClusterList(String topic, long timeoutMillis) throws MQClientException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + // todo:jodie + return Collections.EMPTY_SET; + } + + + public ConsumeStatsList fetchConsumeStatsInBroker(String brokerAddr, boolean isOrder, long timeoutMillis) throws MQClientException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + GetConsumeStatsInBrokerHeader requestHeader = new GetConsumeStatsInBrokerHeader(); + requestHeader.setIsOrder(isOrder); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CONSUME_STATS, requestHeader); + + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return ConsumeStatsList.decode(body, ConsumeStatsList.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public SubscriptionGroupWrapper getAllSubscriptionGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null); + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return SubscriptionGroupWrapper.decode(response.getBody(), SubscriptionGroupWrapper.class); + } + default: + break; + } + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public TopicConfigSerializeWrapper getAllTopicConfig(final String addr, long timeoutMillis) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public void updateNameServerConfig(final Properties properties, final List nameServers, long timeoutMillis) + throws UnsupportedEncodingException, + MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQClientException { + String str = MixAll.properties2String(properties); + if (str == null || str.length() < 1) { + return; + } + List invokeNameServers = (nameServers == null || nameServers.isEmpty()) ? + this.remotingClient.getNameServerAddressList() : nameServers; + if (invokeNameServers == null || invokeNameServers.isEmpty()) { + return; + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_NAMESRV_CONFIG, null); + request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET)); + + RemotingCommand errResponse = null; + for (String nameServer : invokeNameServers) { + RemotingCommand response = this.remotingClient.invokeSync(nameServer, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + + public Map getNameServerConfig(final List nameServers, long timeoutMillis) + throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, + MQClientException, UnsupportedEncodingException { + List invokeNameServers = (nameServers == null || nameServers.isEmpty()) ? + this.remotingClient.getNameServerAddressList() : nameServers; + if (invokeNameServers == null || invokeNameServers.isEmpty()) { + return null; + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_NAMESRV_CONFIG, null); + + Map configMap = new HashMap(4); + for (String nameServer : invokeNameServers) { + RemotingCommand response = this.remotingClient.invokeSync(nameServer, request, timeoutMillis); + + assert response != null; + + if (ResponseCode.SUCCESS == response.getCode()) { + configMap.put(nameServer, MixAll.string2Properties(new String(response.getBody(), MixAll.DEFAULT_CHARSET))); + } else { + throw new MQClientException(response.getCode(), response.getRemark()); + } + } + return configMap; + } + +} \ No newline at end of file diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java new file mode 100644 index 0000000000..19016cae0a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.remoting.RPCHook; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class MQClientManager { + private static MQClientManager instance = new MQClientManager(); + private AtomicInteger factoryIndexGenerator = new AtomicInteger(); + private ConcurrentHashMap factoryTable = + new ConcurrentHashMap(); + + + private MQClientManager() { + + } + + + public static MQClientManager getInstance() { + return instance; + } + + public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig) { + return getAndCreateMQClientInstance(clientConfig, null); + } + + public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) { + String clientId = clientConfig.buildMQClientId(); + MQClientInstance instance = this.factoryTable.get(clientId); + if (null == instance) { + instance = + new MQClientInstance(clientConfig.cloneClientConfig(), + this.factoryIndexGenerator.getAndIncrement(), clientId, rpcHook); + MQClientInstance prev = this.factoryTable.putIfAbsent(clientId, instance); + if (prev != null) { + instance = prev; + } else { + // TODO log + } + } + + return instance; + } + + public void removeClientFactory(final String clientId) { + this.factoryTable.remove(clientId); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java new file mode 100644 index 0000000000..4dee764c1a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java @@ -0,0 +1,471 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeReturnType; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.CMResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import org.slf4j.Logger; + +import java.util.*; +import java.util.concurrent.*; + + +/** + * @author shijia.wxr + */ +public class ConsumeMessageConcurrentlyService implements ConsumeMessageService { + private static final Logger log = ClientLogger.getLog(); + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + private final DefaultMQPushConsumer defaultMQPushConsumer; + private final MessageListenerConcurrently messageListener; + private final BlockingQueue consumeRequestQueue; + private final ThreadPoolExecutor consumeExecutor; + private final String consumerGroup; + + private final ScheduledExecutorService scheduledExecutorService; + private final ScheduledExecutorService cleanExpireMsgExecutors; + + + public ConsumeMessageConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl, + MessageListenerConcurrently messageListener) { + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + this.messageListener = messageListener; + + this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer(); + this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup(); + this.consumeRequestQueue = new LinkedBlockingQueue(); + + this.consumeExecutor = new ThreadPoolExecutor(// + this.defaultMQPushConsumer.getConsumeThreadMin(), // + this.defaultMQPushConsumer.getConsumeThreadMax(), // + 1000 * 60, // + TimeUnit.MILLISECONDS, // + this.consumeRequestQueue, // + new ThreadFactoryImpl("ConsumeMessageThread_")); + + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_")); + this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CleanExpireMsgScheduledThread_")); + } + + + public void start() { + this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + cleanExpireMsg(); + } + + }, this.defaultMQPushConsumer.getConsumeTimeout(), this.defaultMQPushConsumer.getConsumeTimeout(), TimeUnit.MINUTES); + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + this.consumeExecutor.shutdown(); + this.cleanExpireMsgExecutors.shutdown(); + } + + @Override + public void updateCorePoolSize(int corePoolSize) { + if (corePoolSize > 0 // + && corePoolSize <= Short.MAX_VALUE // + && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { + this.consumeExecutor.setCorePoolSize(corePoolSize); + } + } + + @Override + public void incCorePoolSize() { + // long corePoolSize = this.consumeExecutor.getCorePoolSize(); + // if (corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) + // { + // this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize() + // + 1); + // } + // + // log.info("incCorePoolSize Concurrently from {} to {}, ConsumerGroup: + // {}", // + // corePoolSize,// + // this.consumeExecutor.getCorePoolSize(),// + // this.consumerGroup); + } + + @Override + public void decCorePoolSize() { + // long corePoolSize = this.consumeExecutor.getCorePoolSize(); + // if (corePoolSize > this.defaultMQPushConsumer.getConsumeThreadMin()) + // { + // this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize() + // - 1); + // } + // + // log.info("decCorePoolSize Concurrently from {} to {}, ConsumerGroup: + // {}", // + // corePoolSize,// + // this.consumeExecutor.getCorePoolSize(),// + // this.consumerGroup); + } + + @Override + public int getCorePoolSize() { + return this.consumeExecutor.getCorePoolSize(); + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) { + ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); + result.setOrder(false); + result.setAutoCommit(true); + + List msgs = new ArrayList(); + msgs.add(msg); + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(brokerName); + mq.setTopic(msg.getTopic()); + mq.setQueueId(msg.getQueueId()); + + ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(mq); + + this.resetRetryTopic(msgs); + + final long beginTime = System.currentTimeMillis(); + + log.info("consumeMessageDirectly receive new messge: {}", msg); + + try { + ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context); + if (status != null) { + switch (status) { + case CONSUME_SUCCESS: + result.setConsumeResult(CMResult.CR_SUCCESS); + break; + case RECONSUME_LATER: + result.setConsumeResult(CMResult.CR_LATER); + break; + default: + break; + } + } else { + result.setConsumeResult(CMResult.CR_RETURN_NULL); + } + } catch (Throwable e) { + result.setConsumeResult(CMResult.CR_THROW_EXCEPTION); + result.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + + log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s", // + RemotingHelper.exceptionSimpleDesc(e), // + ConsumeMessageConcurrentlyService.this.consumerGroup, // + msgs, // + mq), e); + } + + result.setSpentTimeMills(System.currentTimeMillis() - beginTime); + + log.info("consumeMessageDirectly Result: {}", result); + + return result; + } + + @Override + public void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispatchToConsume) { + final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); + if (msgs.size() <= consumeBatchSize) { + ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue); + try { + this.consumeExecutor.submit(consumeRequest); + } catch (RejectedExecutionException e) { + this.submitConsumeRequestLater(consumeRequest); + } + } else { + for (int total = 0; total < msgs.size(); ) { + List msgThis = new ArrayList(consumeBatchSize); + for (int i = 0; i < consumeBatchSize; i++, total++) { + if (total < msgs.size()) { + msgThis.add(msgs.get(total)); + } else { + break; + } + } + + ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue); + try { + this.consumeExecutor.submit(consumeRequest); + } catch (RejectedExecutionException e) { + for (; total < msgs.size(); total++) { + msgThis.add(msgs.get(total)); + } + + this.submitConsumeRequestLater(consumeRequest); + } + } + } + } + + public void resetRetryTopic(final List msgs) { + final String groupTopic = MixAll.getRetryTopic(consumerGroup); + for (MessageExt msg : msgs) { + String retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (retryTopic != null && groupTopic.equals(msg.getTopic())) { + msg.setTopic(retryTopic); + } + } + } + + private void cleanExpireMsg() { + Iterator> it = + this.defaultMQPushConsumerImpl.getRebalanceImpl().getProcessQueueTable().entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + ProcessQueue pq = next.getValue(); + pq.cleanExpiredMsg(this.defaultMQPushConsumer); + } + } + + public void processConsumeResult(// + final ConsumeConcurrentlyStatus status, // + final ConsumeConcurrentlyContext context, // + final ConsumeRequest consumeRequest// + ) { + int ackIndex = context.getAckIndex(); + + if (consumeRequest.getMsgs().isEmpty()) + return; + + switch (status) { + case CONSUME_SUCCESS: + if (ackIndex >= consumeRequest.getMsgs().size()) { + ackIndex = consumeRequest.getMsgs().size() - 1; + } + int ok = ackIndex + 1; + int failed = consumeRequest.getMsgs().size() - ok; + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok); + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed); + break; + case RECONSUME_LATER: + ackIndex = -1; + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), + consumeRequest.getMsgs().size()); + break; + default: + break; + } + + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { + MessageExt msg = consumeRequest.getMsgs().get(i); + log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString()); + } + break; + case CLUSTERING: + List msgBackFailed = new ArrayList(consumeRequest.getMsgs().size()); + for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { + MessageExt msg = consumeRequest.getMsgs().get(i); + boolean result = this.sendMessageBack(msg, context); + if (!result) { + msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); + msgBackFailed.add(msg); + } + } + + if (!msgBackFailed.isEmpty()) { + consumeRequest.getMsgs().removeAll(msgBackFailed); + + this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue()); + } + break; + default: + break; + } + + long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs()); + if (offset >= 0 && !consumeRequest.getProcessQueue().isDropped()) { + this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true); + } + } + + public ConsumerStatsManager getConsumerStatsManager() { + return this.defaultMQPushConsumerImpl.getConsumerStatsManager(); + } + + public boolean sendMessageBack(final MessageExt msg, final ConsumeConcurrentlyContext context) { + int delayLevel = context.getDelayLevelWhenNextConsume(); + + try { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName()); + return true; + } catch (Exception e) { + log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e); + } + + return false; + } + + private void submitConsumeRequestLater(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue// + ) { + + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + ConsumeMessageConcurrentlyService.this.submitConsumeRequest(msgs, processQueue, messageQueue, true); + } + }, 5000, TimeUnit.MILLISECONDS); + } + + private void submitConsumeRequestLater(final ConsumeRequest consumeRequest// + ) { + + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + ConsumeMessageConcurrentlyService.this.consumeExecutor.submit(consumeRequest); + } + }, 5000, TimeUnit.MILLISECONDS); + } + + class ConsumeRequest implements Runnable { + private final List msgs; + private final ProcessQueue processQueue; + private final MessageQueue messageQueue; + + + public ConsumeRequest(List msgs, ProcessQueue processQueue, MessageQueue messageQueue) { + this.msgs = msgs; + this.processQueue = processQueue; + this.messageQueue = messageQueue; + } + + public List getMsgs() { + return msgs; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + @Override + public void run() { + if (this.processQueue.isDropped()) { + log.info("the message queue not be able to consume, because it's dropped. group={} {}", ConsumeMessageConcurrentlyService.this.consumerGroup, this.messageQueue); + return; + } + + MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener; + ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue); + ConsumeConcurrentlyStatus status = null; + + ConsumeMessageContext consumeMessageContext = null; + if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setConsumerGroup(defaultMQPushConsumer.getConsumerGroup()); + consumeMessageContext.setProps(new HashMap()); + consumeMessageContext.setMq(messageQueue); + consumeMessageContext.setMsgList(msgs); + consumeMessageContext.setSuccess(false); + ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext); + } + + long beginTimestamp = System.currentTimeMillis(); + boolean hasException = false; + ConsumeReturnType returnType = ConsumeReturnType.SUCCESS; + try { + ConsumeMessageConcurrentlyService.this.resetRetryTopic(msgs); + if (msgs != null && !msgs.isEmpty()) { + for (MessageExt msg : msgs) { + MessageAccessor.setConsumeStartTimeStamp(msg, String.valueOf(System.currentTimeMillis())); + } + } + status = listener.consumeMessage(Collections.unmodifiableList(msgs), context); + } catch (Throwable e) { + log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}", + RemotingHelper.exceptionSimpleDesc(e), // + ConsumeMessageConcurrentlyService.this.consumerGroup, + msgs, + messageQueue); + hasException = true; + } + long consumeRT = System.currentTimeMillis() - beginTimestamp; + if (null == status) { + if (hasException) { + returnType = ConsumeReturnType.EXCEPTION; + } else { + returnType = ConsumeReturnType.RETURNNULL; + } + } else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) { + returnType = ConsumeReturnType.TIME_OUT; + } else if (ConsumeConcurrentlyStatus.RECONSUME_LATER == status) { + returnType = ConsumeReturnType.FAILED; + } else if (ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status) { + returnType = ConsumeReturnType.SUCCESS; + } + consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name()); + if (null == status) { + log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}", + ConsumeMessageConcurrentlyService.this.consumerGroup, + msgs, + messageQueue); + status = ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + + if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext.setStatus(status.toString()); + consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status); + ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext); + } + + ConsumeMessageConcurrentlyService.this.getConsumerStatsManager() + .incConsumeRT(ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT); + + if (!processQueue.isDropped()) { + ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this); + } else { + log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}", messageQueue, msgs); + } + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java new file mode 100644 index 0000000000..82903b0eab --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -0,0 +1,536 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.*; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.body.CMResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.*; + + +/** + * @author shijia.wxr + */ +public class ConsumeMessageOrderlyService implements ConsumeMessageService { + private static final Logger log = ClientLogger.getLog(); + private final static long MAX_TIME_CONSUME_CONTINUOUSLY = + Long.parseLong(System.getProperty("rocketmq.client.maxTimeConsumeContinuously", "60000")); + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + private final DefaultMQPushConsumer defaultMQPushConsumer; + private final MessageListenerOrderly messageListener; + private final BlockingQueue consumeRequestQueue; + private final ThreadPoolExecutor consumeExecutor; + private final String consumerGroup; + private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + private final ScheduledExecutorService scheduledExecutorService; + private volatile boolean stopped = false; + + + public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl, MessageListenerOrderly messageListener) { + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + this.messageListener = messageListener; + + this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer(); + this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup(); + this.consumeRequestQueue = new LinkedBlockingQueue(); + + this.consumeExecutor = new ThreadPoolExecutor(// + this.defaultMQPushConsumer.getConsumeThreadMin(), // + this.defaultMQPushConsumer.getConsumeThreadMax(), // + 1000 * 60, // + TimeUnit.MILLISECONDS, // + this.consumeRequestQueue, // + new ThreadFactoryImpl("ConsumeMessageThread_")); + + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ConsumeMessageScheduledThread_")); + } + + + public void start() { + if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + ConsumeMessageOrderlyService.this.lockMQPeriodically(); + } + }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS); + } + } + + + public void shutdown() { + this.stopped = true; + this.scheduledExecutorService.shutdown(); + this.consumeExecutor.shutdown(); + if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { + this.unlockAllMQ(); + } + } + + + public synchronized void unlockAllMQ() { + this.defaultMQPushConsumerImpl.getRebalanceImpl().unlockAll(false); + } + + @Override + public void updateCorePoolSize(int corePoolSize) { + if (corePoolSize > 0 // + && corePoolSize <= Short.MAX_VALUE // + && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { + this.consumeExecutor.setCorePoolSize(corePoolSize); + } + } + + @Override + public void incCorePoolSize() { + } + + @Override + public void decCorePoolSize() { + } + + @Override + public int getCorePoolSize() { + return this.consumeExecutor.getCorePoolSize(); + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) { + ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); + result.setOrder(true); + + List msgs = new ArrayList(); + msgs.add(msg); + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(brokerName); + mq.setTopic(msg.getTopic()); + mq.setQueueId(msg.getQueueId()); + + ConsumeOrderlyContext context = new ConsumeOrderlyContext(mq); + + final long beginTime = System.currentTimeMillis(); + + log.info("consumeMessageDirectly receive new messge: {}", msg); + + try { + ConsumeOrderlyStatus status = this.messageListener.consumeMessage(msgs, context); + if (status != null) { + switch (status) { + case COMMIT: + result.setConsumeResult(CMResult.CR_COMMIT); + break; + case ROLLBACK: + result.setConsumeResult(CMResult.CR_ROLLBACK); + break; + case SUCCESS: + result.setConsumeResult(CMResult.CR_SUCCESS); + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + result.setConsumeResult(CMResult.CR_LATER); + break; + default: + break; + } + } else { + result.setConsumeResult(CMResult.CR_RETURN_NULL); + } + } catch (Throwable e) { + result.setConsumeResult(CMResult.CR_THROW_EXCEPTION); + result.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + + log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s", // + RemotingHelper.exceptionSimpleDesc(e), // + ConsumeMessageOrderlyService.this.consumerGroup, // + msgs, // + mq), e); + } + + result.setAutoCommit(context.isAutoCommit()); + result.setSpentTimeMills(System.currentTimeMillis() - beginTime); + + log.info("consumeMessageDirectly Result: {}", result); + + return result; + } + + @Override + public void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispathToConsume) { + if (dispathToConsume) { + ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue); + this.consumeExecutor.submit(consumeRequest); + } + } + + public synchronized void lockMQPeriodically() { + if (!this.stopped) { + this.defaultMQPushConsumerImpl.getRebalanceImpl().lockAll(); + } + } + + public void tryLockLaterAndReconsume(final MessageQueue mq, final ProcessQueue processQueue, final long delayMills) { + this.scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + boolean lockOK = ConsumeMessageOrderlyService.this.lockOneMQ(mq); + if (lockOK) { + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 10); + } else { + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 3000); + } + } + }, delayMills, TimeUnit.MILLISECONDS); + } + + public synchronized boolean lockOneMQ(final MessageQueue mq) { + if (!this.stopped) { + return this.defaultMQPushConsumerImpl.getRebalanceImpl().lock(mq); + } + + return false; + } + + private void submitConsumeRequestLater(// + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final long suspendTimeMillis// + ) { + long timeMillis = suspendTimeMillis; + if (timeMillis == -1) { + timeMillis = this.defaultMQPushConsumer.getSuspendCurrentQueueTimeMillis(); + } + + if (timeMillis < 10) { + timeMillis = 10; + } else if (timeMillis > 30000) { + timeMillis = 30000; + } + + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + ConsumeMessageOrderlyService.this.submitConsumeRequest(null, processQueue, messageQueue, true); + } + }, timeMillis, TimeUnit.MILLISECONDS); + } + + public boolean processConsumeResult(// + final List msgs, // + final ConsumeOrderlyStatus status, // + final ConsumeOrderlyContext context, // + final ConsumeRequest consumeRequest// + ) { + boolean continueConsume = true; + long commitOffset = -1L; + if (context.isAutoCommit()) { + switch (status) { + case COMMIT: + case ROLLBACK: + log.warn("the message queue consume result is illegal, we think you want to ack these message {}", + consumeRequest.getMessageQueue()); + case SUCCESS: + commitOffset = consumeRequest.getProcessQueue().commit(); + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); + if (checkReconsumeTimes(msgs)) { + consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + } else { + commitOffset = consumeRequest.getProcessQueue().commit(); + } + break; + default: + break; + } + } else { + switch (status) { + case SUCCESS: + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + case COMMIT: + commitOffset = consumeRequest.getProcessQueue().commit(); + break; + case ROLLBACK: + consumeRequest.getProcessQueue().rollback(); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), msgs.size()); + if (checkReconsumeTimes(msgs)) { + consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + } + break; + default: + break; + } + } + + if (commitOffset >= 0 && !consumeRequest.getProcessQueue().isDropped()) { + this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), commitOffset, false); + } + + return continueConsume; + } + + public ConsumerStatsManager getConsumerStatsManager() { + return this.defaultMQPushConsumerImpl.getConsumerStatsManager(); + } + + private int getMaxReconsumeTimes() { + // default reconsume times: Integer.MAX_VALUE + if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) { + return Integer.MAX_VALUE; + } else { + return this.defaultMQPushConsumer.getMaxReconsumeTimes(); + } + } + + private boolean checkReconsumeTimes(List msgs) { + boolean suspend = false; + if (msgs != null && !msgs.isEmpty()) { + for (MessageExt msg : msgs) { + if (msg.getReconsumeTimes() >= getMaxReconsumeTimes()) { + MessageAccessor.setReconsumeTime(msg, String.valueOf(msg.getReconsumeTimes())); + if (!sendMessageBack(msg)) { + suspend = true; + msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); + } + } else { + suspend = true; + msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); + } + } + } + return suspend; + } + + public boolean sendMessageBack(final MessageExt msg) { + try { + // max reconsume times exceeded then send to dead letter queue. + Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes())); + MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); + newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); + + this.defaultMQPushConsumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(newMsg); + return true; + } catch (Exception e) { + log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e); + } + + return false; + } + + class ConsumeRequest implements Runnable { + private final ProcessQueue processQueue; + private final MessageQueue messageQueue; + + + public ConsumeRequest(ProcessQueue processQueue, MessageQueue messageQueue) { + this.processQueue = processQueue; + this.messageQueue = messageQueue; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + @Override + public void run() { + if (this.processQueue.isDropped()) { + log.warn("run, the message queue not be able to consume, because it's dropped. {}", this.messageQueue); + return; + } + + final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue); + synchronized (objLock) { + if (MessageModel.BROADCASTING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel()) + || (this.processQueue.isLocked() && !this.processQueue.isLockExpired())) { + final long beginTime = System.currentTimeMillis(); + for (boolean continueConsume = true; continueConsume; ) { + if (this.processQueue.isDropped()) { + log.warn("the message queue not be able to consume, because it's dropped. {}", this.messageQueue); + break; + } + + if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel()) + && !this.processQueue.isLocked()) { + log.warn("the message queue not locked, so consume later, {}", this.messageQueue); + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 10); + break; + } + + if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel()) + && this.processQueue.isLockExpired()) { + log.warn("the message queue lock expired, so consume later, {}", this.messageQueue); + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 10); + break; + } + + long interval = System.currentTimeMillis() - beginTime; + if (interval > MAX_TIME_CONSUME_CONTINUOUSLY) { + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, messageQueue, 10); + break; + } + + final int consumeBatchSize = + ConsumeMessageOrderlyService.this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); + + List msgs = this.processQueue.takeMessags(consumeBatchSize); + if (!msgs.isEmpty()) { + final ConsumeOrderlyContext context = new ConsumeOrderlyContext(this.messageQueue); + + ConsumeOrderlyStatus status = null; + + ConsumeMessageContext consumeMessageContext = null; + if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext + .setConsumerGroup(ConsumeMessageOrderlyService.this.defaultMQPushConsumer.getConsumerGroup()); + consumeMessageContext.setMq(messageQueue); + consumeMessageContext.setMsgList(msgs); + consumeMessageContext.setSuccess(false); + // init the consume context type + consumeMessageContext.setProps(new HashMap()); + ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.executeHookBefore(consumeMessageContext); + } + + long beginTimestamp = System.currentTimeMillis(); + ConsumeReturnType returnType = ConsumeReturnType.SUCCESS; + boolean hasException = false; + try { + this.processQueue.getLockConsume().lock(); + if (this.processQueue.isDropped()) { + log.warn("consumeMessage, the message queue not be able to consume, because it's dropped. {}", + this.messageQueue); + break; + } + + status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context); + } catch (Throwable e) { + log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}", // + RemotingHelper.exceptionSimpleDesc(e), // + ConsumeMessageOrderlyService.this.consumerGroup, // + msgs, // + messageQueue); + hasException = true; + } finally { + this.processQueue.getLockConsume().unlock(); + } + + if (null == status // + || ConsumeOrderlyStatus.ROLLBACK == status// + || ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) { + log.warn("consumeMessage Orderly return not OK, Group: {} Msgs: {} MQ: {}", // + ConsumeMessageOrderlyService.this.consumerGroup, // + msgs, // + messageQueue); + } + + long consumeRT = System.currentTimeMillis() - beginTimestamp; + if (null == status) { + if (hasException) { + returnType = ConsumeReturnType.EXCEPTION; + } else { + returnType = ConsumeReturnType.RETURNNULL; + } + } else if (consumeRT >= defaultMQPushConsumer.getConsumeTimeout() * 60 * 1000) { + returnType = ConsumeReturnType.TIME_OUT; + } else if (ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) { + returnType = ConsumeReturnType.FAILED; + } else if (ConsumeOrderlyStatus.SUCCESS == status) { + returnType = ConsumeReturnType.SUCCESS; + } + consumeMessageContext.getProps().put(MixAll.CONSUME_CONTEXT_TYPE, returnType.name()); + if (null == status) { + status = ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext.setStatus(status.toString()); + consumeMessageContext + .setSuccess(ConsumeOrderlyStatus.SUCCESS == status || ConsumeOrderlyStatus.COMMIT == status); + ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.executeHookAfter(consumeMessageContext); + } + + ConsumeMessageOrderlyService.this.getConsumerStatsManager() + .incConsumeRT(ConsumeMessageOrderlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT); + + continueConsume = ConsumeMessageOrderlyService.this.processConsumeResult(msgs, status, context, this); + } else { + continueConsume = false; + } + } + } else { + if (this.processQueue.isDropped()) { + log.warn("the message queue not be able to consume, because it's dropped. {}", this.messageQueue); + return; + } + + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, this.processQueue, 100); + } + } + } + + + } + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java new file mode 100644 index 0000000000..1f7f0d92b1 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public interface ConsumeMessageService { + void start(); + + + void shutdown(); + + + void updateCorePoolSize(int corePoolSize); + + + void incCorePoolSize(); + + + void decCorePoolSize(); + + + int getCorePoolSize(); + + + ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg, final String brokerName); + + + void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispathToConsume); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java new file mode 100644 index 0000000000..1785ec9207 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -0,0 +1,706 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.store.LocalFileOffsetStore; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.hook.ConsumeMessageHook; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import org.slf4j.Logger; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class DefaultMQPullConsumerImpl implements MQConsumerInner { + private final Logger log = ClientLogger.getLog(); + private final DefaultMQPullConsumer defaultMQPullConsumer; + private final long consumerStartTimestamp = System.currentTimeMillis(); + private final RPCHook rpcHook; + private final ArrayList consumeMessageHookList = new ArrayList(); + private final ArrayList filterMessageHookList = new ArrayList(); + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private PullAPIWrapper pullAPIWrapper; + private OffsetStore offsetStore; + private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); + + + public DefaultMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { + this.defaultMQPullConsumer = defaultMQPullConsumer; + this.rpcHook = rpcHook; + } + + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.makeSureStateOK(); + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The consumer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + } + } + + public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException { + this.makeSureStateOK(); + return this.offsetStore.readOffset(mq, fromStore ? ReadOffsetType.READ_FROM_STORE : ReadOffsetType.MEMORY_FIRST_THEN_STORE); + } + + public Set fetchMessageQueuesInBalance(String topic) throws MQClientException { + this.makeSureStateOK(); + if (null == topic) { + throw new IllegalArgumentException("topic is null"); + } + + ConcurrentHashMap mqTable = this.rebalanceImpl.getProcessQueueTable(); + Set mqResult = new HashSet(); + for (MessageQueue mq : mqTable.keySet()) { + if (mq.getTopic().equals(topic)) { + mqResult.add(mq); + } + } + + return mqResult; + } + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + } + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return pull(mq, subExpression, offset, maxNums, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis()); + } + + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.pullSyncImpl(mq, subExpression, offset, maxNums, false, timeout); + } + + private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, boolean block, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + + if (null == mq) { + throw new MQClientException("mq is null", null); + + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + this.subscriptionAutomatically(mq.getTopic()); + + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); + + SubscriptionData subscriptionData; + try { + subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), // + mq.getTopic(), subExpression); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + + long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + + PullResult pullResult = this.pullAPIWrapper.pullKernelImpl(// + mq, // 1 + subscriptionData.getSubString(), // 2 + 0L, // 3 + offset, // 4 + maxNums, // 5 + sysFlag, // 6 + 0, // 7 + this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 8 + timeoutMillis, // 9 + CommunicationMode.SYNC, // 10 + null// 11 + ); + this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (!this.consumeMessageHookList.isEmpty()) { + ConsumeMessageContext consumeMessageContext = null; + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setConsumerGroup(this.groupName()); + consumeMessageContext.setMq(mq); + consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); + consumeMessageContext.setSuccess(false); + this.executeHookBefore(consumeMessageContext); + consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + consumeMessageContext.setSuccess(true); + this.executeHookAfter(consumeMessageContext); + } + return pullResult; + } + + public void subscriptionAutomatically(final String topic) { + if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) { + try { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), // + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData); + } catch (Exception e) { + } + } + } + + public void unsubscribe(String topic) { + this.rebalanceImpl.getSubscriptionInner().remove(topic); + } + + @Override + public String groupName() { + return this.defaultMQPullConsumer.getConsumerGroup(); + } + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + } + } + } + } + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + } + } + } + } + + @Override + public MessageModel messageModel() { + return this.defaultMQPullConsumer.getMessageModel(); + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public ConsumeFromWhere consumeFromWhere() { + return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + } + + @Override + public Set subscriptions() { + Set result = new HashSet(); + + Set topics = this.defaultMQPullConsumer.getRegisterTopics(); + if (topics != null) { + synchronized (topics) { + for (String t : topics) { + SubscriptionData ms = null; + try { + ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL); + } catch (Exception e) { + log.error("parse subscription error", e); + } + ms.setSubVersion(0L); + result.add(ms); + } + } + } + + return result; + } + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null) { + this.rebalanceImpl.doRebalance(false); + } + } + + @Override + public void persistConsumerOffset() { + try { + this.makeSureStateOK(); + Set mqs = new HashSet(); + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + if (allocateMq != null) { + mqs.addAll(allocateMq); + } + this.offsetStore.persistAll(mqs); + } catch (Exception e) { + log.error("group: " + this.defaultMQPullConsumer.getConsumerGroup() + " persistConsumerOffset exception", e); + } + } + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info); + } + } + } + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + @Override + public boolean isUnitMode() { + return this.defaultMQPullConsumer.isUnitMode(); + } + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + Properties prop = MixAll.object2Properties(this.defaultMQPullConsumer); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); + info.setProperties(prop); + + info.getSubscriptionSet().addAll(this.subscriptions()); + return info; + } + + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + pull(mq, subExpression, offset, maxNums, pullCallback, this.defaultMQPullConsumer.getConsumerPullTimeoutMillis()); + } + + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, false, timeout); + } + + private void pullAsyncImpl(// + final MessageQueue mq, // + final String subExpression, // + final long offset, // + final int maxNums, // + final PullCallback pullCallback, // + final boolean block, // + final long timeout) throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + if (null == pullCallback) { + throw new MQClientException("pullCallback is null", null); + } + + this.subscriptionAutomatically(mq.getTopic()); + + try { + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); + + final SubscriptionData subscriptionData; + try { + subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), // + mq.getTopic(), subExpression); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + + long timeoutMillis = block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + + this.pullAPIWrapper.pullKernelImpl(// + mq, // 1 + subscriptionData.getSubString(), // 2 + 0L, // 3 + offset, // 4 + maxNums, // 5 + sysFlag, // 6 + 0, // 7 + this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 8 + timeoutMillis, // 9 + CommunicationMode.ASYNC, // 10 + new PullCallback() { + + @Override + public void onSuccess(PullResult pullResult) { + pullCallback + .onSuccess(DefaultMQPullConsumerImpl.this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData)); + } + + @Override + public void onException(Throwable e) { + pullCallback.onException(e); + } + }); + } catch (MQBrokerException e) { + throw new MQClientException("pullAsync unknow exception", e); + } + } + + public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.pullSyncImpl(mq, subExpression, offset, maxNums, true, this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis()); + } + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, PullCallback pullCallback) + throws MQClientException, RemotingException, InterruptedException { + this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, true, + this.getDefaultMQPullConsumer().getConsumerPullTimeoutMillis()); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + sendMessageBack(msg, delayLevel, brokerName, this.defaultMQPullConsumer.getConsumerGroup()); + } + + public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway); + } + + public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) + : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + + if (UtilAll.isBlank(consumerGroup)) { + consumerGroup = this.defaultMQPullConsumer.getConsumerGroup(); + } + + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLevel, 3000, + this.defaultMQPullConsumer.getMaxReconsumeTimes()); + } catch (Exception e) { + log.error("sendMessageBack Exception, " + this.defaultMQPullConsumer.getConsumerGroup(), e); + + Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPullConsumer.getConsumerGroup()), msg.getBody()); + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); + MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(this.defaultMQPullConsumer.getMaxReconsumeTimes())); + newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } + } + + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultMQPullConsumer.getConsumerGroup()); + this.mQClientFactory.shutdown(); + log.info("the consumer [{}] shutdown OK", this.defaultMQPullConsumer.getConsumerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + this.copySubscription(); + + if (this.defaultMQPullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultMQPullConsumer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPullConsumer, this.rpcHook); + + this.rebalanceImpl.setConsumerGroup(this.defaultMQPullConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultMQPullConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPullConsumer.getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + + this.pullAPIWrapper = new PullAPIWrapper(// + mQClientFactory, // + this.defaultMQPullConsumer.getConsumerGroup(), isUnitMode()); + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + + if (this.defaultMQPullConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultMQPullConsumer.getOffsetStore(); + } else { + switch (this.defaultMQPullConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPullConsumer.getConsumerGroup()); + break; + default: + break; + } + } + + this.offsetStore.load(); + + boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPullConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + + throw new MQClientException("The consumer group[" + this.defaultMQPullConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + mQClientFactory.start(); + log.info("the consumer [{}] start OK", this.defaultMQPullConsumer.getConsumerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PullConsumer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + } + + private void checkConfig() throws MQClientException { + // check consumerGroup + Validators.checkGroup(this.defaultMQPullConsumer.getConsumerGroup()); + + // consumerGroup + if (null == this.defaultMQPullConsumer.getConsumerGroup()) { + throw new MQClientException( + "consumerGroup is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumerGroup + if (this.defaultMQPullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException( + "consumerGroup can not equal "// + + MixAll.DEFAULT_CONSUMER_GROUP // + + ", please specify another one."// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // messageModel + if (null == this.defaultMQPullConsumer.getMessageModel()) { + throw new MQClientException( + "messageModel is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // allocateMessageQueueStrategy + if (null == this.defaultMQPullConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException( + "allocateMessageQueueStrategy is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // allocateMessageQueueStrategy + if (this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis()) { + throw new MQClientException( + "Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + } + + private void copySubscription() throws MQClientException { + try { + Set registerTopics = this.defaultMQPullConsumer.getRegisterTopics(); + if (registerTopics != null) { + for (final String topic : registerTopics) { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(), // + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + } + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException { + this.makeSureStateOK(); + this.offsetStore.updateOffset(mq, offset, false); + } + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + public void registerFilterMessageHook(final FilterMessageHook hook) { + this.filterMessageHookList.add(hook); + log.info("register FilterMessageHook Hook, {}", hook.hookName()); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + public PullAPIWrapper getPullAPIWrapper() { + return pullAPIWrapper; + } + + public void setPullAPIWrapper(PullAPIWrapper pullAPIWrapper) { + this.pullAPIWrapper = pullAPIWrapper; + } + + public ServiceState getServiceState() { + return serviceState; + } + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + public long getConsumerStartTimestamp() { + return consumerStartTimestamp; + } + + + public RebalanceImpl getRebalanceImpl() { + return rebalanceImpl; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java new file mode 100644 index 0000000000..82c342f2cf --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -0,0 +1,1071 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.consumer.store.LocalFileOffsetStore; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.hook.ConsumeMessageHook; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.ProcessQueueInfo; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import org.slf4j.Logger; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class DefaultMQPushConsumerImpl implements MQConsumerInner { + /** + * Delay some time when exception occur + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 3000; + /** + * Flow control interval + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50; + /** + * Delay some time when suspend pull service + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000; + private static final long BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15; + private static final long CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND = 1000 * 30; + private final Logger log = ClientLogger.getLog(); + private final DefaultMQPushConsumer defaultMQPushConsumer; + private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this); + private final ArrayList filterMessageHookList = new ArrayList(); + private final long consumerStartTimestamp = System.currentTimeMillis(); + private final ArrayList consumeMessageHookList = new ArrayList(); + private final RPCHook rpcHook; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private PullAPIWrapper pullAPIWrapper; + private volatile boolean pause = false; + private boolean consumeOrderly = false; + private MessageListener messageListenerInner; + private OffsetStore offsetStore; + private ConsumeMessageService consumeMessageService; + private long flowControlTimes1 = 0; + private long flowControlTimes2 = 0; + + + public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { + this.defaultMQPushConsumer = defaultMQPushConsumer; + this.rpcHook = rpcHook; + } + + public void registerFilterMessageHook(final FilterMessageHook hook) { + this.filterMessageHookList.add(hook); + log.info("register FilterMessageHook Hook, {}", hook.hookName()); + } + + public boolean hasHook() { + return !this.consumeMessageHookList.isEmpty(); + } + + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + } + } + } + } + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + } + } + } + } + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + Set result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + if (null == result) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + } + + if (null == result) { + throw new MQClientException("The topic[" + topic + "] not exist", null); + } + + return result; + } + + public DefaultMQPushConsumer getDefaultMQPushConsumer() { + return defaultMQPushConsumer; + } + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + public void pullMessage(final PullRequest pullRequest) { + final ProcessQueue processQueue = pullRequest.getProcessQueue(); + if (processQueue.isDropped()) { + log.info("the pull request[{}] is dropped.", pullRequest.toString()); + return; + } + + pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis()); + + try { + this.makeSureStateOK(); + } catch (MQClientException e) { + log.warn("pullMessage exception, consumer state not ok", e); + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + return; + } + + if (this.isPause()) { + log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup()); + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND); + return; + } + + long size = processQueue.getMsgCount().get(); + if (size > this.defaultMQPushConsumer.getPullThresholdForQueue()) { + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); + if ((flowControlTimes1++ % 1000) == 0) { + log.warn( + "the consumer message buffer is full, so do flow control, minOffset={}, maxOffset={}, size={}, pullRequest={}, flowControlTimes={}", + processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), size, pullRequest, flowControlTimes1); + } + return; + } + + if (!this.consumeOrderly) { + if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) { + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); + if ((flowControlTimes2++ % 1000) == 0) { + log.warn( + "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}", + processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), + pullRequest, flowControlTimes2); + } + return; + } + } else { + if (processQueue.isLocked()) { + if (!pullRequest.isLockedFirst()) { + final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); + boolean brokerBusy = offset < pullRequest.getNextOffset(); + log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", + pullRequest, offset, brokerBusy); + if (brokerBusy) { + log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", + pullRequest, offset); + } + + pullRequest.setLockedFirst(true); + pullRequest.setNextOffset(offset); + } + } else { + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + log.info("pull message later because not locked in broker, {}", pullRequest); + return; + } + } + + final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); + if (null == subscriptionData) { + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + log.warn("find the consumer's subscription failed, {}", pullRequest); + return; + } + + final long beginTimestamp = System.currentTimeMillis(); + + PullCallback pullCallback = new PullCallback() { + @Override + public void onSuccess(PullResult pullResult) { + if (pullResult != null) { + pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, + subscriptionData); + + switch (pullResult.getPullStatus()) { + case FOUND: + long prevRequestOffset = pullRequest.getNextOffset(); + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + long pullRT = System.currentTimeMillis() - beginTimestamp; + DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), + pullRequest.getMessageQueue().getTopic(), pullRT); + + long firstMsgOffset = Long.MAX_VALUE; + if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) { + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + } else { + firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset(); + + DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), + pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size()); + + boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); + DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(// + pullResult.getMsgFoundList(), // + processQueue, // + pullRequest.getMessageQueue(), // + dispathToConsume); + + if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, + DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval()); + } else { + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + } + } + + if (pullResult.getNextBeginOffset() < prevRequestOffset// + || firstMsgOffset < prevRequestOffset) { + log.warn( + "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", // + pullResult.getNextBeginOffset(), // + firstMsgOffset, // + prevRequestOffset); + } + + break; + case NO_NEW_MSG: + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); + + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + break; + case NO_MATCHED_MSG: + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); + + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + break; + case OFFSET_ILLEGAL: + log.warn("the pull request offset illegal, {} {}", // + pullRequest.toString(), pullResult.toString()); + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + pullRequest.getProcessQueue().setDropped(true); + DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() { + + @Override + public void run() { + try { + DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), + pullRequest.getNextOffset(), false); + + DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue()); + + DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue()); + + log.warn("fix the pull request offset, {}", pullRequest); + } catch (Throwable e) { + log.error("executeTaskLater Exception", e); + } + } + }, 10000); + break; + default: + break; + } + } + } + + + @Override + public void onException(Throwable e) { + if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("execute the pull request exception", e); + } + + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + } + }; + + boolean commitOffsetEnable = false; + long commitOffsetValue = 0L; + if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) { + commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY); + if (commitOffsetValue > 0) { + commitOffsetEnable = true; + } + } + + String subExpression = null; + boolean classFilter = false; + SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); + if (sd != null) { + if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { + subExpression = sd.getSubString(); + } + + classFilter = sd.isClassFilterMode(); + } + + int sysFlag = PullSysFlag.buildSysFlag(// + commitOffsetEnable, // commitOffset + true, // suspend + subExpression != null, // subscription + classFilter // class filter + ); + try { + this.pullAPIWrapper.pullKernelImpl(// + pullRequest.getMessageQueue(), // 1 + subExpression, // 2 + subscriptionData.getSubVersion(), // 3 + pullRequest.getNextOffset(), // 4 + this.defaultMQPushConsumer.getPullBatchSize(), // 5 + sysFlag, // 6 + commitOffsetValue, // 7 + BROKER_SUSPEND_MAX_TIME_MILLIS, // 8 + CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, // 9 + CommunicationMode.ASYNC, // 10 + pullCallback// 11 + ); + } catch (Exception e) { + log.error("pullKernelImpl exception", e); + this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + } + } + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The consumer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + } + } + + private void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { + this.mQClientFactory.getPullMessageService().executePullRequestLater(pullRequest, timeDelay); + } + + public boolean isPause() { + return pause; + } + + public void setPause(boolean pause) { + this.pause = pause; + } + + public ConsumerStatsManager getConsumerStatsManager() { + return this.mQClientFactory.getConsumerStatsManager(); + } + + public void executePullRequestImmediately(final PullRequest pullRequest) { + this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest); + } + + private void correctTagsOffset(final PullRequest pullRequest) { + if (0L == pullRequest.getProcessQueue().getMsgCount().get()) { + this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), true); + } + } + + public void executeTaskLater(final Runnable r, final long timeDelay) { + this.mQClientFactory.getPullMessageService().executeTaskLater(r, timeDelay); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws MQClientException, + InterruptedException { + return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); + } + + + public void registerMessageListener(MessageListener messageListener) { + this.messageListenerInner = messageListener; + } + + public void resume() { + this.pause = false; + doRebalance(); + log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); + } + + public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) + : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, + this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); + } catch (Exception e) { + log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); + + Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); + + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); + + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); + MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); + newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); + + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } + } + + private int getMaxReconsumeTimes() { + // default reconsume times: 16 + if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) { + return 16; + } else { + return this.defaultMQPushConsumer.getMaxReconsumeTimes(); + } + } + + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.consumeMessageService.shutdown(); + this.persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup()); + this.mQClientFactory.shutdown(); + log.info("the consumer [{}] shutdown OK", this.defaultMQPushConsumer.getConsumerGroup()); + this.rebalanceImpl.destroy(); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(), + this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode()); + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + this.copySubscription(); + + if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultMQPushConsumer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook); + + this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + + this.pullAPIWrapper = new PullAPIWrapper( + mQClientFactory, + this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + + if (this.defaultMQPushConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultMQPushConsumer.getOffsetStore(); + } else { + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); + break; + default: + break; + } + } + this.offsetStore.load(); + + if (this.getMessageListenerInner() instanceof MessageListenerOrderly) { + this.consumeOrderly = true; + this.consumeMessageService = + new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner()); + } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) { + this.consumeOrderly = false; + this.consumeMessageService = + new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner()); + } + + this.consumeMessageService.start(); + + boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + this.consumeMessageService.shutdown(); + throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + mQClientFactory.start(); + log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PushConsumer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + + this.updateTopicSubscribeInfoWhenSubscriptionChanged(); + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + + this.mQClientFactory.rebalanceImmediately(); + } + + private void checkConfig() throws MQClientException { + Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup()); + + if (null == this.defaultMQPushConsumer.getConsumerGroup()) { + throw new MQClientException( + "consumerGroup is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException( + "consumerGroup can not equal " + + MixAll.DEFAULT_CONSUMER_GROUP + + ", please specify another one." + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + if (null == this.defaultMQPushConsumer.getMessageModel()) { + throw new MQClientException( + "messageModel is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) { + throw new MQClientException( + "consumeFromWhere is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYY_MMDD_HHMMSS); + if (null == dt) { + throw new MQClientException( + "consumeTimestamp is invalid, YYYY_MMDD_HHMMSS" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // allocateMessageQueueStrategy + if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException( + "allocateMessageQueueStrategy is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // subscription + if (null == this.defaultMQPushConsumer.getSubscription()) { + throw new MQClientException( + "subscription is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // messageListener + if (null == this.defaultMQPushConsumer.getMessageListener()) { + throw new MQClientException( + "messageListener is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly; + boolean concurrently = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently; + if (!orderly && !concurrently) { + throw new MQClientException( + "messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // consumeThreadMin + if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1 + || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000 + || this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer.getConsumeThreadMax()) { + throw new MQClientException( + "consumeThreadMin Out of range [1, 1000]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // consumeThreadMax + if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) { + throw new MQClientException( + "consumeThreadMax Out of range [1, 1000]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // consumeConcurrentlyMaxSpan + if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1 + || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) { + throw new MQClientException( + "consumeConcurrentlyMaxSpan Out of range [1, 65535]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // pullThresholdForQueue + if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) { + throw new MQClientException( + "pullThresholdForQueue Out of range [1, 65535]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // pullInterval + if (this.defaultMQPushConsumer.getPullInterval() < 0 || this.defaultMQPushConsumer.getPullInterval() > 65535) { + throw new MQClientException( + "pullInterval Out of range [0, 65535]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // consumeMessageBatchMaxSize + if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1 + || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) { + throw new MQClientException( + "consumeMessageBatchMaxSize Out of range [1, 1024]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // pullBatchSize + if (this.defaultMQPushConsumer.getPullBatchSize() < 1 || this.defaultMQPushConsumer.getPullBatchSize() > 1024) { + throw new MQClientException( + "pullBatchSize Out of range [1, 1024]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + } + + private void copySubscription() throws MQClientException { + try { + Map sub = this.defaultMQPushConsumer.getSubscription(); + if (sub != null) { + for (final Map.Entry entry : sub.entrySet()) { + final String topic = entry.getKey(); + final String subString = entry.getValue(); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), // + topic, subString); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + } + } + + if (null == this.messageListenerInner) { + this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener(); + } + + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + break; + case CLUSTERING: + final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), // + retryTopic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData); + break; + default: + break; + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + public MessageListener getMessageListenerInner() { + return messageListenerInner; + } + + private void updateTopicSubscribeInfoWhenSubscriptionChanged() { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + } + } + + public ConcurrentHashMap getSubscriptionInner() { + return this.rebalanceImpl.getSubscriptionInner(); + } + + public void subscribe(String topic, String subExpression) throws MQClientException { + try { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), // + topic, subExpression); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + if (this.mQClientFactory != null) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { + try { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), // + topic, "*"); + subscriptionData.setSubString(fullClassName); + subscriptionData.setClassFilterMode(true); + subscriptionData.setFilterClassSource(filterClassSource); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + if (this.mQClientFactory != null) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + public void suspend() { + this.pause = true; + log.info("suspend this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); + } + + public void unsubscribe(String topic) { + this.rebalanceImpl.getSubscriptionInner().remove(topic); + } + + public void updateConsumeOffset(MessageQueue mq, long offset) { + this.offsetStore.updateOffset(mq, offset, false); + } + + public void updateCorePoolSize(int corePoolSize) { + this.consumeMessageService.updateCorePoolSize(corePoolSize); + } + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + public RebalanceImpl getRebalanceImpl() { + return rebalanceImpl; + } + + public boolean isConsumeOrderly() { + return consumeOrderly; + } + + public void setConsumeOrderly(boolean consumeOrderly) { + this.consumeOrderly = consumeOrderly; + } + + public void resetOffsetByTimeStamp(long timeStamp) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + for (String topic : rebalanceImpl.getSubscriptionInner().keySet()) { + Set mqs = rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + Map offsetTable = new HashMap(); + if (mqs != null) { + for (MessageQueue mq : mqs) { + long offset = searchOffset(mq, timeStamp); + offsetTable.put(mq, offset); + } + this.mQClientFactory.resetOffset(topic, groupName(), offsetTable); + } + } + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + @Override + public String groupName() { + return this.defaultMQPushConsumer.getConsumerGroup(); + } + + @Override + public MessageModel messageModel() { + return this.defaultMQPushConsumer.getMessageModel(); + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_PASSIVELY; + } + + @Override + public ConsumeFromWhere consumeFromWhere() { + return this.defaultMQPushConsumer.getConsumeFromWhere(); + } + + @Override + public Set subscriptions() { + Set subSet = new HashSet(); + + subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); + + return subSet; + } + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null && !this.pause) { + this.rebalanceImpl.doRebalance(this.isConsumeOrderly()); + } + } + + @Override + public void persistConsumerOffset() { + try { + this.makeSureStateOK(); + Set mqs = new HashSet(); + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + if (allocateMq != null) { + mqs.addAll(allocateMq); + } + + this.offsetStore.persistAll(mqs); + } catch (Exception e) { + log.error("group: " + this.defaultMQPushConsumer.getConsumerGroup() + " persistConsumerOffset exception", e); + } + } + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.topicSubscribeInfoTable.put(topic, info); + } + } + } + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + @Override + public boolean isUnitMode() { + return this.defaultMQPushConsumer.isUnitMode(); + } + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + Properties prop = MixAll.object2Properties(this.defaultMQPushConsumer); + + prop.put(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, String.valueOf(this.consumeOrderly)); + prop.put(ConsumerRunningInfo.PROP_THREADPOOL_CORE_SIZE, String.valueOf(this.consumeMessageService.getCorePoolSize())); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); + + info.setProperties(prop); + + Set subSet = this.subscriptions(); + info.getSubscriptionSet().addAll(subSet); + + Iterator> it = this.rebalanceImpl.getProcessQueueTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueue pq = next.getValue(); + + ProcessQueueInfo pqinfo = new ProcessQueueInfo(); + pqinfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE)); + pq.fillProcessQueueInfo(pqinfo); + info.getMqTable().put(mq, pqinfo); + } + + for (SubscriptionData sd : subSet) { + ConsumeStatus consumeStatus = this.mQClientFactory.getConsumerStatsManager().consumeStatus(this.groupName(), sd.getTopic()); + info.getStatusTable().put(sd.getTopic(), consumeStatus); + } + + return info; + } + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + public void setmQClientFactory(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + public ServiceState getServiceState() { + return serviceState; + } + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + public void adjustThreadPool() { + long computeAccTotal = this.computeAccumulationTotal(); + long adjustThreadPoolNumsThreshold = this.defaultMQPushConsumer.getAdjustThreadPoolNumsThreshold(); + + long incThreshold = (long) (adjustThreadPoolNumsThreshold * 1.0); + + long decThreshold = (long) (adjustThreadPoolNumsThreshold * 0.8); + + if (computeAccTotal >= incThreshold) { + this.consumeMessageService.incCorePoolSize(); + } + + if (computeAccTotal < decThreshold) { + this.consumeMessageService.decCorePoolSize(); + } + } + + private long computeAccumulationTotal() { + long msgAccTotal = 0; + ConcurrentHashMap processQueueTable = this.rebalanceImpl.getProcessQueueTable(); + Iterator> it = processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ProcessQueue value = next.getValue(); + msgAccTotal += value.getMsgAccCnt(); + } + + return msgAccTotal; + } + + public List queryConsumeTimeSpan(final String topic) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + List queueTimeSpan = new ArrayList(); + TopicRouteData routeData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000); + for (BrokerData brokerData : routeData.getBrokerDatas()) { + String addr = brokerData.selectBrokerAddr(); + queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, groupName(), 3000)); + } + + return queueTimeSpan; + } + + + public ConsumeMessageService getConsumeMessageService() { + return consumeMessageService; + } + + + public void setConsumeMessageService(ConsumeMessageService consumeMessageService) { + this.consumeMessageService = consumeMessageService; + + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java new file mode 100644 index 0000000000..1ff430bdca --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + +import java.util.Set; + + +/** + * Consumer inner interface + * + * @author shijia.wxr + */ +public interface MQConsumerInner { + String groupName(); + + + MessageModel messageModel(); + + + ConsumeType consumeType(); + + + ConsumeFromWhere consumeFromWhere(); + + + Set subscriptions(); + + + void doRebalance(); + + + void persistConsumerOffset(); + + + void updateTopicSubscribeInfo(final String topic, final Set info); + + + boolean isSubscribeTopicNeedUpdate(final String topic); + + + boolean isUnitMode(); + + + ConsumerRunningInfo consumerRunningInfo(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java new file mode 100644 index 0000000000..9de7ac0aaa --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.concurrent.ConcurrentHashMap; + + +/** + * Message lock,strictly ensure the single queue only one thread at a time consuming + * + * @author shijia.wxr + */ +public class MessageQueueLock { + private ConcurrentHashMap mqLockTable = + new ConcurrentHashMap(); + + + public Object fetchLockObject(final MessageQueue mq) { + Object objLock = this.mqLockTable.get(mq); + if (null == objLock) { + objLock = new Object(); + Object prevLock = this.mqLockTable.putIfAbsent(mq, objLock); + if (prevLock != null) { + objLock = prevLock; + } + } + + return objLock; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java new file mode 100644 index 0000000000..05ffeb7d49 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java @@ -0,0 +1,451 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.body.ProcessQueueInfo; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * Queue consumption snapshot + * + * @author shijia.wxr + */ +public class ProcessQueue { + public final static long REBALANCE_LOCK_MAX_LIVE_TIME = + Long.parseLong(System.getProperty("rocketmq.client.rebalance.lockMaxLiveTime", "30000")); + public final static long REBALANCE_LOCK_INTERVAL = Long.parseLong(System.getProperty("rocketmq.client.rebalance.lockInterval", "20000")); + private final static long PULL_MAX_IDLE_TIME = Long.parseLong(System.getProperty("rocketmq.client.pull.pullMaxIdleTime", "120000")); + private final Logger log = ClientLogger.getLog(); + private final ReadWriteLock lockTreeMap = new ReentrantReadWriteLock(); + private final TreeMap msgTreeMap = new TreeMap(); + private final AtomicLong msgCount = new AtomicLong(); + private final Lock lockConsume = new ReentrantLock(); + private final TreeMap msgTreeMapTemp = new TreeMap(); + private final AtomicLong tryUnlockTimes = new AtomicLong(0); + private volatile long queueOffsetMax = 0L; + private volatile boolean dropped = false; + private volatile long lastPullTimestamp = System.currentTimeMillis(); + private volatile long lastConsumeTimestamp = System.currentTimeMillis(); + private volatile boolean locked = false; + private volatile long lastLockTimestamp = System.currentTimeMillis(); + private volatile boolean consuming = false; + private volatile long msgAccCnt = 0; + + public boolean isLockExpired() { + boolean result = (System.currentTimeMillis() - this.lastLockTimestamp) > REBALANCE_LOCK_MAX_LIVE_TIME; + return result; + } + + + public boolean isPullExpired() { + boolean result = (System.currentTimeMillis() - this.lastPullTimestamp) > PULL_MAX_IDLE_TIME; + return result; + } + + /** + + * + * @param pushConsumer + */ + public void cleanExpiredMsg(DefaultMQPushConsumer pushConsumer) { + if (pushConsumer.getDefaultMQPushConsumerImpl().isConsumeOrderly()) { + return; + } + + int loop = msgTreeMap.size() < 16 ? msgTreeMap.size() : 16; + for (int i = 0; i < loop; i++) { + MessageExt msg = null; + try { + this.lockTreeMap.readLock().lockInterruptibly(); + try { + if (!msgTreeMap.isEmpty() && System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msgTreeMap.firstEntry().getValue())) > pushConsumer.getConsumeTimeout() * 60 * 1000) { + msg = msgTreeMap.firstEntry().getValue(); + } else { + + break; + } + } finally { + this.lockTreeMap.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getExpiredMsg exception", e); + } + + try { + + pushConsumer.sendMessageBack(msg, 3); + log.info("send expire msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + if (!msgTreeMap.isEmpty() && msg.getQueueOffset() == msgTreeMap.firstKey()) { + try { + msgTreeMap.remove(msgTreeMap.firstKey()); + } catch (Exception e) { + log.error("send expired msg exception", e); + } + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getExpiredMsg exception", e); + } + } catch (Exception e) { + log.error("send expired msg exception", e); + } + } + } + + + public boolean putMessage(final List msgs) { + boolean dispatchToConsume = false; + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + int validMsgCnt = 0; + for (MessageExt msg : msgs) { + MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg); + if (null == old) { + validMsgCnt++; + this.queueOffsetMax = msg.getQueueOffset(); + } + } + msgCount.addAndGet(validMsgCnt); + + if (!msgTreeMap.isEmpty() && !this.consuming) { + dispatchToConsume = true; + this.consuming = true; + } + + if (!msgs.isEmpty()) { + MessageExt messageExt = msgs.get(msgs.size() - 1); + String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET); + if (property != null) { + long accTotal = Long.parseLong(property) - messageExt.getQueueOffset(); + if (accTotal > 0) { + this.msgAccCnt = accTotal; + } + } + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("putMessage exception", e); + } + + return dispatchToConsume; + } + + + public long getMaxSpan() { + try { + this.lockTreeMap.readLock().lockInterruptibly(); + try { + if (!this.msgTreeMap.isEmpty()) { + return this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey(); + } + } finally { + this.lockTreeMap.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getMaxSpan exception", e); + } + + return 0; + } + + + public long removeMessage(final List msgs) { + long result = -1; + final long now = System.currentTimeMillis(); + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + this.lastConsumeTimestamp = now; + try { + if (!msgTreeMap.isEmpty()) { + result = this.queueOffsetMax + 1; + int removedCnt = 0; + for (MessageExt msg : msgs) { + MessageExt prev = msgTreeMap.remove(msg.getQueueOffset()); + if (prev != null) { + removedCnt--; + } + } + msgCount.addAndGet(removedCnt); + + if (!msgTreeMap.isEmpty()) { + result = msgTreeMap.firstKey(); + } + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (Throwable t) { + log.error("removeMessage exception", t); + } + + return result; + } + + + public TreeMap getMsgTreeMap() { + return msgTreeMap; + } + + + public AtomicLong getMsgCount() { + return msgCount; + } + + + public boolean isDropped() { + return dropped; + } + + + public void setDropped(boolean dropped) { + this.dropped = dropped; + } + + public boolean isLocked() { + return locked; + } + + public void setLocked(boolean locked) { + this.locked = locked; + } + + public void rollback() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + this.msgTreeMap.putAll(this.msgTreeMapTemp); + this.msgTreeMapTemp.clear(); + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("rollback exception", e); + } + } + + + public long commit() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + Long offset = this.msgTreeMapTemp.lastKey(); + msgCount.addAndGet(this.msgTreeMapTemp.size() * (-1)); + this.msgTreeMapTemp.clear(); + if (offset != null) { + return offset + 1; + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("commit exception", e); + } + + return -1; + } + + + public void makeMessageToCosumeAgain(List msgs) { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + for (MessageExt msg : msgs) { + this.msgTreeMapTemp.remove(msg.getQueueOffset()); + this.msgTreeMap.put(msg.getQueueOffset(), msg); + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("makeMessageToCosumeAgain exception", e); + } + } + + + public List takeMessags(final int batchSize) { + List result = new ArrayList(batchSize); + final long now = System.currentTimeMillis(); + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + this.lastConsumeTimestamp = now; + try { + if (!this.msgTreeMap.isEmpty()) { + for (int i = 0; i < batchSize; i++) { + Map.Entry entry = this.msgTreeMap.pollFirstEntry(); + if (entry != null) { + result.add(entry.getValue()); + msgTreeMapTemp.put(entry.getKey(), entry.getValue()); + } else { + break; + } + } + } + + if (result.isEmpty()) { + consuming = false; + } + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("take Messages exception", e); + } + + return result; + } + + + public boolean hasTempMessage() { + try { + this.lockTreeMap.readLock().lockInterruptibly(); + try { + return !this.msgTreeMap.isEmpty(); + } finally { + this.lockTreeMap.readLock().unlock(); + } + } catch (InterruptedException e) { + } + + return true; + } + + + public void clear() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + this.msgTreeMap.clear(); + this.msgTreeMapTemp.clear(); + this.msgCount.set(0); + this.queueOffsetMax = 0L; + } finally { + this.lockTreeMap.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("rollback exception", e); + } + } + + + public long getLastLockTimestamp() { + return lastLockTimestamp; + } + + + public void setLastLockTimestamp(long lastLockTimestamp) { + this.lastLockTimestamp = lastLockTimestamp; + } + + + public Lock getLockConsume() { + return lockConsume; + } + + + public long getLastPullTimestamp() { + return lastPullTimestamp; + } + + + public void setLastPullTimestamp(long lastPullTimestamp) { + this.lastPullTimestamp = lastPullTimestamp; + } + + + public long getMsgAccCnt() { + return msgAccCnt; + } + + + public void setMsgAccCnt(long msgAccCnt) { + this.msgAccCnt = msgAccCnt; + } + + + public long getTryUnlockTimes() { + return this.tryUnlockTimes.get(); + } + + + public void incTryUnlockTimes() { + this.tryUnlockTimes.incrementAndGet(); + } + + + public void fillProcessQueueInfo(final ProcessQueueInfo info) { + try { + this.lockTreeMap.readLock().lockInterruptibly(); + + if (!this.msgTreeMap.isEmpty()) { + info.setCachedMsgMinOffset(this.msgTreeMap.firstKey()); + info.setCachedMsgMaxOffset(this.msgTreeMap.lastKey()); + info.setCachedMsgCount(this.msgTreeMap.size()); + } + + if (!this.msgTreeMapTemp.isEmpty()) { + info.setTransactionMsgMinOffset(this.msgTreeMapTemp.firstKey()); + info.setTransactionMsgMaxOffset(this.msgTreeMapTemp.lastKey()); + info.setTransactionMsgCount(this.msgTreeMapTemp.size()); + } + + info.setLocked(this.locked); + info.setTryUnlockTimes(this.tryUnlockTimes.get()); + info.setLastLockTimestamp(this.lastLockTimestamp); + + info.setDroped(this.dropped); + info.setLastPullTimestamp(this.lastPullTimestamp); + info.setLastConsumeTimestamp(this.lastConsumeTimestamp); + } catch (Exception e) { + } finally { + this.lockTreeMap.readLock().unlock(); + } + } + + + public long getLastConsumeTimestamp() { + return lastConsumeTimestamp; + } + + + public void setLastConsumeTimestamp(long lastConsumeTimestamp) { + this.lastConsumeTimestamp = lastConsumeTimestamp; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java new file mode 100644 index 0000000000..730b0905f4 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.FilterMessageContext; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import org.slf4j.Logger; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public class PullAPIWrapper { + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private final String consumerGroup; + private final boolean unitMode; + private ConcurrentHashMap pullFromWhichNodeTable = + new ConcurrentHashMap(32); + private volatile boolean connectBrokerByUser = false; + private volatile long defaultBrokerId = MixAll.MASTER_ID; + private Random random = new Random(System.currentTimeMillis()); + private ArrayList filterMessageHookList = new ArrayList(); + + public PullAPIWrapper(MQClientInstance mQClientFactory, String consumerGroup, boolean unitMode) { + this.mQClientFactory = mQClientFactory; + this.consumerGroup = consumerGroup; + this.unitMode = unitMode; + } + + public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult, + final SubscriptionData subscriptionData) { + PullResultExt pullResultExt = (PullResultExt) pullResult; + + this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId()); + if (PullStatus.FOUND == pullResult.getPullStatus()) { + ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); + List msgList = MessageDecoder.decodes(byteBuffer); + + List msgListFilterAgain = msgList; + if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) { + msgListFilterAgain = new ArrayList(msgList.size()); + for (MessageExt msg : msgList) { + if (msg.getTags() != null) { + if (subscriptionData.getTagsSet().contains(msg.getTags())) { + msgListFilterAgain.add(msg); + } + } + } + } + + if (this.hasHook()) { + FilterMessageContext filterMessageContext = new FilterMessageContext(); + filterMessageContext.setUnitMode(unitMode); + filterMessageContext.setMsgList(msgListFilterAgain); + this.executeHook(filterMessageContext); + } + + for (MessageExt msg : msgListFilterAgain) { + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, + Long.toString(pullResult.getMinOffset())); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, + Long.toString(pullResult.getMaxOffset())); + } + + pullResultExt.setMsgFoundList(msgListFilterAgain); + } + + pullResultExt.setMessageBinary(null); + + return pullResult; + } + + public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) { + AtomicLong suggest = this.pullFromWhichNodeTable.get(mq); + if (null == suggest) { + this.pullFromWhichNodeTable.put(mq, new AtomicLong(brokerId)); + } else { + suggest.set(brokerId); + } + } + + public boolean hasHook() { + return !this.filterMessageHookList.isEmpty(); + } + + public void executeHook(final FilterMessageContext context) { + if (!this.filterMessageHookList.isEmpty()) { + for (FilterMessageHook hook : this.filterMessageHookList) { + try { + hook.filterMessage(context); + } catch (Throwable e) { + log.error("execute hook error. hookName={}", hook.hookName()); + } + } + } + } + + public PullResult pullKernelImpl( + final MessageQueue mq, + final String subExpression, + final long subVersion, + final long offset, + final int maxNums, + final int sysFlag, + final long commitOffset, + final long brokerSuspendMaxTimeMillis, + final long timeoutMillis, + final CommunicationMode communicationMode, + final PullCallback pullCallback + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.recalculatePullFromWhichNode(mq), false); + if (null == findBrokerResult) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.recalculatePullFromWhichNode(mq), false); + } + + if (findBrokerResult != null) { + int sysFlagInner = sysFlag; + + if (findBrokerResult.isSlave()) { + sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner); + } + + PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); + requestHeader.setConsumerGroup(this.consumerGroup); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setQueueOffset(offset); + requestHeader.setMaxMsgNums(maxNums); + requestHeader.setSysFlag(sysFlagInner); + requestHeader.setCommitOffset(commitOffset); + requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis); + requestHeader.setSubscription(subExpression); + requestHeader.setSubVersion(subVersion); + + String brokerAddr = findBrokerResult.getBrokerAddr(); + if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { + brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr); + } + + PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage( + brokerAddr, + requestHeader, + timeoutMillis, + communicationMode, + pullCallback); + + return pullResult; + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + public long recalculatePullFromWhichNode(final MessageQueue mq) { + if (this.isConnectBrokerByUser()) { + return this.defaultBrokerId; + } + + AtomicLong suggest = this.pullFromWhichNodeTable.get(mq); + if (suggest != null) { + return suggest.get(); + } + + return MixAll.MASTER_ID; + } + + private String computPullFromWhichFilterServer(final String topic, final String brokerAddr) + throws MQClientException { + ConcurrentHashMap topicRouteTable = this.mQClientFactory.getTopicRouteTable(); + if (topicRouteTable != null) { + TopicRouteData topicRouteData = topicRouteTable.get(topic); + List list = topicRouteData.getFilterServerTable().get(brokerAddr); + + if (list != null && !list.isEmpty()) { + return list.get(randomNum() % list.size()); + } + } + + throw new MQClientException("Find Filter Server Failed, Broker Addr: " + brokerAddr + " topic: " + + topic, null); + } + + public boolean isConnectBrokerByUser() { + return connectBrokerByUser; + } + + public int randomNum() { + int value = random.nextInt(); + if (value < 0) { + value = Math.abs(value); + if (value < 0) + value = 0; + } + return value; + } + + public void setConnectBrokerByUser(boolean connectBrokerByUser) { + this.connectBrokerByUser = connectBrokerByUser; + + } + + public void registerFilterMessageHook(ArrayList filterMessageHookList) { + this.filterMessageHookList = filterMessageHookList; + } + + public long getDefaultBrokerId() { + return defaultBrokerId; + } + + public void setDefaultBrokerId(long defaultBrokerId) { + this.defaultBrokerId = defaultBrokerId; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java new file mode 100644 index 0000000000..161a03951e --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ServiceThread; +import org.slf4j.Logger; + +import java.util.concurrent.*; + + +/** + * @author shijia.wxr + */ +public class PullMessageService extends ServiceThread { + private final Logger log = ClientLogger.getLog(); + private final LinkedBlockingQueue pullRequestQueue = new LinkedBlockingQueue(); + private final MQClientInstance mQClientFactory; + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "PullMessageServiceScheduledThread"); + } + }); + + public PullMessageService(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + public void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + PullMessageService.this.executePullRequestImmediately(pullRequest); + } + }, timeDelay, TimeUnit.MILLISECONDS); + } + + public void executePullRequestImmediately(final PullRequest pullRequest) { + try { + this.pullRequestQueue.put(pullRequest); + } catch (InterruptedException e) { + log.error("executePullRequestImmediately pullRequestQueue.put", e); + } + } + + public void executeTaskLater(final Runnable r, final long timeDelay) { + this.scheduledExecutorService.schedule(r, timeDelay, TimeUnit.MILLISECONDS); + } + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + private void pullMessage(final PullRequest pullRequest) { + final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(pullRequest.getConsumerGroup()); + if (consumer != null) { + DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer; + impl.pullMessage(pullRequest); + } else { + log.warn("No matched consumer for the PullRequest {}, drop it", pullRequest); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + PullRequest pullRequest = this.pullRequestQueue.take(); + if (pullRequest != null) { + this.pullMessage(pullRequest); + } + } catch (InterruptedException e) { + } catch (Exception e) { + log.error("Pull Message Service Run Method exception", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return PullMessageService.class.getSimpleName(); + } + + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java new file mode 100644 index 0000000000..efc5ab0311 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * @author shijia.wxr + */ +public class PullRequest { + private String consumerGroup; + private MessageQueue messageQueue; + private ProcessQueue processQueue; + private long nextOffset; + private boolean lockedFirst = false; + + public boolean isLockedFirst() { + return lockedFirst; + } + + public void setLockedFirst(boolean lockedFirst) { + this.lockedFirst = lockedFirst; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getNextOffset() { + return nextOffset; + } + + + public void setNextOffset(long nextOffset) { + this.nextOffset = nextOffset; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((consumerGroup == null) ? 0 : consumerGroup.hashCode()); + result = prime * result + ((messageQueue == null) ? 0 : messageQueue.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + PullRequest other = (PullRequest) obj; + if (consumerGroup == null) { + if (other.consumerGroup != null) + return false; + } else if (!consumerGroup.equals(other.consumerGroup)) + return false; + if (messageQueue == null) { + if (other.messageQueue != null) + return false; + } else if (!messageQueue.equals(other.messageQueue)) + return false; + return true; + } + + @Override + public String toString() { + return "PullRequest [consumerGroup=" + consumerGroup + ", messageQueue=" + messageQueue + + ", nextOffset=" + nextOffset + "]"; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + + public void setProcessQueue(ProcessQueue processQueue) { + this.processQueue = processQueue; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java new file mode 100644 index 0000000000..e140b6aa30 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class PullResultExt extends PullResult { + private final long suggestWhichBrokerId; + private byte[] messageBinary; + + + public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, + List msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary) { + super(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList); + this.suggestWhichBrokerId = suggestWhichBrokerId; + this.messageBinary = messageBinary; + } + + + public byte[] getMessageBinary() { + return messageBinary; + } + + + public void setMessageBinary(byte[] messageBinary) { + this.messageBinary = messageBinary; + } + + + public long getSuggestWhichBrokerId() { + return suggestWhichBrokerId; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java new file mode 100644 index 0000000000..641bb75a61 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -0,0 +1,481 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.LockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.slf4j.Logger; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * Base class for rebalance algorithm + * + * @author shijia.wxr + */ +public abstract class RebalanceImpl { + protected static final Logger log = ClientLogger.getLog(); + protected final ConcurrentHashMap processQueueTable = new ConcurrentHashMap(64); + protected final ConcurrentHashMap> topicSubscribeInfoTable = + new ConcurrentHashMap>(); + protected final ConcurrentHashMap subscriptionInner = + new ConcurrentHashMap(); + protected String consumerGroup; + protected MessageModel messageModel; + protected AllocateMessageQueueStrategy allocateMessageQueueStrategy; + protected MQClientInstance mQClientFactory; + + + public RebalanceImpl(String consumerGroup, MessageModel messageModel, AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory) { + this.consumerGroup = consumerGroup; + this.messageModel = messageModel; + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + this.mQClientFactory = mQClientFactory; + } + + public void unlock(final MessageQueue mq, final boolean oneway) { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.getMqSet().add(mq); + + try { + this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000, oneway); + log.warn("unlock messageQueue. group:{}, clientId:{}, mq:{}", // + this.consumerGroup, // + this.mQClientFactory.getClientId(), // + mq); + } catch (Exception e) { + log.error("unlockBatchMQ exception, " + mq, e); + } + } + } + + public void unlockAll(final boolean oneway) { + HashMap> brokerMqs = this.buildProcessQueueTableByBrokerName(); + + for (final Map.Entry> entry : brokerMqs.entrySet()) { + final String brokerName = entry.getKey(); + final Set mqs = entry.getValue(); + + if (mqs.isEmpty()) + continue; + + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.setMqSet(mqs); + + try { + this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000, oneway); + + for (MessageQueue mq : mqs) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + processQueue.setLocked(false); + log.info("the message queue unlock OK, Group: {} {}", this.consumerGroup, mq); + } + } + } catch (Exception e) { + log.error("unlockBatchMQ exception, " + mqs, e); + } + } + } + } + + private HashMap> buildProcessQueueTableByBrokerName() { + HashMap> result = new HashMap>(); + for (MessageQueue mq : this.processQueueTable.keySet()) { + Set mqs = result.get(mq.getBrokerName()); + if (null == mqs) { + mqs = new HashSet(); + result.put(mq.getBrokerName(), mqs); + } + + mqs.add(mq); + } + + return result; + } + + public boolean lock(final MessageQueue mq) { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + LockBatchRequestBody requestBody = new LockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.getMqSet().add(mq); + + try { + Set lockedMq = + this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000); + for (MessageQueue mmqq : lockedMq) { + ProcessQueue processQueue = this.processQueueTable.get(mmqq); + if (processQueue != null) { + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + } + } + + boolean lockOK = lockedMq.contains(mq); + log.info("the message queue lock {}, {} {}", + lockOK ? "OK" : "Failed", + this.consumerGroup, + mq); + return lockOK; + } catch (Exception e) { + log.error("lockBatchMQ exception, " + mq, e); + } + } + + return false; + } + + public void lockAll() { + HashMap> brokerMqs = this.buildProcessQueueTableByBrokerName(); + + Iterator>> it = brokerMqs.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + final String brokerName = entry.getKey(); + final Set mqs = entry.getValue(); + + if (mqs.isEmpty()) + continue; + + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + LockBatchRequestBody requestBody = new LockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.setMqSet(mqs); + + try { + Set lockOKMQSet = + this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ(findBrokerResult.getBrokerAddr(), requestBody, 1000); + + for (MessageQueue mq : lockOKMQSet) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + if (!processQueue.isLocked()) { + log.info("the message queue locked OK, Group: {} {}", this.consumerGroup, mq); + } + + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + } + } + for (MessageQueue mq : mqs) { + if (!lockOKMQSet.contains(mq)) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + processQueue.setLocked(false); + log.warn("the message queue locked Failed, Group: {} {}", this.consumerGroup, mq); + } + } + } + } catch (Exception e) { + log.error("lockBatchMQ exception, " + mqs, e); + } + } + } + } + + public void doRebalance(final boolean isOrder) { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + try { + this.rebalanceByTopic(topic, isOrder); + } catch (Throwable e) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("rebalanceByTopic Exception", e); + } + } + } + } + + this.truncateMessageQueueNotMyTopic(); + } + + public ConcurrentHashMap getSubscriptionInner() { + return subscriptionInner; + } + + private void rebalanceByTopic(final String topic, final boolean isOrder) { + switch (messageModel) { + case BROADCASTING: { + Set mqSet = this.topicSubscribeInfoTable.get(topic); + if (mqSet != null) { + boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder); + if (changed) { + this.messageQueueChanged(topic, mqSet, mqSet); + log.info("messageQueueChanged {} {} {} {}", // + consumerGroup, // + topic, // + mqSet, // + mqSet); + } + } else { + log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic); + } + break; + } + case CLUSTERING: { + Set mqSet = this.topicSubscribeInfoTable.get(topic); + List cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup); + if (null == mqSet) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic); + } + } + + if (null == cidAll) { + log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic); + } + + if (mqSet != null && cidAll != null) { + List mqAll = new ArrayList(); + mqAll.addAll(mqSet); + + Collections.sort(mqAll); + Collections.sort(cidAll); + + AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy; + + List allocateResult = null; + try { + allocateResult = strategy.allocate(// + this.consumerGroup, // + this.mQClientFactory.getClientId(), // + mqAll, // + cidAll); + } catch (Throwable e) { + log.error("AllocateMessageQueueStrategy.allocate Exception. allocateMessageQueueStrategyName={}", strategy.getName(), + e); + return; + } + + Set allocateResultSet = new HashSet(); + if (allocateResult != null) { + allocateResultSet.addAll(allocateResult); + } + + boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder); + if (changed) { + log.info( + "rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, rebalanceResultSize={}, rebalanceResultSet={}", + strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(), + allocateResultSet.size(), allocateResultSet); + this.messageQueueChanged(topic, mqSet, allocateResultSet); + } + } + break; + } + default: + break; + } + } + + private void truncateMessageQueueNotMyTopic() { + Map subTable = this.getSubscriptionInner(); + + for (MessageQueue mq : this.processQueueTable.keySet()) { + if (!subTable.containsKey(mq.getTopic())) { + + ProcessQueue pq = this.processQueueTable.remove(mq); + if (pq != null) { + pq.setDropped(true); + log.info("doRebalance, {}, truncateMessageQueueNotMyTopic remove unnecessary mq, {}", consumerGroup, mq); + } + } + } + } + + private boolean updateProcessQueueTableInRebalance(final String topic, final Set mqSet, final boolean isOrder) { + boolean changed = false; + + Iterator> it = this.processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueue pq = next.getValue(); + + if (mq.getTopic().equals(topic)) { + if (!mqSet.contains(mq)) { + pq.setDropped(true); + if (this.removeUnnecessaryMessageQueue(mq, pq)) { + it.remove(); + changed = true; + log.info("doRebalance, {}, remove unnecessary mq, {}", consumerGroup, mq); + } + } else if (pq.isPullExpired()) { + switch (this.consumeType()) { + case CONSUME_ACTIVELY: + break; + case CONSUME_PASSIVELY: + pq.setDropped(true); + if (this.removeUnnecessaryMessageQueue(mq, pq)) { + it.remove(); + changed = true; + log.error("[BUG]doRebalance, {}, remove unnecessary mq, {}, because pull is pause, so try to fixed it", + consumerGroup, mq); + } + break; + default: + break; + } + } + } + } + + List pullRequestList = new ArrayList(); + for (MessageQueue mq : mqSet) { + if (!this.processQueueTable.containsKey(mq)) { + if (isOrder && !this.lock(mq)) { + log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq); + continue; + } + + this.removeDirtyOffset(mq); + ProcessQueue pq = new ProcessQueue(); + long nextOffset = this.computePullFromWhere(mq); + if (nextOffset >= 0) { + ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq); + if (pre != null) { + log.info("doRebalance, {}, mq already exists, {}", consumerGroup, mq); + } else { + log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq); + PullRequest pullRequest = new PullRequest(); + pullRequest.setConsumerGroup(consumerGroup); + pullRequest.setNextOffset(nextOffset); + pullRequest.setMessageQueue(mq); + pullRequest.setProcessQueue(pq); + pullRequestList.add(pullRequest); + changed = true; + } + } else { + log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq); + } + } + } + + this.dispatchPullRequest(pullRequestList); + + return changed; + } + + public abstract void messageQueueChanged(final String topic, final Set mqAll, final Set mqDivided); + + public abstract boolean removeUnnecessaryMessageQueue(final MessageQueue mq, final ProcessQueue pq); + + public abstract ConsumeType consumeType(); + + public abstract void removeDirtyOffset(final MessageQueue mq); + + public abstract long computePullFromWhere(final MessageQueue mq); + + public abstract void dispatchPullRequest(final List pullRequestList); + + public void removeProcessQueue(final MessageQueue mq) { + ProcessQueue prev = this.processQueueTable.remove(mq); + if (prev != null) { + boolean droped = prev.isDropped(); + prev.setDropped(true); + this.removeUnnecessaryMessageQueue(mq, prev); + log.info("Fix Offset, {}, remove unnecessary mq, {} Droped: {}", consumerGroup, mq, droped); + } + } + + public ConcurrentHashMap getProcessQueueTable() { + return processQueueTable; + } + + + public ConcurrentHashMap> getTopicSubscribeInfoTable() { + return topicSubscribeInfoTable; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + + public void setmQClientFactory(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + public void destroy() { + Iterator> it = this.processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().setDropped(true); + } + + this.processQueueTable.clear(); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java new file mode 100644 index 0000000000..8d2b4658f7 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.MessageQueueListener; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + +import java.util.List; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class RebalancePullImpl extends RebalanceImpl { + private final DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; + + + public RebalancePullImpl(DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) { + this(null, null, null, null, defaultMQPullConsumerImpl); + } + + + public RebalancePullImpl(String consumerGroup, MessageModel messageModel, AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory, DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.defaultMQPullConsumerImpl = defaultMQPullConsumerImpl; + } + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageQueueListener messageQueueListener = this.defaultMQPullConsumerImpl.getDefaultMQPullConsumer().getMessageQueueListener(); + if (messageQueueListener != null) { + try { + messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided); + } catch (Throwable e) { + log.error("messageQueueChanged exception", e); + } + } + } + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.defaultMQPullConsumerImpl.getOffsetStore().persist(mq); + this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); + return true; + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public void removeDirtyOffset(final MessageQueue mq) { + this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); + } + + @Override + public long computePullFromWhere(MessageQueue mq) { + return 0; + } + + @Override + public void dispatchPullRequest(List pullRequestList) { + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java new file mode 100644 index 0000000000..2377d295ca --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class RebalancePushImpl extends RebalanceImpl { + private final static long UNLOCK_DELAY_TIME_MILLS = Long.parseLong(System.getProperty("rocketmq.client.unlockDelayTimeMills", "20000")); + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + + + public RebalancePushImpl(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) { + this(null, null, null, null, defaultMQPushConsumerImpl); + } + + + public RebalancePushImpl(String consumerGroup, MessageModel messageModel, AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory, DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + } + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + } + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq); + this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); + if (this.defaultMQPushConsumerImpl.isConsumeOrderly() + && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { + try { + if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) { + try { + return this.unlockDelay(mq, pq); + } finally { + pq.getLockConsume().unlock(); + } + } else { + log.warn("[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}", // + mq, // + pq.getTryUnlockTimes()); + + pq.incTryUnlockTimes(); + } + } catch (Exception e) { + log.error("removeUnnecessaryMessageQueue Exception", e); + } + + return false; + } + return true; + } + + private boolean unlockDelay(final MessageQueue mq, final ProcessQueue pq) { + + if (pq.hasTempMessage()) { + log.info("[{}]unlockDelay, begin {} ", mq.hashCode(), mq); + this.defaultMQPushConsumerImpl.getmQClientFactory().getScheduledExecutorService().schedule(new Runnable() { + @Override + public void run() { + log.info("[{}]unlockDelay, execute at once {}", mq.hashCode(), mq); + RebalancePushImpl.this.unlock(mq, true); + } + }, UNLOCK_DELAY_TIME_MILLS, TimeUnit.MILLISECONDS); + } else { + this.unlock(mq, true); + } + return true; + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_PASSIVELY; + } + + @Override + public void removeDirtyOffset(final MessageQueue mq) { + this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); + } + + @Override + public long computePullFromWhere(MessageQueue mq) { + long result = -1; + final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere(); + final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore(); + switch (consumeFromWhere) { + case CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST: + case CONSUME_FROM_MIN_OFFSET: + case CONSUME_FROM_MAX_OFFSET: + case CONSUME_FROM_LAST_OFFSET: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } + // First start,no offset + else if (-1 == lastOffset) { + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + result = 0L; + } else { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } catch (MQClientException e) { + result = -1; + } + } + } else { + result = -1; + } + break; + } + case CONSUME_FROM_FIRST_OFFSET: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } else if (-1 == lastOffset) { + result = 0L; + } else { + result = -1; + } + break; + } + case CONSUME_FROM_TIMESTAMP: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + if (lastOffset >= 0) { + result = lastOffset; + } else if (-1 == lastOffset) { + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } catch (MQClientException e) { + result = -1; + } + } else { + try { + long timestamp = UtilAll.parseDate(this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeTimestamp(), + UtilAll.YYYY_MMDD_HHMMSS).getTime(); + result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } catch (MQClientException e) { + result = -1; + } + } + } else { + result = -1; + } + break; + } + + default: + break; + } + + return result; + } + + @Override + public void dispatchPullRequest(List pullRequestList) { + for (PullRequest pullRequest : pullRequestList) { + this.defaultMQPushConsumerImpl.executePullRequestImmediately(pullRequest); + log.info("doRebalance, {}, add a new pull request {}", consumerGroup, pullRequest); + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java new file mode 100644 index 0000000000..47a9da5c85 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ServiceThread; +import org.slf4j.Logger; + + +/** + * Rebalance Service + * + * @author shijia.wxr + */ +public class RebalanceService extends ServiceThread { + private static long waitInterval = + Long.parseLong(System.getProperty( + "rocketmq.client.rebalance.waitInterval", "20000")); + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mqClientFactory; + + public RebalanceService(MQClientInstance mqClientFactory) { + this.mqClientFactory = mqClientFactory; + } + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + this.waitForRunning(waitInterval); + this.mqClientFactory.doRebalance(); + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return RebalanceService.class.getSimpleName(); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java new file mode 100644 index 0000000000..508e24993d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java @@ -0,0 +1,1147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.factory; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.admin.MQAdminExtInner; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.*; +import com.alibaba.rocketmq.client.impl.consumer.*; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.client.impl.producer.MQProducerInner; +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.*; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import org.slf4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.net.DatagramSocket; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class MQClientInstance { + private final static long LOCK_TIMEOUT_MILLIS = 3000; + private final Logger log = ClientLogger.getLog(); + private final ClientConfig clientConfig; + private final int instanceIndex; + private final String clientId; + private final long bootTimestamp = System.currentTimeMillis(); + private final ConcurrentHashMap producerTable = new ConcurrentHashMap(); + private final ConcurrentHashMap consumerTable = new ConcurrentHashMap(); + private final ConcurrentHashMap adminExtTable = new ConcurrentHashMap(); + private final NettyClientConfig nettyClientConfig; + private final MQClientAPIImpl mQClientAPIImpl; + private final MQAdminImpl mQAdminImpl; + private final ConcurrentHashMap topicRouteTable = new ConcurrentHashMap(); + private final Lock lockNamesrv = new ReentrantLock(); + private final Lock lockHeartbeat = new ReentrantLock(); + private final ConcurrentHashMap> brokerAddrTable = + new ConcurrentHashMap>(); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "MQClientFactoryScheduledThread"); + } + }); + private final ClientRemotingProcessor clientRemotingProcessor; + private final PullMessageService pullMessageService; + private final RebalanceService rebalanceService; + private final DefaultMQProducer defaultMQProducer; + private final ConsumerStatsManager consumerStatsManager; + private final AtomicLong storeTimesTotal = new AtomicLong(0); + private ServiceState serviceState = ServiceState.CREATE_JUST; + private DatagramSocket datagramSocket; + private Random random = new Random(); + + + public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId) { + this(clientConfig, instanceIndex, clientId, null); + } + + + public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId, RPCHook rpcHook) { + this.clientConfig = clientConfig; + this.instanceIndex = instanceIndex; + this.nettyClientConfig = new NettyClientConfig(); + this.nettyClientConfig.setClientCallbackExecutorThreads(clientConfig.getClientCallbackExecutorThreads()); + this.clientRemotingProcessor = new ClientRemotingProcessor(this); + this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, this.clientRemotingProcessor, rpcHook, clientConfig); + + if (this.clientConfig.getNamesrvAddr() != null) { + this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); + log.info("user specified name server address: {}", this.clientConfig.getNamesrvAddr()); + } + + this.clientId = clientId; + + this.mQAdminImpl = new MQAdminImpl(this); + + this.pullMessageService = new PullMessageService(this); + + this.rebalanceService = new RebalanceService(this); + + this.defaultMQProducer = new DefaultMQProducer(MixAll.CLIENT_INNER_PRODUCER_GROUP); + this.defaultMQProducer.resetClientConfig(clientConfig); + + this.consumerStatsManager = new ConsumerStatsManager(this.scheduledExecutorService); + + log.info("created a new client Instance, FactoryIndex: {} ClinetID: {} {} {}, serializeType={}", // + this.instanceIndex, // + this.clientId, // + this.clientConfig, // + MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION), RemotingCommand.getSerializeTypeConfigInThisServer()); + } + + public void start() throws MQClientException { + + synchronized (this) { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + // If not specified,looking address from name server + if (null == this.clientConfig.getNamesrvAddr()) { + this.clientConfig.setNamesrvAddr(this.mQClientAPIImpl.fetchNameServerAddr()); + } + // Start request-response channel + this.mQClientAPIImpl.start(); + // Start various schedule tasks + this.startScheduledTask(); + // Start pull service + this.pullMessageService.start(); + // Start rebalance service + this.rebalanceService.start(); + // Start push service + this.defaultMQProducer.getDefaultMQProducerImpl().start(false); + log.info("the client factory [{}] start OK", this.clientId); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + break; + case SHUTDOWN_ALREADY: + break; + case START_FAILED: + throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null); + default: + break; + } + } + } + + + private void startScheduledTask() { + if (null == this.clientConfig.getNamesrvAddr()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.mQClientAPIImpl.fetchNameServerAddr(); + } catch (Exception e) { + log.error("ScheduledTask fetchNameServerAddr exception", e); + } + } + }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + } + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.updateTopicRouteInfoFromNameServer(); + } catch (Exception e) { + log.error("ScheduledTask updateTopicRouteInfoFromNameServer exception", e); + } + } + }, 10, this.clientConfig.getPollNameServerInteval(), TimeUnit.MILLISECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.cleanOfflineBroker(); + MQClientInstance.this.sendHeartbeatToAllBrokerWithLock(); + } catch (Exception e) { + log.error("ScheduledTask sendHeartbeatToAllBroker exception", e); + } + } + }, 1000, this.clientConfig.getHeartbeatBrokerInterval(), TimeUnit.MILLISECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.persistAllConsumerOffset(); + } catch (Exception e) { + log.error("ScheduledTask persistAllConsumerOffset exception", e); + } + } + }, 1000 * 10, this.clientConfig.getPersistConsumerOffsetInterval(), TimeUnit.MILLISECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.adjustThreadPool(); + } catch (Exception e) { + log.error("ScheduledTask adjustThreadPool exception", e); + } + } + }, 1, 1, TimeUnit.MINUTES); + } + + public String getClientId() { + return clientId; + } + + public void updateTopicRouteInfoFromNameServer() { + Set topicList = new HashSet(); + + // Consumer + { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + Set subList = impl.subscriptions(); + if (subList != null) { + for (SubscriptionData subData : subList) { + topicList.add(subData.getTopic()); + } + } + } + } + } + + // Producer + { + Iterator> it = this.producerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + Set lst = impl.getPublishTopicList(); + topicList.addAll(lst); + } + } + } + + for (String topic : topicList) { + this.updateTopicRouteInfoFromNameServer(topic); + } + } + + /** + * Remove offline broker + */ + private void cleanOfflineBroker() { + try { + if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) + try { + ConcurrentHashMap> updatedTable = new ConcurrentHashMap>(); + + Iterator>> itBrokerTable = this.brokerAddrTable.entrySet().iterator(); + while (itBrokerTable.hasNext()) { + Entry> entry = itBrokerTable.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + + HashMap cloneAddrTable = new HashMap(); + cloneAddrTable.putAll(oneTable); + + Iterator> it = cloneAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry ee = it.next(); + String addr = ee.getValue(); + if (!this.isBrokerAddrExistInTopicRouteTable(addr)) { + it.remove(); + log.info("the broker addr[{} {}] is offline, remove it", brokerName, addr); + } + } + + if (cloneAddrTable.isEmpty()) { + itBrokerTable.remove(); + log.info("the broker[{}] name's host is offline, remove it", brokerName); + } else { + updatedTable.put(brokerName, cloneAddrTable); + } + } + + if (!updatedTable.isEmpty()) { + this.brokerAddrTable.putAll(updatedTable); + } + } finally { + this.lockNamesrv.unlock(); + } + } catch (InterruptedException e) { + log.warn("cleanOfflineBroker Exception", e); + } + } + + public void sendHeartbeatToAllBrokerWithLock() { + if (this.lockHeartbeat.tryLock()) { + try { + this.sendHeartbeatToAllBroker(); + this.uploadFilterClassSource(); + } catch (final Exception e) { + log.error("sendHeartbeatToAllBroker exception", e); + } finally { + this.lockHeartbeat.unlock(); + } + } else { + log.warn("lock heartBeat, but failed."); + } + } + + private void persistAllConsumerOffset() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + impl.persistConsumerOffset(); + } + } + + public void adjustThreadPool() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + try { + if (impl instanceof DefaultMQPushConsumerImpl) { + DefaultMQPushConsumerImpl dmq = (DefaultMQPushConsumerImpl) impl; + dmq.adjustThreadPool(); + } + } catch (Exception e) { + } + } + } + } + + public boolean updateTopicRouteInfoFromNameServer(final String topic) { + return updateTopicRouteInfoFromNameServer(topic, false, null); + } + + private boolean isBrokerAddrExistInTopicRouteTable(final String addr) { + Iterator> it = this.topicRouteTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + TopicRouteData topicRouteData = entry.getValue(); + List bds = topicRouteData.getBrokerDatas(); + for (BrokerData bd : bds) { + if (bd.getBrokerAddrs() != null) { + boolean exist = bd.getBrokerAddrs().containsValue(addr); + if (exist) + return true; + } + } + } + + return false; + } + + private void sendHeartbeatToAllBroker() { + final HeartbeatData heartbeatData = this.prepareHeartbeatData(); + final boolean producerEmpty = heartbeatData.getProducerDataSet().isEmpty(); + final boolean consumerEmpty = heartbeatData.getConsumerDataSet().isEmpty(); + if (producerEmpty && consumerEmpty) { + log.warn("sending hearbeat, but no consumer and no producer"); + return; + } + + long times = this.storeTimesTotal.getAndIncrement(); + Iterator>> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + if (oneTable != null) { + for (Map.Entry entry1 : oneTable.entrySet()) { + Long id = entry1.getKey(); + String addr = entry1.getValue(); + if (addr != null) { + if (consumerEmpty) { + if (id != MixAll.MASTER_ID) + continue; + } + + try { + this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000); + if (times % 20 == 0) { + log.info("send heart beat to broker[{} {} {}] success", brokerName, id, addr); + log.info(heartbeatData.toString()); + } + } catch (Exception e) { + if (this.isBrokerInNameServer(addr)) { + log.error("send heart beat to broker exception", e); + } else { + log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName, + id, addr); + } + } + } + } + } + } + } + + private void uploadFilterClassSource() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MQConsumerInner consumer = next.getValue(); + if (ConsumeType.CONSUME_PASSIVELY == consumer.consumeType()) { + Set subscriptions = consumer.subscriptions(); + for (SubscriptionData sub : subscriptions) { + if (sub.isClassFilterMode() && sub.getFilterClassSource() != null) { + final String consumerGroup = consumer.groupName(); + final String className = sub.getSubString(); + final String topic = sub.getTopic(); + final String filterClassSource = sub.getFilterClassSource(); + try { + this.uploadFilterClassToAllFilterServer(consumerGroup, className, topic, filterClassSource); + } catch (Exception e) { + log.error("uploadFilterClassToAllFilterServer Exception", e); + } + } + } + } + } + } + + public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, DefaultMQProducer defaultMQProducer) { + try { + if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + TopicRouteData topicRouteData; + if (isDefault && defaultMQProducer != null) { + topicRouteData = this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer(defaultMQProducer.getCreateTopicKey(), + 1000 * 3); + if (topicRouteData != null) { + for (QueueData data : topicRouteData.getQueueDatas()) { + int queueNums = Math.min(defaultMQProducer.getDefaultTopicQueueNums(), data.getReadQueueNums()); + data.setReadQueueNums(queueNums); + data.setWriteQueueNums(queueNums); + } + } + } else { + topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3); + } + if (topicRouteData != null) { + TopicRouteData old = this.topicRouteTable.get(topic); + boolean changed = topicRouteDataIsChange(old, topicRouteData); + if (!changed) { + changed = this.isNeedUpdateTopicRouteInfo(topic); + } else { + log.info("the topic[{}] route info changed, old[{}] ,new[{}]", topic, old, topicRouteData); + } + + if (changed) { + TopicRouteData cloneTopicRouteData = topicRouteData.cloneTopicRouteData(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs()); + } + + // Update Pub info + { + TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData); + publishInfo.setHaveTopicRouterInfo(true); + Iterator> it = this.producerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + impl.updateTopicPublishInfo(topic, publishInfo); + } + } + } + + // Update sub info + { + Set subscribeInfo = topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + impl.updateTopicSubscribeInfo(topic, subscribeInfo); + } + } + } + log.info("topicRouteTable.put TopicRouteData[{}]", cloneTopicRouteData); + this.topicRouteTable.put(topic, cloneTopicRouteData); + return true; + } + } else { + log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic); + } + } catch (Exception e) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) && !topic.equals(MixAll.DEFAULT_TOPIC)) { + log.warn("updateTopicRouteInfoFromNameServer Exception", e); + } + } finally { + this.lockNamesrv.unlock(); + } + } else { + log.warn("updateTopicRouteInfoFromNameServer tryLock timeout {}ms", LOCK_TIMEOUT_MILLIS); + } + } catch (InterruptedException e) { + log.warn("updateTopicRouteInfoFromNameServer Exception", e); + } + + return false; + } + + private HeartbeatData prepareHeartbeatData() { + HeartbeatData heartbeatData = new HeartbeatData(); + + // clientID + heartbeatData.setClientID(this.clientId); + + // Consumer + for (Map.Entry entry : this.consumerTable.entrySet()) { + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + ConsumerData consumerData = new ConsumerData(); + consumerData.setGroupName(impl.groupName()); + consumerData.setConsumeType(impl.consumeType()); + consumerData.setMessageModel(impl.messageModel()); + consumerData.setConsumeFromWhere(impl.consumeFromWhere()); + consumerData.getSubscriptionDataSet().addAll(impl.subscriptions()); + consumerData.setUnitMode(impl.isUnitMode()); + + heartbeatData.getConsumerDataSet().add(consumerData); + } + } + + + // Producer + for (Map.Entry entry : this.producerTable.entrySet()) { + MQProducerInner impl = entry.getValue(); + if (impl != null) { + ProducerData producerData = new ProducerData(); + producerData.setGroupName(entry.getKey()); + + heartbeatData.getProducerDataSet().add(producerData); + } + } + + return heartbeatData; + } + + private boolean isBrokerInNameServer(final String brokerAddr) { + Iterator> it = this.topicRouteTable.entrySet().iterator(); + while (it.hasNext()) { + Entry itNext = it.next(); + List brokerDatas = itNext.getValue().getBrokerDatas(); + for (BrokerData bd : brokerDatas) { + boolean contain = bd.getBrokerAddrs().containsValue(brokerAddr); + if (contain) + return true; + } + } + + return false; + } + + private void uploadFilterClassToAllFilterServer(final String consumerGroup, final String fullClassName, final String topic, + final String filterClassSource) throws UnsupportedEncodingException { + byte[] classBody = null; + int classCRC = 0; + try { + classBody = filterClassSource.getBytes(MixAll.DEFAULT_CHARSET); + classCRC = UtilAll.crc32(classBody); + } catch (Exception e1) { + log.warn("uploadFilterClassToAllFilterServer Exception, ClassName: {} {}", // + fullClassName, // + RemotingHelper.exceptionSimpleDesc(e1)); + } + + TopicRouteData topicRouteData = this.topicRouteTable.get(topic); + if (topicRouteData != null // + && topicRouteData.getFilterServerTable() != null && !topicRouteData.getFilterServerTable().isEmpty()) { + Iterator>> it = topicRouteData.getFilterServerTable().entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + List value = next.getValue(); + for (final String fsAddr : value) { + try { + this.mQClientAPIImpl.registerMessageFilterClass(fsAddr, consumerGroup, topic, fullClassName, classCRC, classBody, + 5000); + + log.info("register message class filter to {} OK, ConsumerGroup: {} Topic: {} ClassName: {}", fsAddr, consumerGroup, + topic, fullClassName); + + } catch (Exception e) { + log.error("uploadFilterClassToAllFilterServer Exception", e); + } + } + } + } else { + log.warn("register message class filter failed, because no filter server, ConsumerGroup: {} Topic: {} ClassName: {}", + consumerGroup, topic, fullClassName); + } + } + + private boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) { + if (olddata == null || nowdata == null) + return true; + TopicRouteData old = olddata.cloneTopicRouteData(); + TopicRouteData now = nowdata.cloneTopicRouteData(); + Collections.sort(old.getQueueDatas()); + Collections.sort(old.getBrokerDatas()); + Collections.sort(now.getQueueDatas()); + Collections.sort(now.getBrokerDatas()); + return !old.equals(now); + + } + + private boolean isNeedUpdateTopicRouteInfo(final String topic) { + boolean result = false; + { + Iterator> it = this.producerTable.entrySet().iterator(); + while (it.hasNext() && !result) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + result = impl.isPublishTopicNeedUpdate(topic); + } + } + } + + { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext() && !result) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + result = impl.isSubscribeTopicNeedUpdate(topic); + } + } + } + + return result; + } + + public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topic, final TopicRouteData route) { + TopicPublishInfo info = new TopicPublishInfo(); + info.setTopicRouteData(route); + if (route.getOrderTopicConf() != null && route.getOrderTopicConf().length() > 0) { + String[] brokers = route.getOrderTopicConf().split(";"); + for (String broker : brokers) { + String[] item = broker.split(":"); + int nums = Integer.parseInt(item[1]); + for (int i = 0; i < nums; i++) { + MessageQueue mq = new MessageQueue(topic, item[0], i); + info.getMessageQueueList().add(mq); + } + } + + info.setOrderTopic(true); + } else { + List qds = route.getQueueDatas(); + Collections.sort(qds); + for (QueueData qd : qds) { + if (PermName.isWriteable(qd.getPerm())) { + BrokerData brokerData = null; + for (BrokerData bd : route.getBrokerDatas()) { + if (bd.getBrokerName().equals(qd.getBrokerName())) { + brokerData = bd; + break; + } + } + + if (null == brokerData) { + continue; + } + + if (!brokerData.getBrokerAddrs().containsKey(MixAll.MASTER_ID)) { + continue; + } + + for (int i = 0; i < qd.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i); + info.getMessageQueueList().add(mq); + } + } + } + + info.setOrderTopic(false); + } + + return info; + } + + public static Set topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) { + Set mqList = new HashSet(); + List qds = route.getQueueDatas(); + for (QueueData qd : qds) { + if (PermName.isReadable(qd.getPerm())) { + for (int i = 0; i < qd.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i); + mqList.add(mq); + } + } + } + + return mqList; + } + + public void shutdown() { + // Consumer + if (!this.consumerTable.isEmpty()) + return; + + // AdminExt + if (!this.adminExtTable.isEmpty()) + return; + + // Producer + if (this.producerTable.size() > 1) + return; + + synchronized (this) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.defaultMQProducer.getDefaultMQProducerImpl().shutdown(false); + + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + this.pullMessageService.shutdown(true); + this.scheduledExecutorService.shutdown(); + this.mQClientAPIImpl.shutdown(); + this.rebalanceService.shutdown(); + + if (this.datagramSocket != null) { + this.datagramSocket.close(); + this.datagramSocket = null; + } + MQClientManager.getInstance().removeClientFactory(this.clientId); + log.info("the client factory [{}] shutdown OK", this.clientId); + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + } + + public boolean registerConsumer(final String group, final MQConsumerInner consumer) { + if (null == group || null == consumer) { + return false; + } + + MQConsumerInner prev = this.consumerTable.putIfAbsent(group, consumer); + if (prev != null) { + log.warn("the consumer group[" + group + "] exist already."); + return false; + } + + return true; + } + + public void unregisterConsumer(final String group) { + this.consumerTable.remove(group); + this.unregisterClientWithLock(null, group); + } + + private void unregisterClientWithLock(final String producerGroup, final String consumerGroup) { + try { + if (this.lockHeartbeat.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + this.unregisterClient(producerGroup, consumerGroup); + } catch (Exception e) { + log.error("unregisterClient exception", e); + } finally { + this.lockHeartbeat.unlock(); + } + } else { + log.warn("lock heartBeat, but failed."); + } + } catch (InterruptedException e) { + log.warn("unregisterClientWithLock exception", e); + } + } + + private void unregisterClient(final String producerGroup, final String consumerGroup) { + Iterator>> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + + if (oneTable != null) { + for (Map.Entry entry1 : oneTable.entrySet()) { + String addr = entry1.getValue(); + if (addr != null) { + try { + this.mQClientAPIImpl.unregisterClient(addr, this.clientId, producerGroup, consumerGroup, 3000); + log.info("unregister client[Producer: {} Consumer: {}] from broker[{} {} {}] success", producerGroup, + consumerGroup, brokerName, entry1.getKey(), addr); + } catch (RemotingException e) { + log.error("unregister client exception from broker: " + addr, e); + } catch (MQBrokerException e) { + log.error("unregister client exception from broker: " + addr, e); + } catch (InterruptedException e) { + log.error("unregister client exception from broker: " + addr, e); + } + } + } + } + } + } + + public boolean registerProducer(final String group, final DefaultMQProducerImpl producer) { + if (null == group || null == producer) { + return false; + } + + MQProducerInner prev = this.producerTable.putIfAbsent(group, producer); + if (prev != null) { + log.warn("the producer group[{}] exist already.", group); + return false; + } + + return true; + } + + public void unregisterProducer(final String group) { + this.producerTable.remove(group); + this.unregisterClientWithLock(group, null); + } + + public boolean registerAdminExt(final String group, final MQAdminExtInner admin) { + if (null == group || null == admin) { + return false; + } + + MQAdminExtInner prev = this.adminExtTable.putIfAbsent(group, admin); + if (prev != null) { + log.warn("the admin group[{}] exist already.", group); + return false; + } + + return true; + } + + public void unregisterAdminExt(final String group) { + this.adminExtTable.remove(group); + } + + public void rebalanceImmediately() { + this.rebalanceService.wakeup(); + } + + public void doRebalance() { + for (Map.Entry entry : this.consumerTable.entrySet()) { + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + try { + impl.doRebalance(); + } catch (Throwable e) { + log.error("doRebalance exception", e); + } + } + } + } + + public MQProducerInner selectProducer(final String group) { + return this.producerTable.get(group); + } + + public MQConsumerInner selectConsumer(final String group) { + return this.consumerTable.get(group); + } + + public FindBrokerResult findBrokerAddressInAdmin(final String brokerName) { + String brokerAddr = null; + boolean slave = false; + boolean found = false; + + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + FOR_SEG: + for (Map.Entry entry : map.entrySet()) { + Long id = entry.getKey(); + brokerAddr = entry.getValue(); + if (brokerAddr != null) { + found = true; + if (MixAll.MASTER_ID == id) { + slave = false; + break FOR_SEG; + } else { + slave = true; + } + break; + + } + } // end of for + } + + if (found) { + return new FindBrokerResult(brokerAddr, slave); + } + + return null; + } + + public String findBrokerAddressInPublish(final String brokerName) { + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + return map.get(MixAll.MASTER_ID); + } + + return null; + } + + public FindBrokerResult findBrokerAddressInSubscribe(// + final String brokerName, // + final long brokerId, // + final boolean onlyThisBroker// + ) { + String brokerAddr = null; + boolean slave = false; + boolean found = false; + + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + brokerAddr = map.get(brokerId); + slave = brokerId != MixAll.MASTER_ID; + found = brokerAddr != null; + + if (!found && !onlyThisBroker) { + Entry entry = map.entrySet().iterator().next(); + brokerAddr = entry.getValue(); + slave = entry.getKey() != MixAll.MASTER_ID; + found = true; + } + } + + if (found) { + return new FindBrokerResult(brokerAddr, slave); + } + + return null; + } + + public List findConsumerIdList(final String topic, final String group) { + String brokerAddr = this.findBrokerAddrByTopic(topic); + if (null == brokerAddr) { + this.updateTopicRouteInfoFromNameServer(topic); + brokerAddr = this.findBrokerAddrByTopic(topic); + } + + if (null != brokerAddr) { + try { + return this.mQClientAPIImpl.getConsumerIdListByGroup(brokerAddr, group, 3000); + } catch (Exception e) { + log.warn("getConsumerIdListByGroup exception, " + brokerAddr + " " + group, e); + } + } + + return null; + } + + public String findBrokerAddrByTopic(final String topic) { + TopicRouteData topicRouteData = this.topicRouteTable.get(topic); + if (topicRouteData != null) { + List brokers = topicRouteData.getBrokerDatas(); + if (!brokers.isEmpty()) { + int index = random.nextInt(brokers.size()); + BrokerData bd = brokers.get(index % brokers.size()); + return bd.selectBrokerAddr(); + } + } + + return null; + } + + public void resetOffset(String topic, String group, Map offsetTable) { + DefaultMQPushConsumerImpl consumer = null; + try { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null && impl instanceof DefaultMQPushConsumerImpl) { + consumer = (DefaultMQPushConsumerImpl) impl; + } else { + log.info("[reset-offset] consumer dose not exist. group={}", group); + return; + } + consumer.suspend(); + + ConcurrentHashMap processQueueTable = consumer.getRebalanceImpl().getProcessQueueTable(); + for (Map.Entry entry : processQueueTable.entrySet()) { + MessageQueue mq = entry.getKey(); + if (topic.equals(mq.getTopic()) && offsetTable.containsKey(mq)) { + ProcessQueue pq = entry.getValue(); + pq.setDropped(true); + pq.clear(); + } + } + + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + // + } + + Iterator iterator = processQueueTable.keySet().iterator(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + Long offset = offsetTable.get(mq); + if (topic.equals(mq.getTopic()) && offset != null) { + try { + consumer.updateConsumeOffset(mq, offset); + consumer.getRebalanceImpl().removeUnnecessaryMessageQueue(mq, processQueueTable.get(mq)); + iterator.remove(); + } catch (Exception e) { + log.warn("reset offset failed. group={}, {}", group, mq, e); + } + } + } + } finally { + consumer.resume(); + } + } + + public Map getConsumerStatus(String topic, String group) { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null && impl instanceof DefaultMQPushConsumerImpl) { + DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) impl; + return consumer.getOffsetStore().cloneOffsetTable(topic); + } else if (impl != null && impl instanceof DefaultMQPullConsumerImpl) { + DefaultMQPullConsumerImpl consumer = (DefaultMQPullConsumerImpl) impl; + return consumer.getOffsetStore().cloneOffsetTable(topic); + } else { + return Collections.EMPTY_MAP; + } + } + + public TopicRouteData getAnExistTopicRouteData(final String topic) { + return this.topicRouteTable.get(topic); + } + + public MQClientAPIImpl getMQClientAPIImpl() { + return mQClientAPIImpl; + } + + public MQAdminImpl getMQAdminImpl() { + return mQAdminImpl; + } + + public long getBootTimestamp() { + return bootTimestamp; + } + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + public PullMessageService getPullMessageService() { + return pullMessageService; + } + + public DefaultMQProducer getDefaultMQProducer() { + return defaultMQProducer; + } + + public ConcurrentHashMap getTopicRouteTable() { + return topicRouteTable; + } + + public ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg, // + final String consumerGroup, // + final String brokerName) { + MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup); + if (null != mqConsumerInner) { + DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) mqConsumerInner; + + ConsumeMessageDirectlyResult result = consumer.getConsumeMessageService().consumeMessageDirectly(msg, brokerName); + return result; + } + + return null; + } + + + public ConsumerRunningInfo consumerRunningInfo(final String consumerGroup) { + MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup); + + ConsumerRunningInfo consumerRunningInfo = mqConsumerInner.consumerRunningInfo(); + + List nsList = this.mQClientAPIImpl.getRemotingClient().getNameServerAddressList(); + + StringBuffer strBuffer = new StringBuffer(); + if (nsList != null) { + for (String addr : nsList) { + strBuffer.append(addr + ";"); + } + } + + String nsAddr = strBuffer.toString(); + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_NAMESERVER_ADDR, nsAddr); + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CONSUME_TYPE, mqConsumerInner.consumeType().name()); + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CLIENT_VERSION, + MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)); + + return consumerRunningInfo; + } + + + public ConsumerStatsManager getConsumerStatsManager() { + return consumerStatsManager; + } + + public NettyClientConfig getNettyClientConfig() { + return nettyClientConfig; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java new file mode 100644 index 0000000000..b82cde9995 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -0,0 +1,1080 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.common.ClientErrorCode; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.CheckForbiddenContext; +import com.alibaba.rocketmq.client.hook.CheckForbiddenHook; +import com.alibaba.rocketmq.client.hook.SendMessageContext; +import com.alibaba.rocketmq.client.hook.SendMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.latency.MQFaultStrategy; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.*; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.*; + + +/** + * @author shijia.wxr + */ +public class DefaultMQProducerImpl implements MQProducerInner { + private final Logger log = ClientLogger.getLog(); + private final Random random = new Random(); + private final DefaultMQProducer defaultMQProducer; + private final ConcurrentHashMap topicPublishInfoTable = + new ConcurrentHashMap(); + private final ArrayList sendMessageHookList = new ArrayList(); + private final RPCHook rpcHook; + protected BlockingQueue checkRequestQueue; + protected ExecutorService checkExecutor; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private ArrayList checkForbiddenHookList = new ArrayList(); + private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); + + private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); + + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { + this(defaultMQProducer, null); + } + + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { + this.defaultMQProducer = defaultMQProducer; + this.rpcHook = rpcHook; + } + + public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { + this.checkForbiddenHookList.add(checkForbiddenHook); + log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), + checkForbiddenHookList.size()); + } + + public void initTransactionEnv() { + TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; + this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); + this.checkExecutor = new ThreadPoolExecutor(// + producer.getCheckThreadPoolMinSize(), // + producer.getCheckThreadPoolMaxSize(), // + 1000 * 60, // + TimeUnit.MILLISECONDS, // + this.checkRequestQueue); + } + + public void destroyTransactionEnv() { + this.checkExecutor.shutdown(); + this.checkRequestQueue.clear(); + } + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register sendMessage Hook, {}", hook.hookName()); + } + + public void start() throws MQClientException { + this.start(true); + } + + public void start(final boolean startFactory) throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { + this.defaultMQProducer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQProducer, rpcHook); + + boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); + + if (startFactory) { + mQClientFactory.start(); + } + + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), + this.defaultMQProducer.isSendMessageWithVIPChannel()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The producer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + + private void checkConfig() throws MQClientException { + Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); + + if (null == this.defaultMQProducer.getProducerGroup()) { + throw new MQClientException("producerGroup is null", null); + } + + if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { + throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", + null); + } + } + + public void shutdown() { + this.shutdown(true); + } + + public void shutdown(final boolean shutdownFactory) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } + + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + @Override + public Set getPublishTopicList() { + Set topicList = new HashSet(); + for (String key : this.topicPublishInfoTable.keySet()) { + topicList.add(key); + } + + return topicList; + } + + @Override + public boolean isPublishTopicNeedUpdate(String topic) { + TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); + + return null == prev || !prev.ok(); + } + + @Override + public TransactionCheckListener checkListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionCheckListener(); + } + + return null; + } + + @Override + public void checkTransactionState(final String addr, final MessageExt msg, final CheckTransactionStateRequestHeader header) { + Runnable request = new Runnable() { + private final String brokerAddr = addr; + private final MessageExt message = msg; + private final CheckTransactionStateRequestHeader checkRequestHeader = header; + private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); + + + @Override + public void run() { + TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); + if (transactionCheckListener != null) { + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable exception = null; + try { + localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + } catch (Throwable e) { + log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); + exception = e; + } + + this.processTransactionState(// + localTransactionState, // + group, // + exception); + } else { + log.warn("checkTransactionState, pick transactionCheckListener by group[{}] failed", group); + } + } + + + private void processTransactionState(// + final LocalTransactionState localTransactionState, // + final String producerGroup, // + final Throwable exception) { + final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); + thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); + thisHeader.setProducerGroup(producerGroup); + thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); + thisHeader.setFromTransactionCheck(true); + + String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (uniqueKey == null) { + uniqueKey = message.getMsgId(); + } + thisHeader.setMsgId(uniqueKey); + thisHeader.setTransactionId(checkRequestHeader.getTransactionId()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + log.warn("when broker check, client rollback this transaction, {}", thisHeader); + break; + case UNKNOW: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + log.warn("when broker check, client does not know this transaction state, {}", thisHeader); + break; + default: + break; + } + + String remark = null; + if (exception != null) { + remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); + } + + try { + DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, + 3000); + } catch (Exception e) { + log.error("endTransactionOneway exception", e); + } + } + }; + + this.checkExecutor.submit(request); + } + + @Override + public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { + if (info != null && topic != null) { + TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); + if (prev != null) { + log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); + } + } + } + + @Override + public boolean isUnitMode() { + return this.defaultMQProducer.isUnitMode(); + } + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.makeSureStateOK(); + Validators.checkTopic(newTopic); + + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The producer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + } + } + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.makeSureStateOK(); + + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); + } + + /** + * DEFAULT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void send(Message msg, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout); + } catch (MQBrokerException e) { + throw new MQClientException("unknownn exception", e); + } + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { + return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); + } + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); + } + + private SendResult sendDefaultImpl(// + Message msg, // + final CommunicationMode communicationMode, // + final SendCallback sendCallback, // + final long timeout// + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + final long invokeID = random.nextLong(); + long beginTimestampFirst = System.currentTimeMillis(); + long beginTimestampPrev = beginTimestampFirst; + long endTimestamp = beginTimestampFirst; + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + Exception exception = null; + SendResult sendResult = null; + int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; + int times = 0; + String[] brokersSent = new String[timesTotal]; + for (; times < timesTotal; times++) { + String lastBrokerName = null == mq ? null : mq.getBrokerName(); + MessageQueue tmpmq = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); + if (tmpmq != null) { + mq = tmpmq; + brokersSent[times] = mq.getBrokerName(); + try { + beginTimestampPrev = System.currentTimeMillis(); + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + switch (communicationMode) { + case ASYNC: + return null; + case ONEWAY: + return null; + case SYNC: + if (sendResult.getSendStatus() != SendStatus.SEND_OK) { + if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { + continue; + } + } + + return sendResult; + default: + break; + } + } catch (RemotingException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQClientException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQBrokerException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + switch (e.getResponseCode()) { + case ResponseCode.TOPIC_NOT_EXIST: + case ResponseCode.SERVICE_NOT_AVAILABLE: + case ResponseCode.SYSTEM_ERROR: + case ResponseCode.NO_PERMISSION: + case ResponseCode.NO_BUYER_ID: + case ResponseCode.NOT_IN_CURRENT_UNIT: + continue; + default: + if (sendResult != null) { + return sendResult; + } + + throw e; + } + } catch (InterruptedException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + throw e; + } + } else { + break; + } + } + + if (sendResult != null) { + return sendResult; + } + + String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", + times, + System.currentTimeMillis() - beginTimestampFirst, + msg.getTopic(), + Arrays.toString(brokersSent)); + + info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); + + MQClientException mqClientException = new MQClientException(info, exception); + if (exception instanceof MQBrokerException) { + mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); + } else if (exception instanceof RemotingConnectException) { + mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION); + } else if (exception instanceof RemotingTimeoutException) { + mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT); + } else if (exception instanceof MQClientException) { + mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION); + } + + throw mqClientException; + } + + List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); + if (null == nsList || nsList.isEmpty()) { + throw new MQClientException( + "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); + } + + throw new MQClientException("No route info of this topic, " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), + null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); + } + + private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { + TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); + if (null == topicPublishInfo || !topicPublishInfo.ok()) { + this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + } + + if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { + return topicPublishInfo; + } else { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + return topicPublishInfo; + } + } + + private SendResult sendKernelImpl(final Message msg, // + final MessageQueue mq, // + final CommunicationMode communicationMode, // + final SendCallback sendCallback, // + final TopicPublishInfo topicPublishInfo, // + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + tryToFindTopicPublishInfo(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + SendMessageContext context = null; + if (brokerAddr != null) { + brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr); + + byte[] prevBody = msg.getBody(); + try { + + MessageClientIDSetter.setUniqID(msg); + + int sysFlag = 0; + if (this.tryToCompressMessage(msg)) { + sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + } + + final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + } + + if (hasCheckForbiddenHook()) { + CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); + checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); + checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); + checkForbiddenContext.setCommunicationMode(communicationMode); + checkForbiddenContext.setBrokerAddr(brokerAddr); + checkForbiddenContext.setMessage(msg); + checkForbiddenContext.setMq(mq); + checkForbiddenContext.setUnitMode(this.isUnitMode()); + this.executeCheckForbiddenHook(checkForbiddenContext); + } + + if (this.hasSendMessageHook()) { + context = new SendMessageContext(); + context.setProducer(this); + context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + context.setCommunicationMode(communicationMode); + context.setBornHost(this.defaultMQProducer.getClientIP()); + context.setBrokerAddr(brokerAddr); + context.setMessage(msg); + context.setMq(mq); + String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (isTrans != null && isTrans.equals("true")) { + context.setMsgType(MessageType.Trans_Msg_Half); + } + + if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) { + context.setMsgType(MessageType.Delay_Msg); + } + this.executeSendMessageHookBefore(context); + } + + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); + requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setSysFlag(sysFlag); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + requestHeader.setReconsumeTimes(0); + requestHeader.setUnitMode(this.isUnitMode()); + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); + if (reconsumeTimes != null) { + requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); + } + + String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg); + if (maxReconsumeTimes != null) { + requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); + } + } + + SendResult sendResult = null; + switch (communicationMode) { + case ASYNC: + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(// + brokerAddr, // 1 + mq.getBrokerName(), // 2 + msg, // 3 + requestHeader, // 4 + timeout, // 5 + communicationMode, // 6 + sendCallback, // 7 + topicPublishInfo, // 8 + this.mQClientFactory, // 9 + this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(), // 10 + context, // + this); + break; + case ONEWAY: + case SYNC: + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( + brokerAddr, + mq.getBrokerName(), + msg, + requestHeader, + timeout, + communicationMode, + context, + this); + break; + default: + assert false; + break; + } + + if (this.hasSendMessageHook()) { + context.setSendResult(sendResult); + this.executeSendMessageHookAfter(context); + } + + return sendResult; + } catch (RemotingException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (MQBrokerException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (InterruptedException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } finally { + msg.setBody(prevBody); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + private boolean tryToCompressMessage(final Message msg) { + byte[] body = msg.getBody(); + if (body != null) { + if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { + try { + byte[] data = UtilAll.compress(body, zipCompressLevel); + if (data != null) { + msg.setBody(data); + return true; + } + } catch (IOException e) { + log.error("tryToCompressMessage exception", e); + log.warn(msg.toString()); + } + } + } + + return false; + } + + public boolean hasCheckForbiddenHook() { + return !checkForbiddenHookList.isEmpty(); + } + + public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { + if (hasCheckForbiddenHook()) { + for (CheckForbiddenHook hook : checkForbiddenHookList) { + hook.checkForbidden(context); + } + } + } + + public boolean hasSendMessageHook() { + return !this.sendMessageHookList.isEmpty(); + } + + public void executeSendMessageHookBefore(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageBefore(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookBefore", e); + } + } + } + } + + public void executeSendMessageHookAfter(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageAfter(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookAfter", e); + } + } + } + } + + /** + * DEFAULT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * KERNEL SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueue mq) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueue mq, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); + } + + /** + * KERNEL ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, timeout); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * KERNEL ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * SELECT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout); + } + + private SendResult sendSelectImpl(// + Message msg, // + MessageQueueSelector selector, // + Object arg, // + final CommunicationMode communicationMode, // + final SendCallback sendCallback, final long timeout// + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + try { + mq = selector.select(topicPublishInfo.getMessageQueueList(), msg, arg); + } catch (Throwable e) { + throw new MQClientException("select message queue throwed exception.", e); + } + + if (mq != null) { + return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout); + } else { + throw new MQClientException("select message queue return null.", null); + } + } + + throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); + } + + /** + * SELECT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, timeout); + } catch (MQBrokerException e) { + throw new MQClientException("unknownn exception", e); + } + } + + /** + * SELECT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + public TransactionSendResult sendMessageInTransaction(final Message msg, final LocalTransactionExecuter tranExecuter, final Object arg) + throws MQClientException { + if (null == tranExecuter) { + throw new MQClientException("tranExecutor is null", null); + } + Validators.checkMessage(msg, this.defaultMQProducer); + + SendResult sendResult = null; + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup()); + try { + sendResult = this.send(msg); + } catch (Exception e) { + throw new MQClientException("send message Exception", e); + } + + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable localException = null; + switch (sendResult.getSendStatus()) { + case SEND_OK: { + try { + if (sendResult.getTransactionId() != null) { + msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); + } + localTransactionState = tranExecuter.executeLocalTransactionBranch(msg, arg); + if (null == localTransactionState) { + localTransactionState = LocalTransactionState.UNKNOW; + } + + if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { + log.info("executeLocalTransactionBranch return {}", localTransactionState); + log.info(msg.toString()); + } + } catch (Throwable e) { + log.info("executeLocalTransactionBranch exception", e); + log.info(msg.toString()); + localException = e; + } + } + break; + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; + break; + default: + break; + } + + try { + this.endTransaction(sendResult, localTransactionState, localException); + } catch (Exception e) { + log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); + } + + TransactionSendResult transactionSendResult = new TransactionSendResult(); + transactionSendResult.setSendStatus(sendResult.getSendStatus()); + transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); + transactionSendResult.setMsgId(sendResult.getMsgId()); + transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); + transactionSendResult.setTransactionId(sendResult.getTransactionId()); + transactionSendResult.setLocalTransactionState(localTransactionState); + return transactionSendResult; + } + + /** + * DEFAULT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void endTransaction(// + final SendResult sendResult, // + final LocalTransactionState localTransactionState, // + final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { + final MessageId id; + if (sendResult.getOffsetMsgId() != null) { + id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); + } else { + id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); + } + String transactionId = sendResult.getTransactionId(); + final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); + EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); + requestHeader.setTransactionId(transactionId); + requestHeader.setCommitLogOffset(id.getOffset()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + break; + case UNKNOW: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + break; + default: + break; + } + + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); + requestHeader.setMsgId(sendResult.getMsgId()); + String remark = localException != null ? ("executeLocalTransactionBranch exception: " + localException.toString()) : null; + this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark, + this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); + } + + public ConcurrentHashMap getTopicPublishInfoTable() { + return topicPublishInfoTable; + } + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + + public ServiceState getServiceState() { + return serviceState; + } + + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + public long[] getNotAvailableDuration() { + return this.mqFaultStrategy.getNotAvailableDuration(); + } + + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration); + } + + public long[] getLatencyMax() { + return this.mqFaultStrategy.getLatencyMax(); + } + + public void setLatencyMax(final long[] latencyMax) { + this.mqFaultStrategy.setLatencyMax(latencyMax); + } + + public boolean isSendLatencyFaultEnable() { + return this.mqFaultStrategy.isSendLatencyFaultEnable(); + } + + public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { + this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java new file mode 100644 index 0000000000..e2837e2d7a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; + +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public interface MQProducerInner { + Set getPublishTopicList(); + + + boolean isPublishTopicNeedUpdate(final String topic); + + + TransactionCheckListener checkListener(); + + + void checkTransactionState(// + final String addr, // + final MessageExt msg, // + final CheckTransactionStateRequestHeader checkRequestHeader); + + + void updateTopicPublishInfo(final String topic, final TopicPublishInfo info); + + + boolean isUnitMode(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java new file mode 100644 index 0000000000..2f7de22b59 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import com.alibaba.rocketmq.client.common.ThreadLocalIndex; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; + +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class TopicPublishInfo { + private boolean orderTopic = false; + private boolean haveTopicRouterInfo = false; + private List messageQueueList = new ArrayList(); + private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(0); + private TopicRouteData topicRouteData; + + + public boolean isOrderTopic() { + return orderTopic; + } + + public void setOrderTopic(boolean orderTopic) { + this.orderTopic = orderTopic; + } + + public boolean ok() { + return null != this.messageQueueList && !this.messageQueueList.isEmpty(); + } + + public List getMessageQueueList() { + return messageQueueList; + } + + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } + + + public ThreadLocalIndex getSendWhichQueue() { + return sendWhichQueue; + } + + + public void setSendWhichQueue(ThreadLocalIndex sendWhichQueue) { + this.sendWhichQueue = sendWhichQueue; + } + + + public boolean isHaveTopicRouterInfo() { + return haveTopicRouterInfo; + } + + + public void setHaveTopicRouterInfo(boolean haveTopicRouterInfo) { + this.haveTopicRouterInfo = haveTopicRouterInfo; + } + + + public MessageQueue selectOneMessageQueue(final String lastBrokerName) { + if (lastBrokerName == null) { + return selectOneMessageQueue(); + } else { + int index = this.sendWhichQueue.getAndIncrement(); + for (int i = 0; i < this.messageQueueList.size(); i++) { + int pos = Math.abs(index++) % this.messageQueueList.size(); + if (pos < 0) + pos = 0; + MessageQueue mq = this.messageQueueList.get(pos); + if (!mq.getBrokerName().equals(lastBrokerName)) { + return mq; + } + } + return selectOneMessageQueue(); + } + } + + + public MessageQueue selectOneMessageQueue() { + int index = this.sendWhichQueue.getAndIncrement(); + int pos = Math.abs(index) % this.messageQueueList.size(); + if (pos < 0) + pos = 0; + return this.messageQueueList.get(pos); + } + + public int getQueueIdByBroker(final String brokerName) { + for (int i = 0; i < topicRouteData.getQueueDatas().size(); i++) { + final QueueData queueData = this.topicRouteData.getQueueDatas().get(i); + if (queueData.getBrokerName().equals(brokerName)) { + return queueData.getWriteQueueNums(); + } + } + + return -1; + } + + + @Override + public String toString() { + return "TopicPublishInfo [orderTopic=" + orderTopic + ", messageQueueList=" + messageQueueList + + ", sendWhichQueue=" + sendWhichQueue + ", haveTopicRouterInfo=" + haveTopicRouterInfo + "]"; + } + + public TopicRouteData getTopicRouteData() { + return topicRouteData; + } + + public void setTopicRouteData(final TopicRouteData topicRouteData) { + this.topicRouteData = topicRouteData; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultTolerance.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultTolerance.java new file mode 100644 index 0000000000..e6152e4010 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultTolerance.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.latency; + +/** + * @author shijia.wxr + */ +public interface LatencyFaultTolerance { + void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); + + boolean isAvailable(final T name); + + void remove(final T name); + + T pickOneAtLeast(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultToleranceImpl.java new file mode 100644 index 0000000000..8a864497b9 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/LatencyFaultToleranceImpl.java @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.latency; + +import com.alibaba.rocketmq.client.common.ThreadLocalIndex; + +import java.util.Collections; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author shijia.wxr + */ +public class LatencyFaultToleranceImpl implements LatencyFaultTolerance { + private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap(16); + + private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(0); + + @Override + public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) { + FaultItem old = this.faultItemTable.get(name); + if (null == old) { + final FaultItem faultItem = new FaultItem(name); + faultItem.setCurrentLatency(currentLatency); + faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); + + old = this.faultItemTable.putIfAbsent(name, faultItem); + if (old != null) { + old.setCurrentLatency(currentLatency); + old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); + } + } else { + old.setCurrentLatency(currentLatency); + old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); + } + } + + @Override + public boolean isAvailable(final String name) { + final FaultItem faultItem = this.faultItemTable.get(name); + if (faultItem != null) { + return faultItem.isAvailable(); + } + return true; + } + + @Override + public void remove(final String name) { + this.faultItemTable.remove(name); + } + + @Override + public String pickOneAtLeast() { + final Enumeration elements = this.faultItemTable.elements(); + List tmpList = new LinkedList(); + while (elements.hasMoreElements()) { + final FaultItem faultItem = elements.nextElement(); + tmpList.add(faultItem); + } + + if (!tmpList.isEmpty()) { + Collections.shuffle(tmpList); + + Collections.sort(tmpList); + + final int half = tmpList.size() / 2; + if (half <= 0) { + return tmpList.get(0).getName(); + } else { + final int i = this.whichItemWorst.getAndIncrement() % half; + return tmpList.get(i).getName(); + } + } + + return null; + } + + class FaultItem implements Comparable { + private final String name; + private volatile long currentLatency; + private volatile long startTimestamp; + + public FaultItem(final String name) { + this.name = name; + } + + @Override + public int compareTo(final FaultItem other) { + if (this.isAvailable() != other.isAvailable()) { + if (this.isAvailable()) return -1; + + if (other.isAvailable()) return 1; + } + + if (this.currentLatency < other.currentLatency) + return -1; + else if (this.currentLatency > other.currentLatency) { + return 1; + } + + if (this.startTimestamp < other.startTimestamp) + return -1; + else if (this.startTimestamp > other.startTimestamp) { + return 1; + } + + return 0; + } + + public boolean isAvailable() { + return (System.currentTimeMillis() - startTimestamp) >= 0; + } + + @Override + public int hashCode() { + int result = getName() != null ? getName().hashCode() : 0; + result = 31 * result + (int) (getCurrentLatency() ^ (getCurrentLatency() >>> 32)); + result = 31 * result + (int) (getStartTimestamp() ^ (getStartTimestamp() >>> 32)); + return result; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (!(o instanceof FaultItem)) return false; + + final FaultItem faultItem = (FaultItem) o; + + if (getCurrentLatency() != faultItem.getCurrentLatency()) return false; + if (getStartTimestamp() != faultItem.getStartTimestamp()) return false; + return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null; + + } + + @Override + public String toString() { + return "FaultItem{" + + "name='" + name + '\'' + + ", currentLatency=" + currentLatency + + ", startTimestamp=" + startTimestamp + + '}'; + } + + public String getName() { + return name; + } + + public long getCurrentLatency() { + return currentLatency; + } + + public void setCurrentLatency(final long currentLatency) { + this.currentLatency = currentLatency; + } + + public long getStartTimestamp() { + return startTimestamp; + } + + public void setStartTimestamp(final long startTimestamp) { + this.startTimestamp = startTimestamp; + } + + + } + + @Override + public String toString() { + return "LatencyFaultToleranceImpl{" + + "faultItemTable=" + faultItemTable + + ", whichItemWorst=" + whichItemWorst + + '}'; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/MQFaultStrategy.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/MQFaultStrategy.java new file mode 100644 index 0000000000..b323f04199 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/latency/MQFaultStrategy.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.latency; + +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.common.message.MessageQueue; + +/** + * @author shijia.wxr + */ +public class MQFaultStrategy { + private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); + + private boolean sendLatencyFaultEnable = false; + + private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L}; + private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L}; + + public long[] getNotAvailableDuration() { + return notAvailableDuration; + } + + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.notAvailableDuration = notAvailableDuration; + } + + public long[] getLatencyMax() { + return latencyMax; + } + + public void setLatencyMax(final long[] latencyMax) { + this.latencyMax = latencyMax; + } + + public boolean isSendLatencyFaultEnable() { + return sendLatencyFaultEnable; + } + + public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { + this.sendLatencyFaultEnable = sendLatencyFaultEnable; + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { + if (this.sendLatencyFaultEnable) { + try { + int index = tpInfo.getSendWhichQueue().getAndIncrement(); + for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { + int pos = Math.abs(index++) % tpInfo.getMessageQueueList().size(); + if (pos < 0) + pos = 0; + MessageQueue mq = tpInfo.getMessageQueueList().get(pos); + if (latencyFaultTolerance.isAvailable(mq.getBrokerName())) { + if (null == lastBrokerName || mq.getBrokerName().equals(lastBrokerName)) + return mq; + } + } + + final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); + int writeQueueNums = tpInfo.getQueueIdByBroker(notBestBroker); + if (writeQueueNums > 0) { + final MessageQueue mq = tpInfo.selectOneMessageQueue(); + if (notBestBroker != null) { + mq.setBrokerName(notBestBroker); + mq.setQueueId(tpInfo.getSendWhichQueue().getAndIncrement() % writeQueueNums); + } + return mq; + } else { + latencyFaultTolerance.remove(notBestBroker); + } + } catch (Exception e) { + } + + return tpInfo.selectOneMessageQueue(); + } + + return tpInfo.selectOneMessageQueue(lastBrokerName); + } + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { + if (this.sendLatencyFaultEnable) { + long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency); + this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration); + } + } + + private long computeNotAvailableDuration(final long currentLatency) { + for (int i = latencyMax.length - 1; i >= 0; i--) { + if (currentLatency >= latencyMax[i]) return this.notAvailableDuration[i]; + } + + return 0; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java new file mode 100644 index 0000000000..02af207ee3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.log; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Method; +import java.net.URL; + + +/** + * @author shijia.wxr + */ +public class ClientLogger { + private static Logger log; + public static final String CLIENT_LOG_ROOT = "rocketmq.client.logRoot"; + public static final String CLIENT_LOG_MAXINDEX = "rocketmq.client.logFileMaxIndex"; + public static final String CLIENT_LOG_LEVEL = "rocketmq.client.logLevel"; + + static { + log = createLogger(LoggerName.CLIENT_LOGGER_NAME); + } + + + private static Logger createLogger(final String loggerName) { + String logConfigFilePath = + System.getProperty("rocketmq.client.log.configFile", + System.getenv("ROCKETMQ_CLIENT_LOG_CONFIGFILE")); + Boolean isloadconfig = + Boolean.parseBoolean(System.getProperty("rocketmq.client.log.loadconfig", "true")); + + final String log4JResourceFile = + System.getProperty("rocketmq.client.log4j.resource.fileName", "log4j_rocketmq_client.xml"); + + final String logbackResourceFile = + System.getProperty("rocketmq.client.logback.resource.fileName", "logback_rocketmq_client.xml"); + + String clientLogRoot = System.getProperty(CLIENT_LOG_ROOT, "${user.home}/logs/rocketmqlogs"); + System.setProperty("client.logRoot", clientLogRoot); + String clientLogLevel = System.getProperty(CLIENT_LOG_LEVEL, "INFO"); + System.setProperty("client.logLevel", clientLogLevel); + String clientLogMaxIndex = System.getProperty(CLIENT_LOG_MAXINDEX, "10"); + System.setProperty("client.logFileMaxIndex", clientLogMaxIndex); + + if (isloadconfig) { + try { + ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory(); + Class classType = iLoggerFactory.getClass(); + if (classType.getName().equals("org.slf4j.impl.Log4jLoggerFactory")) { + Class domconfigurator; + Object domconfiguratorobj; + domconfigurator = Class.forName("org.apache.log4j.xml.DOMConfigurator"); + domconfiguratorobj = domconfigurator.newInstance(); + if (null == logConfigFilePath) { + Method configure = domconfiguratorobj.getClass().getMethod("configure", URL.class); + URL url = ClientLogger.class.getClassLoader().getResource(log4JResourceFile); + configure.invoke(domconfiguratorobj, url); + } else { + Method configure = domconfiguratorobj.getClass().getMethod("configure", String.class); + configure.invoke(domconfiguratorobj, logConfigFilePath); + } + + } else if (classType.getName().equals("ch.qos.logback.classic.LoggerContext")) { + Class joranConfigurator; + Class context = Class.forName("ch.qos.logback.core.Context"); + Object joranConfiguratoroObj; + joranConfigurator = Class.forName("ch.qos.logback.classic.joran.JoranConfigurator"); + joranConfiguratoroObj = joranConfigurator.newInstance(); + Method setContext = joranConfiguratoroObj.getClass().getMethod("setContext", context); + setContext.invoke(joranConfiguratoroObj, iLoggerFactory); + if (null == logConfigFilePath) { + URL url = ClientLogger.class.getClassLoader().getResource(logbackResourceFile); + Method doConfigure = + joranConfiguratoroObj.getClass().getMethod("doConfigure", URL.class); + doConfigure.invoke(joranConfiguratoroObj, url); + } else { + Method doConfigure = + joranConfiguratoroObj.getClass().getMethod("doConfigure", String.class); + doConfigure.invoke(joranConfiguratoroObj, logConfigFilePath); + } + + } + } catch (Exception e) { + System.err.println(e); + } + } + return LoggerFactory.getLogger(LoggerName.CLIENT_LOGGER_NAME); + } + + + public static Logger getLog() { + return log; + } + + + public static void setLog(Logger log) { + ClientLogger.log = log; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java new file mode 100644 index 0000000000..6f861d3193 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java @@ -0,0 +1,380 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class DefaultMQProducer extends ClientConfig implements MQProducer { + protected final transient DefaultMQProducerImpl defaultMQProducerImpl; + private String producerGroup; + /** + * Just for testing or demo program + */ + private String createTopicKey = MixAll.DEFAULT_TOPIC; + private volatile int defaultTopicQueueNums = 4; + private int sendMsgTimeout = 3000; + private int compressMsgBodyOverHowmuch = 1024 * 4; + private int retryTimesWhenSendFailed = 2; + private int retryTimesWhenSendAsyncFailed = 2; + + private boolean retryAnotherBrokerWhenNotStoreOK = false; + private int maxMessageSize = 1024 * 1024 * 4; // 4M + public DefaultMQProducer() { + this(MixAll.DEFAULT_PRODUCER_GROUP, null); + } + + + public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { + this.producerGroup = producerGroup; + defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); + } + + + public DefaultMQProducer(final String producerGroup) { + this(producerGroup, null); + } + + + public DefaultMQProducer(RPCHook rpcHook) { + this(MixAll.DEFAULT_PRODUCER_GROUP, rpcHook); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQProducerImpl.start(); + } + + @Override + public void shutdown() { + this.defaultMQProducerImpl.shutdown(); + } + + + @Override + public List fetchPublishMessageQueues(String topic) throws MQClientException { + return this.defaultMQProducerImpl.fetchPublishMessageQueues(topic); + } + + + @Override + public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg); + } + + + @Override + public SendResult send(Message msg, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, timeout); + } + + + @Override + public void send(Message msg, SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, sendCallback); + } + + + @Override + public void send(Message msg, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, sendCallback, timeout); + } + + + @Override + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg); + } + + + @Override + public SendResult send(Message msg, MessageQueue mq) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, mq); + } + + + @Override + public SendResult send(Message msg, MessageQueue mq, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, mq, timeout); + } + + + @Override + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, mq, sendCallback); + } + + + @Override + public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, mq, sendCallback, timeout); + } + + + @Override + public void sendOneway(Message msg, MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg, mq); + } + + + @Override + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, selector, arg); + } + + + @Override + public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, selector, arg, timeout); + } + + + @Override + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback); + } + + + @Override + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback, timeout); + } + + + @Override + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg, selector, arg); + } + + + @Override + public TransactionSendResult sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, final Object arg) + throws MQClientException { + throw new RuntimeException("sendMessageInTransaction not implement, please use TransactionMQProducer class"); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.defaultMQProducerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQProducerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.defaultMQProducerImpl.viewMessage(offsetMsgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQProducerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + @Override + public MessageExt viewMessage(String topic, String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + MessageId oldMsgId = MessageDecoder.decodeMessageId(msgId); + return this.viewMessage(msgId); + } catch (Exception e) { + } + return this.defaultMQProducerImpl.queryMessageByUniqKey(topic, msgId); + } + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getCreateTopicKey() { + return createTopicKey; + } + + + public void setCreateTopicKey(String createTopicKey) { + this.createTopicKey = createTopicKey; + } + + + public int getSendMsgTimeout() { + return sendMsgTimeout; + } + + + public void setSendMsgTimeout(int sendMsgTimeout) { + this.sendMsgTimeout = sendMsgTimeout; + } + + + public int getCompressMsgBodyOverHowmuch() { + return compressMsgBodyOverHowmuch; + } + + + public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) { + this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch; + } + + + public DefaultMQProducerImpl getDefaultMQProducerImpl() { + return defaultMQProducerImpl; + } + + + public boolean isRetryAnotherBrokerWhenNotStoreOK() { + return retryAnotherBrokerWhenNotStoreOK; + } + + + public void setRetryAnotherBrokerWhenNotStoreOK(boolean retryAnotherBrokerWhenNotStoreOK) { + this.retryAnotherBrokerWhenNotStoreOK = retryAnotherBrokerWhenNotStoreOK; + } + + + public int getMaxMessageSize() { + return maxMessageSize; + } + + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + + public int getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(int defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public int getRetryTimesWhenSendFailed() { + return retryTimesWhenSendFailed; + } + + + public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) { + this.retryTimesWhenSendFailed = retryTimesWhenSendFailed; + } + + + public boolean isSendMessageWithVIPChannel() { + return isVipChannelEnabled(); + } + + + public void setSendMessageWithVIPChannel(final boolean sendMessageWithVIPChannel) { + this.setVipChannelEnabled(sendMessageWithVIPChannel); + } + + + public long[] getNotAvailableDuration() { + return this.defaultMQProducerImpl.getNotAvailableDuration(); + } + + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.defaultMQProducerImpl.setNotAvailableDuration(notAvailableDuration); + } + + public long[] getLatencyMax() { + return this.defaultMQProducerImpl.getLatencyMax(); + } + + public void setLatencyMax(final long[] latencyMax) { + this.defaultMQProducerImpl.setLatencyMax(latencyMax); + } + + public boolean isSendLatencyFaultEnable() { + return this.defaultMQProducerImpl.isSendLatencyFaultEnable(); + } + + public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { + this.defaultMQProducerImpl.setSendLatencyFaultEnable(sendLatencyFaultEnable); + } + + public int getRetryTimesWhenSendAsyncFailed() { + return retryTimesWhenSendAsyncFailed; + } + + public void setRetryTimesWhenSendAsyncFailed(final int retryTimesWhenSendAsyncFailed) { + this.retryTimesWhenSendAsyncFailed = retryTimesWhenSendAsyncFailed; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java new file mode 100644 index 0000000000..af3723ae65 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.common.message.Message; + + +/** + * @author shijia.wxr + */ +public interface LocalTransactionExecuter { + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java new file mode 100644 index 0000000000..ee2a93ae47 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * @author shijia.wxr + */ +public enum LocalTransactionState { + COMMIT_MESSAGE, + ROLLBACK_MESSAGE, + UNKNOW, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java new file mode 100644 index 0000000000..e21bc00858 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public interface MQProducer extends MQAdmin { + void start() throws MQClientException; + + void shutdown(); + + + List fetchPublishMessageQueues(final String topic) throws MQClientException; + + + SendResult send(final Message msg) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + + SendResult send(final Message msg, final long timeout) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException; + + + void send(final Message msg, final SendCallback sendCallback) throws MQClientException, + RemotingException, InterruptedException; + + + void send(final Message msg, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException; + + + void sendOneway(final Message msg) throws MQClientException, RemotingException, + InterruptedException; + + + SendResult send(final Message msg, final MessageQueue mq) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException; + + + SendResult send(final Message msg, final MessageQueue mq, final long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + + + void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException; + + + void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException; + + + void sendOneway(final Message msg, final MessageQueue mq) throws MQClientException, + RemotingException, InterruptedException; + + + SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + + + SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + + void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback) throws MQClientException, RemotingException, + InterruptedException; + + + void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, + InterruptedException; + + + void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg) + throws MQClientException, RemotingException, InterruptedException; + + + TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java new file mode 100644 index 0000000000..924c14521b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public interface MessageQueueSelector { + MessageQueue select(final List mqs, final Message msg, final Object arg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java new file mode 100644 index 0000000000..35d1a729b2 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * @author shijia.wxr + */ +public interface SendCallback { + public void onSuccess(final SendResult sendResult); + + + public void onException(final Throwable e); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java new file mode 100644 index 0000000000..183accfe28 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.fastjson.JSON; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * @author shijia.wxr + */ +public class SendResult { + private SendStatus sendStatus; + private String msgId; + private MessageQueue messageQueue; + private long queueOffset; + private String transactionId; + private String offsetMsgId; + private String regionId; + private boolean traceOn = true; + + public SendResult() { + } + + public SendResult(SendStatus sendStatus, String msgId, String offsetMsgId, MessageQueue messageQueue, long queueOffset) { + this.sendStatus = sendStatus; + this.msgId = msgId; + this.offsetMsgId = offsetMsgId; + this.messageQueue = messageQueue; + this.queueOffset = queueOffset; + } + + public SendResult(final SendStatus sendStatus, final String msgId, final MessageQueue messageQueue, final long queueOffset, final String transactionId, final String offsetMsgId, final String regionId) { + this.sendStatus = sendStatus; + this.msgId = msgId; + this.messageQueue = messageQueue; + this.queueOffset = queueOffset; + this.transactionId = transactionId; + this.offsetMsgId = offsetMsgId; + this.regionId = regionId; + } + + public boolean isTraceOn() { + return traceOn; + } + + public void setTraceOn(final boolean traceOn) { + this.traceOn = traceOn; + } + + public String getRegionId() { + return regionId; + } + + public void setRegionId(final String regionId) { + this.regionId = regionId; + } + + public static String encoderSendResultToJson(final Object obj) { + return JSON.toJSONString(obj); + } + + public static SendResult decoderSendResultFromJson(String json) { + return JSON.parseObject(json, SendResult.class); + } + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public SendStatus getSendStatus() { + return sendStatus; + } + + + public void setSendStatus(SendStatus sendStatus) { + this.sendStatus = sendStatus; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(long queueOffset) { + this.queueOffset = queueOffset; + } + + + public String getTransactionId() { + return transactionId; + } + + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + + public String getOffsetMsgId() { + return offsetMsgId; + } + + public void setOffsetMsgId(String offsetMsgId) { + this.offsetMsgId = offsetMsgId; + } + + @Override + public String toString() { + return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue + + ", queueOffset=" + queueOffset + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java new file mode 100644 index 0000000000..3bc572f2c8 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * @author shijia.wxr + */ +public enum SendStatus { + SEND_OK, + FLUSH_DISK_TIMEOUT, + FLUSH_SLAVE_TIMEOUT, + SLAVE_NOT_AVAILABLE, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java new file mode 100644 index 0000000000..8440537b4a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * @author shijia.wxr + */ +public interface TransactionCheckListener { + LocalTransactionState checkLocalTransactionState(final MessageExt msg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java new file mode 100644 index 0000000000..08dd4abb56 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.RPCHook; + + +/** + * @author shijia.wxr + */ +public class TransactionMQProducer extends DefaultMQProducer { + private TransactionCheckListener transactionCheckListener; + private int checkThreadPoolMinSize = 1; + private int checkThreadPoolMaxSize = 1; + private int checkRequestHoldMax = 2000; + + + public TransactionMQProducer() { + } + + + public TransactionMQProducer(final String producerGroup) { + super(producerGroup); + } + + public TransactionMQProducer(final String producerGroup, RPCHook rpcHook) { + super(producerGroup, rpcHook); + } + + @Override + public void start() throws MQClientException { + this.defaultMQProducerImpl.initTransactionEnv(); + super.start(); + } + + + @Override + public void shutdown() { + super.shutdown(); + this.defaultMQProducerImpl.destroyTransactionEnv(); + } + + + @Override + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException { + if (null == this.transactionCheckListener) { + throw new MQClientException("localTransactionBranchCheckListener is null", null); + } + + return this.defaultMQProducerImpl.sendMessageInTransaction(msg, tranExecuter, arg); + } + + + public TransactionCheckListener getTransactionCheckListener() { + return transactionCheckListener; + } + + + public void setTransactionCheckListener(TransactionCheckListener transactionCheckListener) { + this.transactionCheckListener = transactionCheckListener; + } + + + public int getCheckThreadPoolMinSize() { + return checkThreadPoolMinSize; + } + + + public void setCheckThreadPoolMinSize(int checkThreadPoolMinSize) { + this.checkThreadPoolMinSize = checkThreadPoolMinSize; + } + + + public int getCheckThreadPoolMaxSize() { + return checkThreadPoolMaxSize; + } + + + public void setCheckThreadPoolMaxSize(int checkThreadPoolMaxSize) { + this.checkThreadPoolMaxSize = checkThreadPoolMaxSize; + } + + + public int getCheckRequestHoldMax() { + return checkRequestHoldMax; + } + + + public void setCheckRequestHoldMax(int checkRequestHoldMax) { + this.checkRequestHoldMax = checkRequestHoldMax; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java new file mode 100644 index 0000000000..e7dcd0eb21 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * @author shijia.wxr + */ +public class TransactionSendResult extends SendResult { + private LocalTransactionState localTransactionState; + + + public TransactionSendResult() { + } + + + public LocalTransactionState getLocalTransactionState() { + return localTransactionState; + } + + + public void setLocalTransactionState(LocalTransactionState localTransactionState) { + this.localTransactionState = localTransactionState; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java new file mode 100644 index 0000000000..648356be97 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class SelectMessageQueueByHash implements MessageQueueSelector { + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + int value = arg.hashCode(); + if (value < 0) { + value = Math.abs(value); + } + + value = value % mqs.size(); + return mqs.get(value); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java new file mode 100644 index 0000000000..a21339191d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class SelectMessageQueueByMachineRoom implements MessageQueueSelector { + private Set consumeridcs; + + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + return null; + } + + + public Set getConsumeridcs() { + return consumeridcs; + } + + + public void setConsumeridcs(Set consumeridcs) { + this.consumeridcs = consumeridcs; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java new file mode 100644 index 0000000000..3f381e4653 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; +import java.util.Random; + + +/** + * @author shijia.wxr + */ +public class SelectMessageQueueByRandoom implements MessageQueueSelector { + private Random random = new Random(System.currentTimeMillis()); + + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + int value = random.nextInt(); + if (value < 0) { + value = Math.abs(value); + } + + value = value % mqs.size(); + return mqs.get(value); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java new file mode 100644 index 0000000000..e07e2334f0 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client.stat; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.common.stats.StatsItemSet; +import com.alibaba.rocketmq.common.stats.StatsSnapshot; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledExecutorService; + + +public class ConsumerStatsManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.CLIENT_LOGGER_NAME); + + private static final String TOPIC_AND_GROUP_CONSUME_OK_TPS = "CONSUME_OK_TPS"; + private static final String TOPIC_AND_GROUP_CONSUME_FAILED_TPS = "CONSUME_FAILED_TPS"; + private static final String TOPIC_AND_GROUP_CONSUME_RT = "CONSUME_RT"; + private static final String TOPIC_AND_GROUP_PULL_TPS = "PULL_TPS"; + private static final String TOPIC_AND_GROUP_PULL_RT = "PULL_RT"; + + private final StatsItemSet topicAndGroupConsumeOKTPS; + private final StatsItemSet topicAndGroupConsumeRT; + private final StatsItemSet topicAndGroupConsumeFailedTPS; + private final StatsItemSet topicAndGroupPullTPS; + private final StatsItemSet topicAndGroupPullRT; + + + public ConsumerStatsManager(final ScheduledExecutorService scheduledExecutorService) { + this.topicAndGroupConsumeOKTPS = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_OK_TPS, scheduledExecutorService, log); + + this.topicAndGroupConsumeRT = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_RT, scheduledExecutorService, log); + + this.topicAndGroupConsumeFailedTPS = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_FAILED_TPS, scheduledExecutorService, log); + + this.topicAndGroupPullTPS = new StatsItemSet(TOPIC_AND_GROUP_PULL_TPS, scheduledExecutorService, log); + + this.topicAndGroupPullRT = new StatsItemSet(TOPIC_AND_GROUP_PULL_RT, scheduledExecutorService, log); + } + + + public void start() { + } + + + public void shutdown() { + } + + + public void incPullRT(final String group, final String topic, final long rt) { + this.topicAndGroupPullRT.addValue(topic + "@" + group, (int) rt, 1); + } + + + public void incPullTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupPullTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + + public void incConsumeRT(final String group, final String topic, final long rt) { + this.topicAndGroupConsumeRT.addValue(topic + "@" + group, (int) rt, 1); + } + + + public void incConsumeOKTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupConsumeOKTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + + public void incConsumeFailedTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupConsumeFailedTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + public ConsumeStatus consumeStatus(final String group, final String topic) { + ConsumeStatus cs = new ConsumeStatus(); + { + StatsSnapshot ss = this.getPullRT(group, topic); + if (ss != null) { + cs.setPullRT(ss.getAvgpt()); + } + } + + { + StatsSnapshot ss = this.getPullTPS(group, topic); + if (ss != null) { + cs.setPullTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.getConsumeRT(group, topic); + if (ss != null) { + cs.setConsumeRT(ss.getAvgpt()); + } + } + + { + StatsSnapshot ss = this.getConsumeOKTPS(group, topic); + if (ss != null) { + cs.setConsumeOKTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.getConsumeFailedTPS(group, topic); + if (ss != null) { + cs.setConsumeFailedTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group); + if (ss != null) { + cs.setConsumeFailedMsgs(ss.getSum()); + } + } + + return cs; + } + + private StatsSnapshot getPullRT(final String group, final String topic) { + return this.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group); + } + + private StatsSnapshot getPullTPS(final String group, final String topic) { + return this.topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group); + } + + private StatsSnapshot getConsumeRT(final String group, final String topic) { + StatsSnapshot statsData = this.topicAndGroupConsumeRT.getStatsDataInMinute(topic + "@" + group); + if (0 == statsData.getSum()) { + statsData = this.topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group); + } + + return statsData; + } + + private StatsSnapshot getConsumeOKTPS(final String group, final String topic) { + return this.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group); + } + + private StatsSnapshot getConsumeFailedTPS(final String group, final String topic) { + return this.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group); + } +} diff --git a/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml b/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml new file mode 100644 index 0000000000..bf4b885e3a --- /dev/null +++ b/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rocketmq-client/src/main/resources/logback_rocketmq_client.xml b/rocketmq-client/src/main/resources/logback_rocketmq_client.xml new file mode 100644 index 0000000000..a845ee4c11 --- /dev/null +++ b/rocketmq-client/src/main/resources/logback_rocketmq_client.xml @@ -0,0 +1,58 @@ + + + + + + ${client.logRoot}/rocketmq_client.log + true + + ${client.logRoot}/otherdays/rocketmq_client.%properties.log + + 1 + ${client.logFileMaxIndex} + + + 100MB + + + %defaultTopicQueueNums{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + diff --git a/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java new file mode 100644 index 0000000000..6dadafb499 --- /dev/null +++ b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.client; + +import org.junit.Assert; +import org.junit.Test; + + +public class ValidatorsTest { + + @Test + public void topicValidatorTest() { + try { + Validators.checkTopic("Hello"); + Validators.checkTopic("%RETRY%Hello"); + Validators.checkTopic("_%RETRY%Hello"); + Validators.checkTopic("-%RETRY%Hello"); + Validators.checkTopic("223-%RETRY%Hello"); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + } +} diff --git a/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java new file mode 100644 index 0000000000..5ef75ede6b --- /dev/null +++ b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * @author yubao.fyb@taoboa.com + * @version $id$ + */ +package com.alibaba.rocketmq.client.consumer.loadbalance; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle; +import com.alibaba.rocketmq.common.message.MessageQueue; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + + +/** + * @author yubao.fyb@alibaba-inc.com created on 2013-07-03 16:24 + */ +public class AllocateMessageQueueAveragelyTest { + private AllocateMessageQueueStrategy allocateMessageQueueAveragely; + private String currentCID; + private String topic; + private List messageQueueList; + private List consumerIdList; + + @Before + public void init() { + allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(); + topic = "topic_test"; + } + + @Test + public void testConsumer1() { + currentCID = "0"; + createConsumerIdList(1); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer1"); + Assert.assertEquals(result.size(), 5); + Assert.assertEquals(result.containsAll(getMessageQueueList()), true); + } + + public void createConsumerIdList(int size) { + consumerIdList = new ArrayList(size); + for (int i = 0; i < size; i++) { + consumerIdList.add(String.valueOf(i)); + } + } + + public void createMessageQueueList(int size) { + messageQueueList = new ArrayList(size); + for (int i = 0; i < size; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + messageQueueList.add(mq); + } + } + + public void printMessageQueue(List messageQueueList, String name) { + if (messageQueueList == null || messageQueueList.size() < 1) + return; + System.out.println(name + ".......................................start"); + for (MessageQueue messageQueue : messageQueueList) { + System.out.println(messageQueue); + } + System.out.println(name + ".......................................end"); + } + + public List getMessageQueueList() { + return messageQueueList; + } + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } + + @Test + public void testConsumer2() { + currentCID = "1"; + createConsumerIdList(2); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer2"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true); + + } + + @Test + public void testConsumer3CurrentCID0() { + currentCID = "0"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID0"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(0, 1)), true); + } + + @Test + public void testConsumer3CurrentCID1() { + currentCID = "1"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID1"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + @Test + public void testConsumer3CurrentCID2() { + currentCID = "2"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID2"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true); + } + + @Test + public void testConsumer4() { + currentCID = "1"; + createConsumerIdList(4); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer4"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + @Test + public void testConsumer5() { + currentCID = "1"; + createConsumerIdList(5); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer5"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + @Test + public void testConsumer6() { + currentCID = "1"; + createConsumerIdList(2); + createMessageQueueList(6); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(3, 6)), true); + } + + @Test + public void testCurrentCIDNotExists() { + currentCID = String.valueOf(Integer.MAX_VALUE); + createConsumerIdList(2); + createMessageQueueList(6); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testCurrentCIDNotExists"); + Assert.assertEquals(result.size(), 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testCurrentCIDIllegalArgument() { + createConsumerIdList(2); + createMessageQueueList(6); + allocateMessageQueueAveragely.allocate("", "", getMessageQueueList(), getConsumerIdList()); + } + + public List getConsumerIdList() { + return consumerIdList; + } + + public void setConsumerIdList(List consumerIdList) { + this.consumerIdList = consumerIdList; + } + + @Test(expected = IllegalArgumentException.class) + public void testMessageQueueIllegalArgument() { + currentCID = "0"; + createConsumerIdList(2); + allocateMessageQueueAveragely.allocate("", currentCID, null, getConsumerIdList()); + } + + @Test(expected = IllegalArgumentException.class) + public void testConsumerIdIllegalArgument() { + currentCID = "0"; + createMessageQueueList(6); + allocateMessageQueueAveragely.allocate("", currentCID, getMessageQueueList(), null); + } + + @Test + public void testAllocate() { + AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(); + String topic = "topic_test"; + String currentCID = "CID"; + int queueSize = 19; + int consumerSize = 10; + List mqAll = new ArrayList(); + for (int i = 0; i < queueSize; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + mqAll.add(mq); + } + + List cidAll = new ArrayList(); + for (int j = 0; j < consumerSize; j++) { + cidAll.add("CID" + j); + } + System.out.println(mqAll.toString()); + System.out.println(cidAll.toString()); + for (int i = 0; i < consumerSize; i++) { + List rs = allocateMessageQueueAveragely.allocate("", currentCID + i, mqAll, cidAll); + System.out.println("rs[" + currentCID + i + "]:" + rs.toString()); + } + } + + + @Test + public void testAllocateByCircle() { + AllocateMessageQueueAveragelyByCircle circle = new AllocateMessageQueueAveragelyByCircle(); + String topic = "topic_test"; + String currentCID = "CID"; + int consumerSize = 3; + int queueSize = 13; + List mqAll = new ArrayList(); + for (int i = 0; i < queueSize; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + mqAll.add(mq); + } + + List cidAll = new ArrayList(); + for (int j = 0; j < consumerSize; j++) { + cidAll.add("CID" + j); + } + System.out.println(mqAll.toString()); + System.out.println(cidAll.toString()); + for (int i = 0; i < consumerSize; i++) { + List rs = circle.allocate("", currentCID + i, mqAll, cidAll); + System.out.println("rs[" + currentCID + i + "]:" + rs.toString()); + } + } +} diff --git a/rocketmq-common/pom.xml b/rocketmq-common/pom.xml new file mode 100644 index 0000000000..72cc2b0955 --- /dev/null +++ b/rocketmq-common/pom.xml @@ -0,0 +1,43 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-common + rocketmq-common ${project.version} + + + + + junit + junit + test + + + ${project.groupId} + rocketmq-remoting + + + diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java new file mode 100644 index 0000000000..6eae0a7429 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java @@ -0,0 +1,540 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + +import java.net.InetAddress; +import java.net.UnknownHostException; + + +/** + * @author shijia.wxr + */ +public class BrokerConfig { + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + @ImportantField + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV)); + @ImportantField + private String brokerIP1 = RemotingUtil.getLocalAddress(); + private String brokerIP2 = RemotingUtil.getLocalAddress(); + @ImportantField + private String brokerName = localHostName(); + @ImportantField + private String brokerClusterName = "DefaultCluster"; + @ImportantField + private long brokerId = MixAll.MASTER_ID; + private int brokerPermission = PermName.PERM_READ | PermName.PERM_WRITE; + private int defaultTopicQueueNums = 8; + @ImportantField + private boolean autoCreateTopicEnable = true; + + private boolean clusterTopicEnable = true; + + private boolean brokerTopicEnable = true; + @ImportantField + private boolean autoCreateSubscriptionGroup = true; + private String messageStorePlugIn = ""; + + private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4; + private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; + private int adminBrokerThreadPoolNums = 16; + private int clientManageThreadPoolNums = 32; + private int consumerManageThreadPoolNums = 32; + + private int flushConsumerOffsetInterval = 1000 * 5; + + private int flushConsumerOffsetHistoryInterval = 1000 * 60; + + @ImportantField + private boolean rejectTransactionMessage = false; + @ImportantField + private boolean fetchNamesrvAddrByAddressServer = false; + private int sendThreadPoolQueueCapacity = 10000; + private int pullThreadPoolQueueCapacity = 100000; + private int clientManagerThreadPoolQueueCapacity = 1000000; + private int consumerManagerThreadPoolQueueCapacity = 1000000; + + private int filterServerNums = 0; + + private boolean longPollingEnable = true; + + private long shortPollingTimeMills = 1000; + + private boolean notifyConsumerIdsChangedEnable = true; + + private boolean highSpeedMode = false; + + private boolean commercialEnable = true; + private int commercialTimerCount = 1; + private int commercialTransCount = 1; + private int commercialBigCount = 1; + + private boolean transferMsgByHeap = true; + private int maxDelayTime = 40; + + + private String regionId = MixAll.DEFAULT_TRACE_REGION_ID; + private int registerBrokerTimeoutMills = 6000; + + private boolean slaveReadEnable = false; + + private boolean disableConsumeIfConsumerReadSlowly = false; + private long consumerFallbehindThreshold = 1024 * 1024 * 1024 * 16; + + private long waitTimeMillsInSendQueue = 200; + + private long startAcceptSendRequestTimeStamp = 0L; + + private boolean traceOn = true; + + public boolean isTraceOn() { + return traceOn; + } + + public void setTraceOn(final boolean traceOn) { + this.traceOn = traceOn; + } + + public long getStartAcceptSendRequestTimeStamp() { + return startAcceptSendRequestTimeStamp; + } + + public void setStartAcceptSendRequestTimeStamp(final long startAcceptSendRequestTimeStamp) { + this.startAcceptSendRequestTimeStamp = startAcceptSendRequestTimeStamp; + } + + public long getWaitTimeMillsInSendQueue() { + return waitTimeMillsInSendQueue; + } + + public void setWaitTimeMillsInSendQueue(final long waitTimeMillsInSendQueue) { + this.waitTimeMillsInSendQueue = waitTimeMillsInSendQueue; + } + + public long getConsumerFallbehindThreshold() { + return consumerFallbehindThreshold; + } + + public void setConsumerFallbehindThreshold(final long consumerFallbehindThreshold) { + this.consumerFallbehindThreshold = consumerFallbehindThreshold; + } + + public boolean isDisableConsumeIfConsumerReadSlowly() { + return disableConsumeIfConsumerReadSlowly; + } + + public void setDisableConsumeIfConsumerReadSlowly(final boolean disableConsumeIfConsumerReadSlowly) { + this.disableConsumeIfConsumerReadSlowly = disableConsumeIfConsumerReadSlowly; + } + + public boolean isSlaveReadEnable() { + return slaveReadEnable; + } + + public void setSlaveReadEnable(final boolean slaveReadEnable) { + this.slaveReadEnable = slaveReadEnable; + } + + public static String localHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + return "DEFAULT_BROKER"; + } + + public int getRegisterBrokerTimeoutMills() { + return registerBrokerTimeoutMills; + } + + public void setRegisterBrokerTimeoutMills(final int registerBrokerTimeoutMills) { + this.registerBrokerTimeoutMills = registerBrokerTimeoutMills; + } + + public String getRegionId() { + return regionId; + } + + public void setRegionId(final String regionId) { + this.regionId = regionId; + } + + public boolean isTransferMsgByHeap() { + return transferMsgByHeap; + } + + public void setTransferMsgByHeap(final boolean transferMsgByHeap) { + this.transferMsgByHeap = transferMsgByHeap; + } + + public String getMessageStorePlugIn() { + return messageStorePlugIn; + } + + public void setMessageStorePlugIn(String messageStorePlugIn) { + this.messageStorePlugIn = messageStorePlugIn; + } + + public boolean isHighSpeedMode() { + return highSpeedMode; + } + + + public void setHighSpeedMode(final boolean highSpeedMode) { + this.highSpeedMode = highSpeedMode; + } + + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getBrokerPermission() { + return brokerPermission; + } + + + public void setBrokerPermission(int brokerPermission) { + this.brokerPermission = brokerPermission; + } + + + public int getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(int defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public boolean isAutoCreateTopicEnable() { + return autoCreateTopicEnable; + } + + + public void setAutoCreateTopicEnable(boolean autoCreateTopic) { + this.autoCreateTopicEnable = autoCreateTopic; + } + + + public String getBrokerClusterName() { + return brokerClusterName; + } + + + public void setBrokerClusterName(String brokerClusterName) { + this.brokerClusterName = brokerClusterName; + } + + + public String getBrokerIP1() { + return brokerIP1; + } + + + public void setBrokerIP1(String brokerIP1) { + this.brokerIP1 = brokerIP1; + } + + + public String getBrokerIP2() { + return brokerIP2; + } + + + public void setBrokerIP2(String brokerIP2) { + this.brokerIP2 = brokerIP2; + } + + public int getSendMessageThreadPoolNums() { + return sendMessageThreadPoolNums; + } + + public void setSendMessageThreadPoolNums(int sendMessageThreadPoolNums) { + this.sendMessageThreadPoolNums = sendMessageThreadPoolNums; + } + + + public int getPullMessageThreadPoolNums() { + return pullMessageThreadPoolNums; + } + + + public void setPullMessageThreadPoolNums(int pullMessageThreadPoolNums) { + this.pullMessageThreadPoolNums = pullMessageThreadPoolNums; + } + + + public int getAdminBrokerThreadPoolNums() { + return adminBrokerThreadPoolNums; + } + + + public void setAdminBrokerThreadPoolNums(int adminBrokerThreadPoolNums) { + this.adminBrokerThreadPoolNums = adminBrokerThreadPoolNums; + } + + + public int getFlushConsumerOffsetInterval() { + return flushConsumerOffsetInterval; + } + + + public void setFlushConsumerOffsetInterval(int flushConsumerOffsetInterval) { + this.flushConsumerOffsetInterval = flushConsumerOffsetInterval; + } + + + public int getFlushConsumerOffsetHistoryInterval() { + return flushConsumerOffsetHistoryInterval; + } + + + public void setFlushConsumerOffsetHistoryInterval(int flushConsumerOffsetHistoryInterval) { + this.flushConsumerOffsetHistoryInterval = flushConsumerOffsetHistoryInterval; + } + + + public boolean isClusterTopicEnable() { + return clusterTopicEnable; + } + + + public void setClusterTopicEnable(boolean clusterTopicEnable) { + this.clusterTopicEnable = clusterTopicEnable; + } + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public boolean isAutoCreateSubscriptionGroup() { + return autoCreateSubscriptionGroup; + } + + + public void setAutoCreateSubscriptionGroup(boolean autoCreateSubscriptionGroup) { + this.autoCreateSubscriptionGroup = autoCreateSubscriptionGroup; + } + + + public boolean isRejectTransactionMessage() { + return rejectTransactionMessage; + } + + + public void setRejectTransactionMessage(boolean rejectTransactionMessage) { + this.rejectTransactionMessage = rejectTransactionMessage; + } + + + public boolean isFetchNamesrvAddrByAddressServer() { + return fetchNamesrvAddrByAddressServer; + } + + + public void setFetchNamesrvAddrByAddressServer(boolean fetchNamesrvAddrByAddressServer) { + this.fetchNamesrvAddrByAddressServer = fetchNamesrvAddrByAddressServer; + } + + + public int getSendThreadPoolQueueCapacity() { + return sendThreadPoolQueueCapacity; + } + + + public void setSendThreadPoolQueueCapacity(int sendThreadPoolQueueCapacity) { + this.sendThreadPoolQueueCapacity = sendThreadPoolQueueCapacity; + } + + + public int getPullThreadPoolQueueCapacity() { + return pullThreadPoolQueueCapacity; + } + + + public void setPullThreadPoolQueueCapacity(int pullThreadPoolQueueCapacity) { + this.pullThreadPoolQueueCapacity = pullThreadPoolQueueCapacity; + } + + + public boolean isBrokerTopicEnable() { + return brokerTopicEnable; + } + + + public void setBrokerTopicEnable(boolean brokerTopicEnable) { + this.brokerTopicEnable = brokerTopicEnable; + } + + + public int getFilterServerNums() { + return filterServerNums; + } + + + public void setFilterServerNums(int filterServerNums) { + this.filterServerNums = filterServerNums; + } + + + public boolean isLongPollingEnable() { + return longPollingEnable; + } + + + public void setLongPollingEnable(boolean longPollingEnable) { + this.longPollingEnable = longPollingEnable; + } + + + public boolean isNotifyConsumerIdsChangedEnable() { + return notifyConsumerIdsChangedEnable; + } + + + public void setNotifyConsumerIdsChangedEnable(boolean notifyConsumerIdsChangedEnable) { + this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable; + } + + + public long getShortPollingTimeMills() { + return shortPollingTimeMills; + } + + + public void setShortPollingTimeMills(long shortPollingTimeMills) { + this.shortPollingTimeMills = shortPollingTimeMills; + } + + + public int getClientManageThreadPoolNums() { + return clientManageThreadPoolNums; + } + + + public void setClientManageThreadPoolNums(int clientManageThreadPoolNums) { + this.clientManageThreadPoolNums = clientManageThreadPoolNums; + } + + + public boolean isCommercialEnable() { + return commercialEnable; + } + + + public void setCommercialEnable(final boolean commercialEnable) { + this.commercialEnable = commercialEnable; + } + + public int getCommercialTimerCount() { + return commercialTimerCount; + } + + public void setCommercialTimerCount(final int commercialTimerCount) { + this.commercialTimerCount = commercialTimerCount; + } + + public int getCommercialTransCount() { + return commercialTransCount; + } + + public void setCommercialTransCount(final int commercialTransCount) { + this.commercialTransCount = commercialTransCount; + } + + public int getCommercialBigCount() { + return commercialBigCount; + } + + public void setCommercialBigCount(final int commercialBigCount) { + this.commercialBigCount = commercialBigCount; + } + + public int getMaxDelayTime() { + return maxDelayTime; + } + + + public void setMaxDelayTime(final int maxDelayTime) { + this.maxDelayTime = maxDelayTime; + } + + public int getClientManagerThreadPoolQueueCapacity() { + return clientManagerThreadPoolQueueCapacity; + } + + public void setClientManagerThreadPoolQueueCapacity(int clientManagerThreadPoolQueueCapacity) { + this.clientManagerThreadPoolQueueCapacity = clientManagerThreadPoolQueueCapacity; + } + + public int getConsumerManagerThreadPoolQueueCapacity() { + return consumerManagerThreadPoolQueueCapacity; + } + + public void setConsumerManagerThreadPoolQueueCapacity(int consumerManagerThreadPoolQueueCapacity) { + this.consumerManagerThreadPoolQueueCapacity = consumerManagerThreadPoolQueueCapacity; + } + + public int getConsumerManageThreadPoolNums() { + return consumerManageThreadPoolNums; + } + + public void setConsumerManageThreadPoolNums(int consumerManageThreadPoolNums) { + this.consumerManageThreadPoolNums = consumerManageThreadPoolNums; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfigSingleton.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfigSingleton.java new file mode 100644 index 0000000000..f1bc4534d4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfigSingleton.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class BrokerConfigSingleton { + private static AtomicBoolean isInit = new AtomicBoolean(); + private static BrokerConfig brokerConfig; + + public static BrokerConfig getBrokerConfig() { + if (brokerConfig == null) { + throw new IllegalArgumentException("brokerConfig Cannot be null !"); + } + return brokerConfig; + } + + public static void setBrokerConfig(BrokerConfig brokerConfig) { + if (!isInit.compareAndSet(false, true)) { + throw new IllegalArgumentException("broker config have inited !"); + } + BrokerConfigSingleton.brokerConfig = brokerConfig; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java new file mode 100644 index 0000000000..3191509496 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + + +/** + * @author shijia.wxr + */ +public abstract class ConfigManager { + private static final Logger PLOG = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + + public abstract String encode(); + + public boolean load() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + + if (null == jsonString || jsonString.length() == 0) { + return this.loadBak(); + } else { + this.decode(jsonString); + PLOG.info("load {} OK", fileName); + return true; + } + } catch (Exception e) { + PLOG.error("load " + fileName + " Failed, and try to load backup file", e); + return this.loadBak(); + } + } + + public abstract String configFilePath(); + + private boolean loadBak() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName + ".bak"); + if (jsonString != null && jsonString.length() > 0) { + this.decode(jsonString); + PLOG.info("load " + fileName + " OK"); + return true; + } + } catch (Exception e) { + PLOG.error("load " + fileName + " Failed", e); + return false; + } + + return true; + } + + public abstract void decode(final String jsonString); + + public synchronized void persist() { + String jsonString = this.encode(true); + if (jsonString != null) { + String fileName = this.configFilePath(); + try { + MixAll.string2File(jsonString, fileName); + } catch (IOException e) { + PLOG.error("persist file Exception, " + fileName, e); + } + } + } + + public abstract String encode(final boolean prettyFormat); +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Configuration.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Configuration.java new file mode 100644 index 0000000000..8b69c1f623 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Configuration.java @@ -0,0 +1,310 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import org.slf4j.Logger; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * @author xigu.lx + */ +public class Configuration { + + private final Logger log; + + private List configObjectList = new ArrayList(4); + private String storePath; + private boolean storePathFromConfig = false; + private Object storePathObject; + private Field storePathField; + private DataVersion dataVersion = new DataVersion(); + private ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + /** + * All properties include configs in object and extend properties. + */ + private Properties allConfigs = new Properties(); + + public Configuration(Logger log) { + this.log = log; + } + + public Configuration(Logger log, Object... configObjects) { + this.log = log; + if (configObjects == null || configObjects.length == 0) { + return; + } + for (Object configObject : configObjects) { + registerConfig(configObject); + } + } + + public Configuration(Logger log, String storePath, Object... configObjects) { + this(log, configObjects); + this.storePath = storePath; + } + + /** + * register config object + * + * @param configObject + * @return the current Configuration object + */ + public Configuration registerConfig(Object configObject) { + try { + readWriteLock.writeLock().lockInterruptibly(); + + try { + + Properties registerProps = MixAll.object2Properties(configObject); + + merge(registerProps, this.allConfigs); + + configObjectList.add(configObject); + } finally { + readWriteLock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("registerConfig lock error"); + } + return this; + } + + /** + * register config properties + * + * @param extProperties + * @return the current Configuration object + */ + public Configuration registerConfig(Properties extProperties) { + if (extProperties == null) { + return this; + } + + try { + readWriteLock.writeLock().lockInterruptibly(); + + try { + merge(extProperties, this.allConfigs); + } finally { + readWriteLock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("register lock error. {}" + extProperties); + } + + return this; + } + + /** + * The store path will be gotten from the field of object. + * + * @param object + * @param fieldName + * + * @throws java.lang.RuntimeException if the field of object is not exist. + */ + public void setStorePathFromConfig(Object object, String fieldName) { + assert object != null; + + try { + readWriteLock.writeLock().lockInterruptibly(); + + try { + this.storePathFromConfig = true; + this.storePathObject = object; + // check + this.storePathField = object.getClass().getDeclaredField(fieldName); + assert this.storePathField != null + && !Modifier.isStatic(this.storePathField.getModifiers()); + this.storePathField.setAccessible(true); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } finally { + readWriteLock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("setStorePathFromConfig lock error"); + } + } + + private String getStorePath() { + String realStorePath = null; + try { + readWriteLock.readLock().lockInterruptibly(); + + try { + realStorePath = this.storePath; + + if (this.storePathFromConfig) { + try { + realStorePath = (String) storePathField.get(this.storePathObject); + } catch (IllegalAccessException e) { + log.error("getStorePath error, ", e); + } + } + } finally { + readWriteLock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getStorePath lock error"); + } + + return realStorePath; + } + + public void update(Properties properties) { + try { + readWriteLock.writeLock().lockInterruptibly(); + + try { + // the property must be exist when update + mergeIfExist(properties, this.allConfigs); + + for (Object configObject : configObjectList) { + // not allConfigs to update... + MixAll.properties2Object(properties, configObject); + } + + this.dataVersion.nextVersion(); + + } finally { + readWriteLock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("update lock error, {}", properties); + return; + } + + persist(); + } + + public void persist() { + try { + readWriteLock.readLock().lockInterruptibly(); + + try { + String allConfigs = getAllConfigsInternal(); + + MixAll.string2File(allConfigs, getStorePath()); + } catch (IOException e) { + log.error("persist string2File error, ", e); + } finally { + readWriteLock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("persist lock error"); + } + } + + public String getAllConfigsFormatString() { + try { + readWriteLock.readLock().lockInterruptibly(); + + try { + + return getAllConfigsInternal(); + + } finally { + readWriteLock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getAllConfigsFormatString lock error"); + } + + return null; + } + + public String getDataVersionJson() { + return this.dataVersion.toJson(); + } + + public Properties getAllConfigs() { + try { + readWriteLock.readLock().lockInterruptibly(); + + try { + + return this.allConfigs; + + } finally { + readWriteLock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getAllConfigs lock error"); + } + + return null; + } + + private String getAllConfigsInternal() { + StringBuilder stringBuilder = new StringBuilder(); + + // reload from config object ? + for (Object configObject : this.configObjectList) { + Properties properties = MixAll.object2Properties(configObject); + if (properties != null) { + merge(properties, this.allConfigs); + } else { + log.warn("getAllConfigsInternal object2Properties is null, {}", configObject.getClass()); + } + } + + { + stringBuilder.append(MixAll.properties2String(this.allConfigs)); + } + + return stringBuilder.toString(); + } + + public void setStorePath(final String storePath) { + this.storePath = storePath; + } + + private void merge(Properties from, Properties to) { + for (Object key : from.keySet()) { + Object fromObj = from.get(key), toObj = to.get(key); + if (toObj != null && !toObj.equals(fromObj)) { + log.info("Replace, key: {}, value: {} -> {}", key, toObj, fromObj); + } + to.put(key, fromObj); + } + } + + private void mergeIfExist(Properties from, Properties to) { + for (Object key : from.keySet()) { + if (!to.containsKey(key)) { + continue; + } + + Object fromObj = from.get(key), toObj = to.get(key); + if (toObj != null && !toObj.equals(fromObj)) { + log.info("Replace, key: {}, value: {} -> {}", key, toObj, fromObj); + } + to.put(key, fromObj); + } + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/CountDownLatch.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/CountDownLatch.java new file mode 100644 index 0000000000..a5cc9a14cd --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/CountDownLatch.java @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; + +/** + * Add reset feature for @see java.util.concurrent.CountDownLatch + * + * @author xinyuzhou.zxy + */ +public class CountDownLatch { + /** + * Synchronization control For CountDownLatch. + * Uses AQS state to represent count. + */ + private static final class Sync extends AbstractQueuedSynchronizer { + private static final long serialVersionUID = 4982264981922014374L; + + private final int startCount; + + Sync(int count) { + this.startCount = count; + setState(count); + } + + int getCount() { + return getState(); + } + + protected int tryAcquireShared(int acquires) { + return (getState() == 0) ? 1 : -1; + } + + protected boolean tryReleaseShared(int releases) { + // Decrement count; signal when transition to zero + for (;;) { + int c = getState(); + if (c == 0) + return false; + int nextc = c - 1; + if (compareAndSetState(c, nextc)) + return nextc == 0; + } + } + + protected void reset() { + setState(startCount); + } + } + + private final Sync sync; + + /** + * Constructs a {@code CountDownLatch} initialized with the given count. + * + * @param count + * the number of times {@link #countDown} must be invoked + * before threads can pass through {@link #await} + * + * @throws IllegalArgumentException + * if {@code count} is negative + */ + public CountDownLatch(int count) { + if (count < 0) throw new IllegalArgumentException("count < 0"); + this.sync = new Sync(count); + } + + /** + * Causes the current thread to wait until the latch has counted down to + * zero, unless the thread is {@linkplain Thread#interrupt interrupted}. + * + *

If the current count is zero then this method returns immediately. + * + *

If the current count is greater than zero then the current + * thread becomes disabled for thread scheduling purposes and lies + * dormant until one of two things happen: + *

    + *
  • The count reaches zero due to invocations of the + * {@link #countDown} method; or + *
  • Some other thread {@linkplain Thread#interrupt interrupts} + * the current thread. + *
+ * + *

If the current thread: + *

    + *
  • has its interrupted status set on entry to this method; or + *
  • is {@linkplain Thread#interrupt interrupted} while waiting, + *
+ * then {@link InterruptedException} is thrown and the current thread's + * interrupted status is cleared. + * + * @throws InterruptedException + * if the current thread is interrupted + * while waiting + */ + public void await() throws InterruptedException { + sync.acquireSharedInterruptibly(1); + } + + /** + * Causes the current thread to wait until the latch has counted down to + * zero, unless the thread is {@linkplain Thread#interrupt interrupted}, + * or the specified waiting time elapses. + * + *

If the current count is zero then this method returns immediately + * with the value {@code true}. + * + *

If the current count is greater than zero then the current + * thread becomes disabled for thread scheduling purposes and lies + * dormant until one of three things happen: + *

    + *
  • The count reaches zero due to invocations of the + * {@link #countDown} method; or + *
  • Some other thread {@linkplain Thread#interrupt interrupts} + * the current thread; or + *
  • The specified waiting time elapses. + *
+ * + *

If the count reaches zero then the method returns with the + * value {@code true}. + * + *

If the current thread: + *

    + *
  • has its interrupted status set on entry to this method; or + *
  • is {@linkplain Thread#interrupt interrupted} while waiting, + *
+ * then {@link InterruptedException} is thrown and the current thread's + * interrupted status is cleared. + * + *

If the specified waiting time elapses then the value {@code false} + * is returned. If the time is less than or equal to zero, the method + * will not wait at all. + * + * @param timeout + * the maximum time to wait + * @param unit + * the time unit of the {@code timeout} argument + * + * @return {@code true} if the count reached zero and {@code false} + * if the waiting time elapsed before the count reached zero + * + * @throws InterruptedException + * if the current thread is interrupted + * while waiting + */ + public boolean await(long timeout, TimeUnit unit) + throws InterruptedException { + return sync.tryAcquireSharedNanos(1, unit.toNanos(timeout)); + } + + /** + * Decrements the count of the latch, releasing all waiting threads if + * the count reaches zero. + * + *

If the current count is greater than zero then it is decremented. + * If the new count is zero then all waiting threads are re-enabled for + * thread scheduling purposes. + * + *

If the current count equals zero then nothing happens. + */ + public void countDown() { + sync.releaseShared(1); + } + + /** + * Returns the current count. + * + *

This method is typically used for debugging and testing purposes. + * + * @return the current count + */ + public long getCount() { + return sync.getCount(); + } + + public void reset() { + sync.reset(); + } + + /** + * Returns a string identifying this latch, as well as its state. + * The state, in brackets, includes the String {@code "Count ="} + * followed by the current count. + * + * @return a string identifying this latch, as well as its state + */ + public String toString() { + return super.toString() + "[Count = " + sync.getCount() + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java new file mode 100644 index 0000000000..eb78ba16a3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public class DataVersion extends RemotingSerializable { + private long timestatmp = System.currentTimeMillis(); + private AtomicLong counter = new AtomicLong(0); + + + public void assignNewOne(final DataVersion dataVersion) { + this.timestatmp = dataVersion.timestatmp; + this.counter.set(dataVersion.counter.get()); + } + + + public void nextVersion() { + this.timestatmp = System.currentTimeMillis(); + this.counter.incrementAndGet(); + } + + + public long getTimestatmp() { + return timestatmp; + } + + + public void setTimestatmp(long timestatmp) { + this.timestatmp = timestatmp; + } + + + public AtomicLong getCounter() { + return counter; + } + + + public void setCounter(AtomicLong counter) { + this.counter = counter; + } + + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final DataVersion that = (DataVersion) o; + + if (timestatmp != that.timestatmp) return false; + return counter != null ? counter.equals(that.counter) : that.counter == null; + + } + + @Override + public int hashCode() { + int result = (int) (timestatmp ^ (timestatmp >>> 32)); + result = 31 * result + (counter != null ? counter.hashCode() : 0); + return result; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java new file mode 100644 index 0000000000..19afb09255 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java @@ -0,0 +1,362 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author shijia.wxr + */ +public class MQVersion { + + public static final int CURRENT_VERSION = Version.V4_0_0_SNAPSHOT.ordinal(); + + + public static String getVersionDesc(int value) { + try { + Version v = Version.values()[value]; + return v.name(); + } catch (Exception e) { + } + + return "HigherVersion"; + } + + + public static Version value2Version(int value) { + return Version.values()[value]; + } + + public enum Version { + V3_0_0_SNAPSHOT, + V3_0_0_ALPHA1, + V3_0_0_BETA1, + V3_0_0_BETA2, + V3_0_0_BETA3, + V3_0_0_BETA4, + V3_0_0_BETA5, + V3_0_0_BETA6_SNAPSHOT, + V3_0_0_BETA6, + V3_0_0_BETA7_SNAPSHOT, + V3_0_0_BETA7, + V3_0_0_BETA8_SNAPSHOT, + V3_0_0_BETA8, + V3_0_0_BETA9_SNAPSHOT, + V3_0_0_BETA9, + V3_0_0_FINAL, + V3_0_1_SNAPSHOT, + V3_0_1, + V3_0_2_SNAPSHOT, + V3_0_2, + V3_0_3_SNAPSHOT, + V3_0_3, + V3_0_4_SNAPSHOT, + V3_0_4, + V3_0_5_SNAPSHOT, + V3_0_5, + V3_0_6_SNAPSHOT, + V3_0_6, + V3_0_7_SNAPSHOT, + V3_0_7, + V3_0_8_SNAPSHOT, + V3_0_8, + V3_0_9_SNAPSHOT, + V3_0_9, + + V3_0_10_SNAPSHOT, + V3_0_10, + + V3_0_11_SNAPSHOT, + V3_0_11, + + V3_0_12_SNAPSHOT, + V3_0_12, + + V3_0_13_SNAPSHOT, + V3_0_13, + + V3_0_14_SNAPSHOT, + V3_0_14, + + V3_0_15_SNAPSHOT, + V3_0_15, + + V3_1_0_SNAPSHOT, + V3_1_0, + + V3_1_1_SNAPSHOT, + V3_1_1, + + V3_1_2_SNAPSHOT, + V3_1_2, + + V3_1_3_SNAPSHOT, + V3_1_3, + + V3_1_4_SNAPSHOT, + V3_1_4, + + V3_1_5_SNAPSHOT, + V3_1_5, + + V3_1_6_SNAPSHOT, + V3_1_6, + + V3_1_7_SNAPSHOT, + V3_1_7, + + V3_1_8_SNAPSHOT, + V3_1_8, + + V3_1_9_SNAPSHOT, + V3_1_9, + + V3_2_0_SNAPSHOT, + V3_2_0, + + V3_2_1_SNAPSHOT, + V3_2_1, + + V3_2_2_SNAPSHOT, + V3_2_2, + + V3_2_3_SNAPSHOT, + V3_2_3, + + V3_2_4_SNAPSHOT, + V3_2_4, + + V3_2_5_SNAPSHOT, + V3_2_5, + + V3_2_6_SNAPSHOT, + V3_2_6, + + V3_2_7_SNAPSHOT, + V3_2_7, + + V3_2_8_SNAPSHOT, + V3_2_8, + + V3_2_9_SNAPSHOT, + V3_2_9, + + V3_3_1_SNAPSHOT, + V3_3_1, + + V3_3_2_SNAPSHOT, + V3_3_2, + + V3_3_3_SNAPSHOT, + V3_3_3, + + V3_3_4_SNAPSHOT, + V3_3_4, + + V3_3_5_SNAPSHOT, + V3_3_5, + + V3_3_6_SNAPSHOT, + V3_3_6, + + V3_3_7_SNAPSHOT, + V3_3_7, + + V3_3_8_SNAPSHOT, + V3_3_8, + + V3_3_9_SNAPSHOT, + V3_3_9, + + V3_4_1_SNAPSHOT, + V3_4_1, + + V3_4_2_SNAPSHOT, + V3_4_2, + + V3_4_3_SNAPSHOT, + V3_4_3, + + V3_4_4_SNAPSHOT, + V3_4_4, + + V3_4_5_SNAPSHOT, + V3_4_5, + + V3_4_6_SNAPSHOT, + V3_4_6, + + V3_4_7_SNAPSHOT, + V3_4_7, + + V3_4_8_SNAPSHOT, + V3_4_8, + + V3_4_9_SNAPSHOT, + V3_4_9, + V3_5_1_SNAPSHOT, + V3_5_1, + + V3_5_2_SNAPSHOT, + V3_5_2, + + V3_5_3_SNAPSHOT, + V3_5_3, + + V3_5_4_SNAPSHOT, + V3_5_4, + + V3_5_5_SNAPSHOT, + V3_5_5, + + V3_5_6_SNAPSHOT, + V3_5_6, + + V3_5_7_SNAPSHOT, + V3_5_7, + + V3_5_8_SNAPSHOT, + V3_5_8, + + V3_5_9_SNAPSHOT, + V3_5_9, + + V3_6_1_SNAPSHOT, + V3_6_1, + + V3_6_2_SNAPSHOT, + V3_6_2, + + V3_6_3_SNAPSHOT, + V3_6_3, + + V3_6_4_SNAPSHOT, + V3_6_4, + + V3_6_5_SNAPSHOT, + V3_6_5, + + V3_6_6_SNAPSHOT, + V3_6_6, + + V3_6_7_SNAPSHOT, + V3_6_7, + + V3_6_8_SNAPSHOT, + V3_6_8, + + V3_6_9_SNAPSHOT, + V3_6_9, + + V3_7_1_SNAPSHOT, + V3_7_1, + + V3_7_2_SNAPSHOT, + V3_7_2, + + V3_7_3_SNAPSHOT, + V3_7_3, + + V3_7_4_SNAPSHOT, + V3_7_4, + + V3_7_5_SNAPSHOT, + V3_7_5, + + V3_7_6_SNAPSHOT, + V3_7_6, + + V3_7_7_SNAPSHOT, + V3_7_7, + + V3_7_8_SNAPSHOT, + V3_7_8, + + V3_7_9_SNAPSHOT, + V3_7_9, + + V3_8_1_SNAPSHOT, + V3_8_1, + + V3_8_2_SNAPSHOT, + V3_8_2, + + V3_8_3_SNAPSHOT, + V3_8_3, + + V3_8_4_SNAPSHOT, + V3_8_4, + + V3_8_5_SNAPSHOT, + V3_8_5, + + V3_8_6_SNAPSHOT, + V3_8_6, + + V3_8_7_SNAPSHOT, + V3_8_7, + + V3_8_8_SNAPSHOT, + V3_8_8, + + V3_8_9_SNAPSHOT, + V3_8_9, + + V3_9_1_SNAPSHOT, + V3_9_1, + + V3_9_2_SNAPSHOT, + V3_9_2, + + V3_9_3_SNAPSHOT, + V3_9_3, + + V3_9_4_SNAPSHOT, + V3_9_4, + + V3_9_5_SNAPSHOT, + V3_9_5, + + V3_9_6_SNAPSHOT, + V3_9_6, + + V3_9_7_SNAPSHOT, + V3_9_7, + + V3_9_8_SNAPSHOT, + V3_9_8, + + V3_9_9_SNAPSHOT, + V3_9_9, + + V4_0_0_SNAPSHOT, + V4_0_0, + + V4_1_0_SNAPSHOT, + V4_1_0, + + V4_2_0_SNAPSHOT, + V4_2_0, + + V4_3_0_SNAPSHOT, + V4_3_0, + + V4_4_0_SNAPSHOT, + V4_4_0, + + V4_5_0_SNAPSHOT, + V4_5_0, + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java new file mode 100644 index 0000000000..508111cabe --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java @@ -0,0 +1,486 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.common.help.FAQUrl; +import org.slf4j.Logger; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public class MixAll { + public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; + public static final String ROCKETMQ_HOME_PROPERTY = "rocketmq.home.dir"; + public static final String NAMESRV_ADDR_ENV = "NAMESRV_ADDR"; + public static final String NAMESRV_ADDR_PROPERTY = "rocketmq.namesrv.addr"; + public static final String MESSAGE_COMPRESS_LEVEL = "rocketmq.message.compressLevel"; + public static final String WS_DOMAIN_NAME = System.getProperty("rocketmq.namesrv.domain", "jmenv.tbsite.net"); + public static final String WS_DOMAIN_SUBGROUP = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); + // http://jmenv.tbsite.net:8080/rocketmq/nsaddr + public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP; + public static final String DEFAULT_TOPIC = "TBW102"; + public static final String BENCHMARK_TOPIC = "BenchmarkTest"; + public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER"; + public static final String DEFAULT_CONSUMER_GROUP = "DEFAULT_CONSUMER"; + public static final String TOOLS_CONSUMER_GROUP = "TOOLS_CONSUMER"; + public static final String FILTERSRV_CONSUMER_GROUP = "FILTERSRV_CONSUMER"; + public static final String MONITOR_CONSUMER_GROUP = "__MONITOR_CONSUMER"; + public static final String CLIENT_INNER_PRODUCER_GROUP = "CLIENT_INNER_PRODUCER"; + public static final String SELF_TEST_PRODUCER_GROUP = "SELF_TEST_P_GROUP"; + public static final String SELF_TEST_CONSUMER_GROUP = "SELF_TEST_C_GROUP"; + public static final String SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; + public static final String OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; + public static final String ONS_HTTP_PROXY_GROUP = "CID_ONS-HTTP-PROXY"; + public static final String CID_ONSAPI_PERMISSION_GROUP = "CID_ONSAPI_PERMISSION"; + public static final String CID_ONSAPI_OWNER_GROUP = "CID_ONSAPI_OWNER"; + public static final String CID_ONSAPI_PULL_GROUP = "CID_ONSAPI_PULL"; + public static final String CID_RMQ_SYS_PREFIX = "CID_RMQ_SYS_"; + + public static final List LOCAL_INET_ADDRESS = getLocalInetAddress(); + public static final String LOCALHOST = localhost(); + public static final String DEFAULT_CHARSET = "UTF-8"; + public static final long MASTER_ID = 0L; + public static final long CURRENT_JVM_PID = getPID(); + + public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; + + public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; + public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; + public static final String UNIQUE_MSG_QUERY_FLAG = "_UNIQUE_KEY_QUERY"; + public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; + public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; + + public static String getRetryTopic(final String consumerGroup) { + return RETRY_GROUP_TOPIC_PREFIX + consumerGroup; + } + + + public static boolean isSysConsumerGroup(final String consumerGroup) { + return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX); + } + + public static boolean isSystemTopic(final String topic) { + return topic.startsWith(SYSTEM_TOPIC_PREFIX); + } + + public static String getDLQTopic(final String consumerGroup) { + return DLQ_GROUP_TOPIC_PREFIX + consumerGroup; + } + + + public static String brokerVIPChannel(final boolean isChange, final String brokerAddr) { + if (isChange) { + String[] ipAndPort = brokerAddr.split(":"); + String brokerAddrNew = ipAndPort[0] + ":" + (Integer.parseInt(ipAndPort[1]) - 2); + return brokerAddrNew; + } else { + return brokerAddr; + } + } + + + public static long getPID() { + String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + if (processName != null && processName.length() > 0) { + try { + return Long.parseLong(processName.split("@")[0]); + } catch (Exception e) { + return 0; + } + } + + return 0; + } + + + public static long createBrokerId(final String ip, final int port) { + InetSocketAddress isa = new InetSocketAddress(ip, port); + byte[] ipArray = isa.getAddress().getAddress(); + ByteBuffer bb = ByteBuffer.allocate(8); + bb.put(ipArray); + bb.putInt(port); + long value = bb.getLong(0); + return Math.abs(value); + } + + public static final void string2File(final String str, final String fileName) throws IOException { + + String tmpFile = fileName + ".tmp"; + string2FileNotSafe(str, tmpFile); + + + String bakFile = fileName + ".bak"; + String prevContent = file2String(fileName); + if (prevContent != null) { + string2FileNotSafe(prevContent, bakFile); + } + + + File file = new File(fileName); + file.delete(); + + + file = new File(tmpFile); + file.renameTo(new File(fileName)); + } + + + public static final void string2FileNotSafe(final String str, final String fileName) throws IOException { + File file = new File(fileName); + File fileParent = file.getParentFile(); + if (fileParent != null) { + fileParent.mkdirs(); + } + FileWriter fileWriter = null; + + try { + fileWriter = new FileWriter(file); + fileWriter.write(str); + } catch (IOException e) { + throw e; + } finally { + if (fileWriter != null) { + try { + fileWriter.close(); + } catch (IOException e) { + throw e; + } + } + } + } + + + public static final String file2String(final String fileName) { + File file = new File(fileName); + return file2String(file); + } + + public static final String file2String(final File file) { + if (file.exists()) { + char[] data = new char[(int) file.length()]; + boolean result = false; + + FileReader fileReader = null; + try { + fileReader = new FileReader(file); + int len = fileReader.read(data); + result = len == data.length; + } catch (IOException e) { + // e.printStackTrace(); + } finally { + if (fileReader != null) { + try { + fileReader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + if (result) { + return new String(data); + } + } + return null; + } + + public static final String file2String(final URL url) { + InputStream in = null; + try { + URLConnection urlConnection = url.openConnection(); + urlConnection.setUseCaches(false); + in = urlConnection.getInputStream(); + int len = in.available(); + byte[] data = new byte[len]; + in.read(data, 0, len); + return new String(data, "UTF-8"); + } catch (Exception e) { + } finally { + if (null != in) { + try { + in.close(); + } catch (IOException e) { + } + } + } + + return null; + } + + public static String findClassPath(Class c) { + URL url = c.getProtectionDomain().getCodeSource().getLocation(); + return url.getPath(); + } + + + public static void printObjectProperties(final Logger log, final Object object) { + printObjectProperties(log, object, false); + } + + + public static void printObjectProperties(final Logger log, final Object object, final boolean onlyImportantField) { + Field[] fields = object.getClass().getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(object); + if (null == value) { + value = ""; + } + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + if (onlyImportantField) { + Annotation annotation = field.getAnnotation(ImportantField.class); + if (null == annotation) { + continue; + } + } + + if (log != null) { + log.info(name + "=" + value); + } else { + } + } + } + } + } + + + public static String properties2String(final Properties properties) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getValue() != null) { + sb.append(entry.getKey().toString() + "=" + entry.getValue().toString() + "\n"); + } + } + return sb.toString(); + } + + public static Properties string2Properties(final String str) { + Properties properties = new Properties(); + try { + InputStream in = new ByteArrayInputStream(str.getBytes(DEFAULT_CHARSET)); + properties.load(in); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return null; + } catch (IOException e) { + e.printStackTrace(); + return null; + } + + return properties; + } + + public static Properties object2Properties(final Object object) { + Properties properties = new Properties(); + + Field[] fields = object.getClass().getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(object); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + if (value != null) { + properties.setProperty(name, value.toString()); + } + } + } + } + + return properties; + } + + public static void properties2Object(final Properties p, final Object object) { + Method[] methods = object.getClass().getMethods(); + for (Method method : methods) { + String mn = method.getName(); + if (mn.startsWith("set")) { + try { + String tmp = mn.substring(4); + String first = mn.substring(3, 4); + + String key = first.toLowerCase() + tmp; + String property = p.getProperty(key); + if (property != null) { + Class[] pt = method.getParameterTypes(); + if (pt != null && pt.length > 0) { + String cn = pt[0].getSimpleName(); + Object arg = null; + if (cn.equals("int") || cn.equals("Integer")) { + arg = Integer.parseInt(property); + } else if (cn.equals("long") || cn.equals("Long")) { + arg = Long.parseLong(property); + } else if (cn.equals("double") || cn.equals("Double")) { + arg = Double.parseDouble(property); + } else if (cn.equals("boolean") || cn.equals("Boolean")) { + arg = Boolean.parseBoolean(property); + } else if (cn.equals("float") || cn.equals("Float")) { + arg = Float.parseFloat(property); + } else if (cn.equals("String")) { + arg = property; + } else { + continue; + } + method.invoke(object, new Object[]{arg}); + } + } + } catch (Throwable e) { + } + } + } + } + + + public static boolean isPropertiesEqual(final Properties p1, final Properties p2) { + return p1.equals(p2); + } + + + public static List getLocalInetAddress() { + List inetAddressList = new ArrayList(); + try { + Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); + while (enumeration.hasMoreElements()) { + NetworkInterface networkInterface = enumeration.nextElement(); + Enumeration addrs = networkInterface.getInetAddresses(); + while (addrs.hasMoreElements()) { + inetAddressList.add(addrs.nextElement().getHostAddress()); + } + } + } catch (SocketException e) { + throw new RuntimeException("get local inet address fail", e); + } + + return inetAddressList; + } + + + public static boolean isLocalAddr(String address) { + for (String addr : LOCAL_INET_ADDRESS) { + if (address.contains(addr)) + return true; + } + return false; + } + + + private static String localhost() { + try { + InetAddress addr = InetAddress.getLocalHost(); + return addr.getHostAddress(); + } catch (Throwable e) { + throw new RuntimeException("InetAddress java.net.InetAddress.getLocalHost() throws UnknownHostException" + + FAQUrl.suggestTodo(FAQUrl.UNKNOWN_HOST_EXCEPTION), + e); + } + } + + + public static boolean compareAndIncreaseOnly(final AtomicLong target, final long value) { + long prev = target.get(); + while (value > prev) { + boolean updated = target.compareAndSet(prev, value); + if (updated) + return true; + + prev = target.get(); + } + + return false; + } + + public static String localhostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (Throwable e) { + throw new RuntimeException("InetAddress java.net.InetAddress.getLocalHost() throws UnknownHostException" + + FAQUrl.suggestTodo(FAQUrl.UNKNOWN_HOST_EXCEPTION), + e); + } + } + + public Set list2Set(List values) { + Set result = new HashSet(); + for (String v : values) { + result.add(v); + } + return result; + } + + public List set2List(Set values) { + List result = new ArrayList(); + for (String v : values) { + result.add(v); + } + return result; + } + + public static String humanReadableByteCount(long bytes, boolean si) { + int unit = si ? 1000 : 1024; + if (bytes < unit) return bytes + " B"; + int exp = (int) (Math.log(bytes) / Math.log(unit)); + String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); + return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java new file mode 100644 index 0000000000..ada614492a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author shijia.wxr + */ +public class Pair { + private T1 object1; + private T2 object2; + + + public Pair(T1 object1, T2 object2) { + this.object1 = object1; + this.object2 = object2; + } + + + public T1 getObject1() { + return object1; + } + + + public void setObject1(T1 object1) { + this.object1 = object1; + } + + + public T2 getObject2() { + return object2; + } + + + public void setObject2(T2 object2) { + this.object2 = object2; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java new file mode 100644 index 0000000000..a580cf4147 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author shijia.wxr + */ +public enum ServiceState { + /** + * Service just created,not start + */ + CREATE_JUST, + /** + * Service Running + */ + RUNNING, + /** + * Service shutdown + */ + SHUTDOWN_ALREADY, + /** + * Service Start failure + */ + START_FAILED; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java new file mode 100644 index 0000000000..d6da0e361c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author shijia.wxr + * @author xinyuzhou.zxy + */ +public abstract class ServiceThread implements Runnable { + private static final Logger STLOG = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private static final long JOIN_TIME = 90 * 1000; + + protected final Thread thread; + + protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false); + + protected volatile boolean stopped = false; + + protected final CountDownLatch waitPoint = new CountDownLatch(1); + + + public ServiceThread() { + this.thread = new Thread(this, this.getServiceName()); + } + + + public abstract String getServiceName(); + + + public void start() { + this.thread.start(); + } + + + public void shutdown() { + this.shutdown(false); + } + + public void shutdown(final boolean interrupt) { + this.stopped = true; + STLOG.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt); + + if (hasNotified.compareAndSet(false, true)) { + waitPoint.countDown(); // notify + } + + try { + if (interrupt) { + this.thread.interrupt(); + } + + long beginTime = System.currentTimeMillis(); + if (!this.thread.isDaemon()) { + this.thread.join(this.getJointime()); + } + long eclipseTime = System.currentTimeMillis() - beginTime; + STLOG.info("join thread " + this.getServiceName() + " eclipse time(ms) " + eclipseTime + " " + + this.getJointime()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public long getJointime() { + return JOIN_TIME; + } + + public void stop() { + this.stop(false); + } + + public void stop(final boolean interrupt) { + this.stopped = true; + STLOG.info("stop thread " + this.getServiceName() + " interrupt " + interrupt); + + if (hasNotified.compareAndSet(false, true)) { + waitPoint.countDown(); // notify + } + + if (interrupt) { + this.thread.interrupt(); + } + } + + public void makeStop() { + this.stopped = true; + STLOG.info("makestop thread " + this.getServiceName()); + } + + public void wakeup() { + if (hasNotified.compareAndSet(false, true)) { + waitPoint.countDown(); // notify + } + } + + protected void waitForRunning(long interval) { + if (hasNotified.compareAndSet(true, false)) { + this.onWaitEnd(); + return; + } + + //entry to wait + waitPoint.reset(); + + try { + waitPoint.await(interval, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + hasNotified.set(false); + this.onWaitEnd(); + } + } + + protected void onWaitEnd() { + } + + public boolean isStopped() { + return stopped; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java new file mode 100644 index 0000000000..36c0448a7d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author vintage.wang + */ +public class SystemClock { + public long now() { + return System.currentTimeMillis(); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java new file mode 100644 index 0000000000..b4d85cd849 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; + + +public class ThreadFactoryImpl implements ThreadFactory { + private final AtomicLong threadIndex = new AtomicLong(0); + private final String threadNamePrefix; + + + public ThreadFactoryImpl(final String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + } + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet()); + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java new file mode 100644 index 0000000000..16019dfa6a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.constant.PermName; + + +/** + * @author shijia.wxr + */ +public class TopicConfig { + private static final String SEPARATOR = " "; + public static int defaultReadQueueNums = 16; + public static int defaultWriteQueueNums = 16; + private String topicName; + private int readQueueNums = defaultReadQueueNums; + private int writeQueueNums = defaultWriteQueueNums; + private int perm = PermName.PERM_READ | PermName.PERM_WRITE; + private TopicFilterType topicFilterType = TopicFilterType.SINGLE_TAG; + private int topicSysFlag = 0; + private boolean order = false; + + + public TopicConfig() { + } + + + public TopicConfig(String topicName) { + this.topicName = topicName; + } + + + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) { + this.topicName = topicName; + this.readQueueNums = readQueueNums; + this.writeQueueNums = writeQueueNums; + this.perm = perm; + } + + + public String encode() { + StringBuilder sb = new StringBuilder(); + + // 1 + sb.append(this.topicName); + sb.append(SEPARATOR); + + // 2 + sb.append(this.readQueueNums); + sb.append(SEPARATOR); + + // 3 + sb.append(this.writeQueueNums); + sb.append(SEPARATOR); + + // 4 + sb.append(this.perm); + sb.append(SEPARATOR); + + // 5 + sb.append(this.topicFilterType); + + return sb.toString(); + } + + + public boolean decode(final String in) { + String[] strs = in.split(SEPARATOR); + if (strs != null && strs.length == 5) { + this.topicName = strs[0]; + + this.readQueueNums = Integer.parseInt(strs[1]); + + this.writeQueueNums = Integer.parseInt(strs[2]); + + this.perm = Integer.parseInt(strs[3]); + + this.topicFilterType = TopicFilterType.valueOf(strs[4]); + + return true; + } + + return false; + } + + + public String getTopicName() { + return topicName; + } + + + public void setTopicName(String topicName) { + this.topicName = topicName; + } + + + public int getReadQueueNums() { + return readQueueNums; + } + + + public void setReadQueueNums(int readQueueNums) { + this.readQueueNums = readQueueNums; + } + + + public int getWriteQueueNums() { + return writeQueueNums; + } + + + public void setWriteQueueNums(int writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + + public int getPerm() { + return perm; + } + + + public void setPerm(int perm) { + this.perm = perm; + } + + + public TopicFilterType getTopicFilterType() { + return topicFilterType; + } + + + public void setTopicFilterType(TopicFilterType topicFilterType) { + this.topicFilterType = topicFilterType; + } + + + public int getTopicSysFlag() { + return topicSysFlag; + } + + + public void setTopicSysFlag(int topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + + public boolean isOrder() { + return order; + } + + + public void setOrder(boolean isOrder) { + this.order = isOrder; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final TopicConfig that = (TopicConfig) o; + + if (readQueueNums != that.readQueueNums) return false; + if (writeQueueNums != that.writeQueueNums) return false; + if (perm != that.perm) return false; + if (topicSysFlag != that.topicSysFlag) return false; + if (order != that.order) return false; + if (topicName != null ? !topicName.equals(that.topicName) : that.topicName != null) return false; + return topicFilterType == that.topicFilterType; + + } + + @Override + public int hashCode() { + int result = topicName != null ? topicName.hashCode() : 0; + result = 31 * result + readQueueNums; + result = 31 * result + writeQueueNums; + result = 31 * result + perm; + result = 31 * result + (topicFilterType != null ? topicFilterType.hashCode() : 0); + result = 31 * result + topicSysFlag; + result = 31 * result + (order ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "TopicConfig [topicName=" + topicName + ", readQueueNums=" + readQueueNums + + ", writeQueueNums=" + writeQueueNums + ", perm=" + PermName.perm2String(perm) + + ", topicFilterType=" + topicFilterType + ", topicSysFlag=" + topicSysFlag + ", order=" + + order + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java new file mode 100644 index 0000000000..7a20dc9e4d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author shijia.wxr + */ +public enum TopicFilterType { + SINGLE_TAG, + MULTI_TAG +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java new file mode 100644 index 0000000000..4429e3d410 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java @@ -0,0 +1,525 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.zip.CRC32; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; + + +/** + * @author shijia.wxr + */ +public class UtilAll { + public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; + public static final String YYYY_MM_DD_HH_MM_SS_SSS = "yyyy-MM-dd#HH:mm:ss:SSS"; + public static final String YYYY_MMDD_HHMMSS = "yyyyMMddHHmmss"; + + + public static int getPid() { + RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + String name = runtime.getName(); // format: "pid@hostname" + try { + return Integer.parseInt(name.substring(0, name.indexOf('@'))); + } catch (Exception e) { + return -1; + } + } + + public static String currentStackTrace() { + StringBuilder sb = new StringBuilder(); + StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); + for (StackTraceElement ste : stackTrace) { + sb.append("\n\t"); + sb.append(ste.toString()); + } + + return sb.toString(); + } + + public static String offset2FileName(final long offset) { + final NumberFormat nf = NumberFormat.getInstance(); + nf.setMinimumIntegerDigits(20); + nf.setMaximumFractionDigits(0); + nf.setGroupingUsed(false); + return nf.format(offset); + } + + public static long computeEclipseTimeMilliseconds(final long beginTime) { + return System.currentTimeMillis() - beginTime; + } + + + public static boolean isItTimeToDo(final String when) { + String[] whiles = when.split(";"); + if (whiles != null && whiles.length > 0) { + Calendar now = Calendar.getInstance(); + for (String w : whiles) { + int nowHour = Integer.parseInt(w); + if (nowHour == now.get(Calendar.HOUR_OF_DAY)) { + return true; + } + } + } + + return false; + } + + + public static String timeMillisToHumanString() { + return timeMillisToHumanString(System.currentTimeMillis()); + } + + + public static String timeMillisToHumanString(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d%02d%02d%02d%02d%02d%03d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, + cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY), cal.get(Calendar.MINUTE), cal.get(Calendar.SECOND), + cal.get(Calendar.MILLISECOND)); + } + + + public static long computNextMorningTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextMinutesTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 0); + cal.add(Calendar.MINUTE, 1); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextHourTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 1); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextHalfHourTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 1); + cal.set(Calendar.MINUTE, 30); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static String timeMillisToHumanString2(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d-%02d-%02d %02d:%02d:%02d,%03d", + cal.get(Calendar.YEAR), + cal.get(Calendar.MONTH) + 1, + cal.get(Calendar.DAY_OF_MONTH), + cal.get(Calendar.HOUR_OF_DAY), + cal.get(Calendar.MINUTE), + cal.get(Calendar.SECOND), + cal.get(Calendar.MILLISECOND)); + } + + + public static String timeMillisToHumanString3(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d%02d%02d%02d%02d%02d", + cal.get(Calendar.YEAR), + cal.get(Calendar.MONTH) + 1, + cal.get(Calendar.DAY_OF_MONTH), + cal.get(Calendar.HOUR_OF_DAY), + cal.get(Calendar.MINUTE), + cal.get(Calendar.SECOND)); + } + + + public static double getDiskPartitionSpaceUsedPercent(final String path) { + if (null == path || path.isEmpty()) + return -1; + + try { + File file = new File(path); + if (!file.exists()) { + boolean result = file.mkdirs(); + if (!result) { + } + } + + long totalSpace = file.getTotalSpace(); + long freeSpace = file.getFreeSpace(); + long usedSpace = totalSpace - freeSpace; + if (totalSpace > 0) { + return usedSpace / (double) totalSpace; + } + } catch (Exception e) { + return -1; + } + + return -1; + } + + + public static final int crc32(byte[] array) { + if (array != null) { + return crc32(array, 0, array.length); + } + + return 0; + } + + + public static final int crc32(byte[] array, int offset, int length) { + CRC32 crc32 = new CRC32(); + crc32.update(array, offset, length); + return (int) (crc32.getValue() & 0x7FFFFFFF); + } + + final static char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray(); + + public static String bytes2string(byte[] src) { + char[] hexChars = new char[src.length * 2]; + for (int j = 0; j < src.length; j++) { + int v = src[j] & 0xFF; + hexChars[j * 2] = HEX_ARRAY[v >>> 4]; + hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F]; + } + return new String(hexChars); + } + + public static byte[] string2bytes(String hexString) { + if (hexString == null || hexString.equals("")) { + return null; + } + hexString = hexString.toUpperCase(); + int length = hexString.length() / 2; + char[] hexChars = hexString.toCharArray(); + byte[] d = new byte[length]; + for (int i = 0; i < length; i++) { + int pos = i * 2; + d[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1])); + } + return d; + } + + + private static byte charToByte(char c) { + return (byte) "0123456789ABCDEF".indexOf(c); + } + + + public static byte[] uncompress(final byte[] src) throws IOException { + byte[] result = src; + byte[] uncompressData = new byte[src.length]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src); + InflaterInputStream inflaterInputStream = new InflaterInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length); + + try { + while (true) { + int len = inflaterInputStream.read(uncompressData, 0, uncompressData.length); + if (len <= 0) { + break; + } + byteArrayOutputStream.write(uncompressData, 0, len); + } + byteArrayOutputStream.flush(); + result = byteArrayOutputStream.toByteArray(); + } catch (IOException e) { + throw e; + } finally { + try { + byteArrayInputStream.close(); + } catch (IOException e) { + } + try { + inflaterInputStream.close(); + } catch (IOException e) { + } + try { + byteArrayOutputStream.close(); + } catch (IOException e) { + } + } + + return result; + } + + + public static byte[] compress(final byte[] src, final int level) throws IOException { + byte[] result = src; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length); + java.util.zip.Deflater defeater = new java.util.zip.Deflater(level); + DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(byteArrayOutputStream, defeater); + try { + deflaterOutputStream.write(src); + deflaterOutputStream.finish(); + deflaterOutputStream.close(); + result = byteArrayOutputStream.toByteArray(); + } catch (IOException e) { + defeater.end(); + throw e; + } finally { + try { + byteArrayOutputStream.close(); + } catch (IOException ignored) { + } + + defeater.end(); + } + + return result; + } + + + public static int asInt(String str, int defaultValue) { + try { + return Integer.parseInt(str); + } catch (Exception e) { + return defaultValue; + } + } + + + public static long asLong(String str, long defaultValue) { + try { + return Long.parseLong(str); + } catch (Exception e) { + return defaultValue; + } + } + + + public static String formatDate(Date date, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + return df.format(date); + } + + + public static Date parseDate(String date, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(date); + } catch (ParseException e) { + return null; + } + } + + + public static String responseCode2String(final int code) { + return Integer.toString(code); + } + + + public static String frontStringAtLeast(final String str, final int size) { + if (str != null) { + if (str.length() > size) { + return str.substring(0, size); + } + } + + return str; + } + + + public static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + return true; + } + + + public static String jstack() { + return jstack(Thread.getAllStackTraces()); + } + + + public static String jstack(Map map) { + StringBuilder result = new StringBuilder(); + try { + Iterator> ite = map.entrySet().iterator(); + while (ite.hasNext()) { + Map.Entry entry = ite.next(); + StackTraceElement[] elements = entry.getValue(); + Thread thread = entry.getKey(); + if (elements != null && elements.length > 0) { + String threadName = entry.getKey().getName(); + result.append(String.format("%-40sTID: %d STATE: %s%n", threadName, thread.getId(), thread.getState())); + for (StackTraceElement el : elements) { + result.append(String.format("%-40s%s%n", threadName, el.toString())); + } + result.append("\n"); + } + } + } catch (Throwable e) { + result.append(RemotingHelper.exceptionSimpleDesc(e)); + } + + return result.toString(); + } + + public static boolean isInternalIP(byte[] ip) { + if (ip.length != 4) { + throw new RuntimeException("illegal ipv4 bytes"); + } + + + //10.0.0.0~10.255.255.255 + //172.16.0.0~172.31.255.255 + //192.168.0.0~192.168.255.255 + if (ip[0] == (byte) 10) { + + return true; + } else if (ip[0] == (byte) 172) { + if (ip[1] >= (byte) 16 && ip[1] <= (byte) 31) { + return true; + } + } else if (ip[0] == (byte) 192) { + if (ip[1] == (byte) 168) { + return true; + } + } + return false; + } + + private static boolean ipCheck(byte[] ip) { + if (ip.length != 4) { + throw new RuntimeException("illegal ipv4 bytes"); + } + +// if (ip[0] == (byte)30 && ip[1] == (byte)10 && ip[2] == (byte)163 && ip[3] == (byte)120) { +// } + + + if (ip[0] >= (byte) 1 && ip[0] <= (byte) 126) { + if (ip[1] == (byte) 1 && ip[2] == (byte) 1 && ip[3] == (byte) 1) { + return false; + } + if (ip[1] == (byte) 0 && ip[2] == (byte) 0 && ip[3] == (byte) 0) { + return false; + } + return true; + } else if (ip[0] >= (byte) 128 && ip[0] <= (byte) 191) { + if (ip[2] == (byte) 1 && ip[3] == (byte) 1) { + return false; + } + if (ip[2] == (byte) 0 && ip[3] == (byte) 0) { + return false; + } + return true; + } else if (ip[0] >= (byte) 192 && ip[0] <= (byte) 223) { + if (ip[3] == (byte) 1) { + return false; + } + if (ip[3] == (byte) 0) { + return false; + } + return true; + } + return false; + } + + public static String ipToIPv4Str(byte[] ip) { + if (ip.length != 4) { + return null; + } + return new StringBuilder().append(ip[0] & 0xFF).append(".").append( + ip[1] & 0xFF).append(".").append(ip[2] & 0xFF) + .append(".").append(ip[3] & 0xFF).toString(); + } + + public static byte[] getIP() { + try { + Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces(); + InetAddress ip = null; + byte[] internalIP = null; + while (allNetInterfaces.hasMoreElements()) { + NetworkInterface netInterface = (NetworkInterface) allNetInterfaces.nextElement(); + Enumeration addresses = netInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + ip = (InetAddress) addresses.nextElement(); + if (ip != null && ip instanceof Inet4Address) { + byte[] ipByte = ip.getAddress(); + if (ipByte.length == 4) { + if (ipCheck(ipByte)) { + if (!isInternalIP(ipByte)) { + return ipByte; + } else if (internalIP == null) { + internalIP = ipByte; + } + } + } + } + } + } + if (internalIP != null) { + return internalIP; + } else { + throw new RuntimeException("Can not get local ip"); + } + } catch (Exception e) { + throw new RuntimeException("Can not get local ip", e); + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java new file mode 100644 index 0000000000..d8c931115b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; + + +/** + * + * @author shijia.wxr + * + */ +public class ConsumeStats extends RemotingSerializable { + private HashMap offsetTable = new HashMap(); + private double consumeTps = 0; + + + public long computeTotalDiff() { + long diffTotal = 0L; + + Iterator> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long diff = next.getValue().getBrokerOffset() - next.getValue().getConsumerOffset(); + diffTotal += diff; + } + + return diffTotal; + } + + + public HashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(HashMap offsetTable) { + this.offsetTable = offsetTable; + } + + public double getConsumeTps() { + return consumeTps; + } + + public void setConsumeTps(double consumeTps) { + this.consumeTps = consumeTps; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java new file mode 100644 index 0000000000..07785c2683 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +/** + * + * @author shijia.wxr + * + */ +public class OffsetWrapper { + private long brokerOffset; + private long consumerOffset; + + private long lastTimestamp; + + + public long getBrokerOffset() { + return brokerOffset; + } + + + public void setBrokerOffset(long brokerOffset) { + this.brokerOffset = brokerOffset; + } + + + public long getConsumerOffset() { + return consumerOffset; + } + + + public void setConsumerOffset(long consumerOffset) { + this.consumerOffset = consumerOffset; + } + + + public long getLastTimestamp() { + return lastTimestamp; + } + + + public void setLastTimestamp(long lastTimestamp) { + this.lastTimestamp = lastTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java new file mode 100644 index 0000000000..03d94a2529 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.admin; + +/** + * + * @author manhong.yqd + */ +public class RollbackStats { + private String brokerName; + private long queueId; + private long brokerOffset; + private long consumerOffset; + private long timestampOffset; + private long rollbackOffset; + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public long getQueueId() { + return queueId; + } + + + public void setQueueId(long queueId) { + this.queueId = queueId; + } + + + public long getBrokerOffset() { + return brokerOffset; + } + + + public void setBrokerOffset(long brokerOffset) { + this.brokerOffset = brokerOffset; + } + + + public long getConsumerOffset() { + return consumerOffset; + } + + + public void setConsumerOffset(long consumerOffset) { + this.consumerOffset = consumerOffset; + } + + + public long getTimestampOffset() { + return timestampOffset; + } + + + public void setTimestampOffset(long timestampOffset) { + this.timestampOffset = timestampOffset; + } + + + public long getRollbackOffset() { + return rollbackOffset; + } + + + public void setRollbackOffset(long rollbackOffset) { + this.rollbackOffset = rollbackOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java new file mode 100644 index 0000000000..076d6ebb09 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +/** + * + * @author shijia.wxr + * + */ +public class TopicOffset { + private long minOffset; + private long maxOffset; + private long lastUpdateTimestamp; + + + public long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(long minOffset) { + this.minOffset = minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java new file mode 100644 index 0000000000..12d1d4b8c8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; + + +/** + * + * @author shijia.wxr + * + */ +public class TopicStatsTable extends RemotingSerializable { + private HashMap offsetTable = new HashMap(); + + + public HashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(HashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java new file mode 100644 index 0000000000..fe0cb12b6c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) +public @interface ImportantField { +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/DBMsgConstants.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/DBMsgConstants.java new file mode 100644 index 0000000000..54bc04df81 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/DBMsgConstants.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.constant; + +public class DBMsgConstants { + public static final int MAX_BODY_SIZE = 64 * 1024 * 1204; //64KB +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java new file mode 100644 index 0000000000..917566903f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.constant; + +/** + * @author shijia.wxr + */ +public class LoggerName { + public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; + public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; + public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; + public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; + public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; + public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; + public static final String STORE_LOGGER_NAME = "RocketmqStore"; + public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; + public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; + public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; + public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; + public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; + public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; + public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; + public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; + public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; + public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java new file mode 100644 index 0000000000..95c2510934 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.constant; + +/** + * @author shijia.wxr + */ +public class PermName { + public static final int PERM_PRIORITY = 0x1 << 3; + public static final int PERM_READ = 0x1 << 2; + public static final int PERM_WRITE = 0x1 << 1; + public static final int PERM_INHERIT = 0x1 << 0; + + public static String perm2String(final int perm) { + final StringBuffer sb = new StringBuffer("---"); + if (isReadable(perm)) { + sb.replace(0, 1, "R"); + } + + if (isWriteable(perm)) { + sb.replace(1, 2, "W"); + } + + if (isInherited(perm)) { + sb.replace(2, 3, "X"); + } + + return sb.toString(); + } + + public static boolean isReadable(final int perm) { + return (perm & PERM_READ) == PERM_READ; + } + + public static boolean isWriteable(final int perm) { + return (perm & PERM_WRITE) == PERM_WRITE; + } + + public static boolean isInherited(final int perm) { + return (perm & PERM_INHERIT) == PERM_INHERIT; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java new file mode 100644 index 0000000000..ededc90687 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.consumer; + +/** + * + * @author shijia.wxr + */ +public enum ConsumeFromWhere { + CONSUME_FROM_LAST_OFFSET, + + @Deprecated + CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST, + @Deprecated + CONSUME_FROM_MIN_OFFSET, + @Deprecated + CONSUME_FROM_MAX_OFFSET, + CONSUME_FROM_FIRST_OFFSET, + CONSUME_FROM_TIMESTAMP, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java new file mode 100644 index 0000000000..2b26b83fd8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.filter; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + +import java.net.URL; + + +/** + * @author shijia.wxr + * + */ +public class FilterAPI { + public static URL classFile(final String className) { + final String javaSource = simpleClassName(className) + ".java"; + URL url = FilterAPI.class.getClassLoader().getResource(javaSource); + return url; + } + + public static String simpleClassName(final String className) { + String simple = className; + int index = className.lastIndexOf("."); + if (index >= 0) { + simple = className.substring(index + 1); + } + + return simple; + } + + public static SubscriptionData buildSubscriptionData(final String consumerGroup, String topic, + String subString) throws Exception { + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTopic(topic); + subscriptionData.setSubString(subString); + + if (null == subString || subString.equals(SubscriptionData.SUB_ALL) || subString.length() == 0) { + subscriptionData.setSubString(SubscriptionData.SUB_ALL); + } else { + String[] tags = subString.split("\\|\\|"); + if (tags != null && tags.length > 0) { + for (String tag : tags) { + if (tag.length() > 0) { + String trimString = tag.trim(); + if (trimString.length() > 0) { + subscriptionData.getTagsSet().add(trimString); + subscriptionData.getCodeSet().add(trimString.hashCode()); + } + } + } + } else { + throw new Exception("subString split error"); + } + } + + return subscriptionData; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterContext.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterContext.java new file mode 100644 index 0000000000..50cc5fc0b3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterContext.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter; + +public class FilterContext { + private String consumerGroup; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java new file mode 100644 index 0000000000..8a1252effe --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +public interface MessageFilter { + boolean match(final MessageExt msg, final FilterContext context); +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java new file mode 100644 index 0000000000..f83a5f583b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter.impl; + +public abstract class Op { + + private String symbol; + + + protected Op(String symbol) { + this.symbol = symbol; + } + + + public String getSymbol() { + return symbol; + } + + + public String toString() { + return symbol; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java new file mode 100644 index 0000000000..95ca663964 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter.impl; + +public class Operand extends Op { + + public Operand(String symbol) { + super(symbol); + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java new file mode 100644 index 0000000000..c906d72fa0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter.impl; + +public class Operator extends Op { + + public static final Operator LEFTPARENTHESIS = new Operator("(", 30, false); + public static final Operator RIGHTPARENTHESIS = new Operator(")", 30, false); + public static final Operator AND = new Operator("&&", 20, true); + public static final Operator OR = new Operator("||", 15, true); + + private int priority; + private boolean compareable; + + + private Operator(String symbol, int priority, boolean compareable) { + super(symbol); + this.priority = priority; + this.compareable = compareable; + } + + public static Operator createOperator(String operator) { + if (LEFTPARENTHESIS.getSymbol().equals(operator)) + return LEFTPARENTHESIS; + else if (RIGHTPARENTHESIS.getSymbol().equals(operator)) + return RIGHTPARENTHESIS; + else if (AND.getSymbol().equals(operator)) + return AND; + else if (OR.getSymbol().equals(operator)) + return OR; + else + throw new IllegalArgumentException("unsupport operator " + operator); + } + + public int getPriority() { + return priority; + } + + public boolean isCompareable() { + return compareable; + } + + + public int compare(Operator operator) { + if (this.priority > operator.priority) + return 1; + else if (this.priority == operator.priority) + return 0; + else + return -1; + } + + public boolean isSpecifiedOp(String operator) { + return this.getSymbol().equals(operator); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java new file mode 100644 index 0000000000..518c45e479 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter.impl; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +import static com.alibaba.rocketmq.common.filter.impl.Operator.LEFTPARENTHESIS; +import static com.alibaba.rocketmq.common.filter.impl.Operator.RIGHTPARENTHESIS; +import static com.alibaba.rocketmq.common.filter.impl.Operator.createOperator; + +public class PolishExpr { + + public static List reversePolish(String expression) { + return reversePolish(participle(expression)); + } + + /** + * Shunting-yard algorithm
+ * http://en.wikipedia.org/wiki/Shunting_yard_algorithm + * + * @param tokens + * @return the compute result of Shunting-yard algorithm + */ + public static List reversePolish(List tokens) { + List segments = new ArrayList(); + Stack operatorStack = new Stack(); + + for (int i = 0; i < tokens.size(); i++) { + Op token = tokens.get(i); + if (isOperand(token)) { + + segments.add(token); + } else if (isLeftParenthesis(token)) { + + operatorStack.push((Operator) token); + } else if (isRightParenthesis(token)) { + + Operator opNew = null; + while (!operatorStack.empty() && LEFTPARENTHESIS != (opNew = operatorStack.pop())) { + segments.add(opNew); + } + if (null == opNew || LEFTPARENTHESIS != opNew) + throw new IllegalArgumentException("mismatched parentheses"); + } else if (isOperator(token)) { + + Operator opNew = (Operator) token; + if (!operatorStack.empty()) { + Operator opOld = operatorStack.peek(); + if (opOld.isCompareable() && opNew.compare(opOld) != 1) { + segments.add(operatorStack.pop()); + } + } + operatorStack.push(opNew); + } else + throw new IllegalArgumentException("illegal token " + token); + } + + while (!operatorStack.empty()) { + Operator operator = operatorStack.pop(); + if (LEFTPARENTHESIS == operator || RIGHTPARENTHESIS == operator) + throw new IllegalArgumentException("mismatched parentheses " + operator); + segments.add(operator); + } + + return segments; + } + + /** + * + * @param expression + * + * @return + * + * @throws Exception + */ + private static List participle(String expression) { + List segments = new ArrayList(); + + int size = expression.length(); + int wordStartIndex = -1; + int wordLen = 0; + Type preType = Type.NULL; + + for (int i = 0; i < size; i++) { + int chValue = (int) expression.charAt(i); + + if ((97 <= chValue && chValue <= 122) || (65 <= chValue && chValue <= 90) + || (49 <= chValue && chValue <= 57) || 95 == chValue) { + + + if (Type.OPERATOR == preType || Type.SEPAERATOR == preType || Type.NULL == preType + || Type.PARENTHESIS == preType) { + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression.substring(wordStartIndex, wordStartIndex + + wordLen))); + } + wordStartIndex = i; + wordLen = 0; + } + preType = Type.OPERAND; + wordLen++; + } else if (40 == chValue || 41 == chValue) { + + + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression + .substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } else if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + + preType = Type.PARENTHESIS; + segments.add(createOperator((char) chValue + "")); + } else if (38 == chValue || 124 == chValue) { + + if (Type.OPERAND == preType || Type.SEPAERATOR == preType || Type.PARENTHESIS == preType) { + if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + + wordLen))); + } + wordStartIndex = i; + wordLen = 0; + } + preType = Type.OPERATOR; + wordLen++; + } else if (32 == chValue || 9 == chValue) { + + + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression + .substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } else if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + preType = Type.SEPAERATOR; + } else { + + throw new IllegalArgumentException("illegal expression, at index " + i + " " + (char) chValue); + } + + } + + if (wordLen > 0) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + } + return segments; + } + + public static boolean isOperand(Op token) { + return token instanceof Operand; + } + + public static boolean isLeftParenthesis(Op token) { + return token instanceof Operator && LEFTPARENTHESIS == (Operator) token; + } + + public static boolean isRightParenthesis(Op token) { + return token instanceof Operator && RIGHTPARENTHESIS == (Operator) token; + } + + public static boolean isOperator(Op token) { + return token instanceof Operator; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java new file mode 100644 index 0000000000..1c0b343777 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter.impl; + +public enum Type { + NULL, + OPERAND, + OPERATOR, + PARENTHESIS, + SEPAERATOR; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java new file mode 100644 index 0000000000..06a74a66ba --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.help; + +/** + * @author shijia.wxr + */ +public class FAQUrl { + + public static final String APPLY_TOPIC_URL = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&topic_not_exist"; + + + public static final String NAME_SERVER_ADDR_NOT_EXIST_URL = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&namesrv_not_exist"; + + + public static final String GROUP_NAME_DUPLICATE_URL = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&group_duplicate"; + + + public static final String CLIENT_PARAMETER_CHECK_URL = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions¶meter_check_failed"; + + + public static final String SUBSCRIPTION_GROUP_NOT_EXIST = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&subGroup_not_exist"; + + + public static final String CLIENT_SERVICE_NOT_OK = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&service_not_ok"; + + // FAQ: No route info of this topic, TopicABC + public static final String NO_TOPIC_ROUTE_INFO = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&topic_not_exist"; + + + public static final String LOAD_JSON_EXCEPTION = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&load_json_exception"; + + + public static final String SAME_GROUP_DIFFERENT_TOPIC = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&subscription_exception"; + + + public static final String MQLIST_NOT_EXIST = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&queue_not_exist"; + + public static final String UNEXPECTED_EXCEPTION_URL = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&unexpected_exception"; + + + public static final String SEND_MSG_FAILED = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&send_msg_failed"; + + + public static final String UNKNOWN_HOST_EXCEPTION = // + "http://docs.aliyun.com/cn#/pub/ons/faq/exceptions&unknown_host"; + + private static final String TIP_STRING_BEGIN = "\nSee "; + private static final String TIP_STRING_END = " for further details."; + + + public static String suggestTodo(final String url) { + StringBuilder sb = new StringBuilder(); + sb.append(TIP_STRING_BEGIN); + sb.append(url); + sb.append(TIP_STRING_END); + return sb.toString(); + } + + public static String attachDefaultURL(final String errorMessage) { + if (errorMessage != null) { + int index = errorMessage.indexOf(TIP_STRING_BEGIN); + if (-1 == index) { + StringBuilder sb = new StringBuilder(); + sb.append(errorMessage); + sb.append("\n"); + sb.append("For more information, please visit the url, "); + sb.append(UNEXPECTED_EXCEPTION_URL); + return sb.toString(); + } + } + + return errorMessage; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java new file mode 100644 index 0000000000..f5d9d7ec19 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.hook; + +import java.nio.ByteBuffer; + + +/** + * + * @author manhong.yqd + * + */ +public interface FilterCheckHook { + public String hookName(); + + + public boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java new file mode 100644 index 0000000000..eeb6f52dd9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.io.Serializable; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + + +/** + * + * @author shijia.wxr + * + */ +public class Message implements Serializable { + private static final long serialVersionUID = 8445773977080406428L; + + private String topic; + private int flag; + private Map properties; + private byte[] body; + + + public Message() { + } + + + public Message(String topic, byte[] body) { + this(topic, "", "", 0, body, true); + } + + + public Message(String topic, String tags, String keys, int flag, byte[] body, boolean waitStoreMsgOK) { + this.topic = topic; + this.flag = flag; + this.body = body; + + if (tags != null && tags.length() > 0) + this.setTags(tags); + + if (keys != null && keys.length() > 0) + this.setKeys(keys); + + this.setWaitStoreMsgOK(waitStoreMsgOK); + } + + public void setKeys(String keys) { + this.putProperty(MessageConst.PROPERTY_KEYS, keys); + } + + void putProperty(final String name, final String value) { + if (null == this.properties) { + this.properties = new HashMap(); + } + + this.properties.put(name, value); + } + + + public Message(String topic, String tags, byte[] body) { + this(topic, tags, "", 0, body, true); + } + + + public Message(String topic, String tags, String keys, byte[] body) { + this(topic, tags, keys, 0, body, true); + } + + void clearProperty(final String name) { + if (null != this.properties) { + this.properties.remove(name); + } + } + + public void putUserProperty(final String name, final String value) { + if (MessageConst.STRING_HASH_SET.contains(name)) { + throw new RuntimeException(String.format( + "The Property<%s> is used by system, input another please", name)); + } + this.putProperty(name, value); + } + + public String getUserProperty(final String name) { + return this.getProperty(name); + } + + public String getProperty(final String name) { + if (null == this.properties) { + this.properties = new HashMap(); + } + + return this.properties.get(name); + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getTags() { + return this.getProperty(MessageConst.PROPERTY_TAGS); + } + + public void setTags(String tags) { + this.putProperty(MessageConst.PROPERTY_TAGS, tags); + } + + public String getKeys() { + return this.getProperty(MessageConst.PROPERTY_KEYS); + } + + public void setKeys(Collection keys) { + StringBuffer sb = new StringBuffer(); + for (String k : keys) { + sb.append(k); + sb.append(MessageConst.KEY_SEPARATOR); + } + + this.setKeys(sb.toString().trim()); + } + + + public int getDelayTimeLevel() { + String t = this.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL); + if (t != null) { + return Integer.parseInt(t); + } + + return 0; + } + + + public void setDelayTimeLevel(int level) { + this.putProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL, String.valueOf(level)); + } + + + public boolean isWaitStoreMsgOK() { + String result = this.getProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); + if (null == result) + return true; + + return Boolean.parseBoolean(result); + } + + + public void setWaitStoreMsgOK(boolean waitStoreMsgOK) { + this.putProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, Boolean.toString(waitStoreMsgOK)); + } + + + public int getFlag() { + return flag; + } + + + public void setFlag(int flag) { + this.flag = flag; + } + + + public byte[] getBody() { + return body; + } + + + public void setBody(byte[] body) { + this.body = body; + } + + + public Map getProperties() { + return properties; + } + + + void setProperties(Map properties) { + this.properties = properties; + } + + public String getBuyerId() { + return getProperty(MessageConst.PROPERTY_BUYER_ID); + } + + public void setBuyerId(String buyerId) { + putProperty(MessageConst.PROPERTY_BUYER_ID, buyerId); + } + + @Override + public String toString() { + return "Message [topic=" + topic + ", flag=" + flag + ", properties=" + properties + ", body=" + + (body != null ? body.length : 0) + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java new file mode 100644 index 0000000000..bbbca1af2a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.message; + +import java.util.Map; + + +public class MessageAccessor { + + public static void clearProperty(final Message msg, final String name) { + msg.clearProperty(name); + } + + public static void setProperties(final Message msg, Map properties) { + msg.setProperties(properties); + } + + public static void setTransferFlag(final Message msg, String unit) { + putProperty(msg, MessageConst.PROPERTY_TRANSFER_FLAG, unit); + } + + public static void putProperty(final Message msg, final String name, final String value) { + msg.putProperty(name, value); + } + + public static String getTransferFlag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_TRANSFER_FLAG); + } + + + public static void setCorrectionFlag(final Message msg, String unit) { + putProperty(msg, MessageConst.PROPERTY_CORRECTION_FLAG, unit); + } + + + public static String getCorrectionFlag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_CORRECTION_FLAG); + } + + + public static void setOriginMessageId(final Message msg, String originMessageId) { + putProperty(msg, MessageConst.PROPERTY_ORIGIN_MESSAGE_ID, originMessageId); + } + + + public static String getOriginMessageId(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_ORIGIN_MESSAGE_ID); + } + + + public static void setMQ2Flag(final Message msg, String flag) { + putProperty(msg, MessageConst.PROPERTY_MQ2_FLAG, flag); + } + + + public static String getMQ2Flag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_MQ2_FLAG); + } + + + public static void setReconsumeTime(final Message msg, String reconsumeTimes) { + putProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME, reconsumeTimes); + } + + + public static String getReconsumeTime(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_RECONSUME_TIME); + } + + + public static void setMaxReconsumeTimes(final Message msg, String maxReconsumeTimes) { + putProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES, maxReconsumeTimes); + } + + + public static String getMaxReconsumeTimes(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_MAX_RECONSUME_TIMES); + } + + public static void setConsumeStartTimeStamp(final Message msg, String propertyConsumeStartTimeStamp) { + putProperty(msg, MessageConst.PROPERTY_CONSUME_START_TIMESTAMP, propertyConsumeStartTimeStamp); + } + + + public static String getConsumeStartTimeStamp(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_CONSUME_START_TIMESTAMP); + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientExt.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientExt.java new file mode 100644 index 0000000000..0ab372ec4d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientExt.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +public class MessageClientExt extends MessageExt { + + public void setOffsetMsgId(String offsetMsgId) { + super.setMsgId(offsetMsgId); + } + + + public String getOffsetMsgId() { + return super.getMsgId(); + } + + public void setMsgId(String msgId) { + //DO NOTHING + //MessageClientIDSetter.setUniqID(this); + } + + @Override + public String getMsgId() { + String uniqID = MessageClientIDSetter.getUniqID(this); + if (uniqID == null) { + return this.getOffsetMsgId(); + } + else { + return uniqID; + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientIDSetter.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientIDSetter.java new file mode 100644 index 0000000000..82cd3d1b54 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageClientIDSetter.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import com.alibaba.rocketmq.common.UtilAll; + +import java.nio.ByteBuffer; +import java.util.Calendar; +import java.util.Date; +import java.util.concurrent.atomic.AtomicInteger; + +public class MessageClientIDSetter { + private static final String TOPIC_KEY_SPLITTER = "#"; + private static final int LEN; + private static final String FIX_STRING; + private static final AtomicInteger COUNTER; + private static long startTime; + private static long nextStartTime; + + static { + LEN = 4 + 2 + 4 + 4 + 2; + ByteBuffer tempBuffer = ByteBuffer.allocate(10); + tempBuffer.position(2); + tempBuffer.putInt(UtilAll.getPid()); + tempBuffer.position(0); + try { + tempBuffer.put(UtilAll.getIP()); + } catch (Exception e) { + tempBuffer.put(createFakeIP()); + } + tempBuffer.position(6); + tempBuffer.putInt(MessageClientIDSetter.class.getClassLoader().hashCode()); //4 + FIX_STRING = UtilAll.bytes2string(tempBuffer.array()); + setStartTime(System.currentTimeMillis()); + COUNTER = new AtomicInteger(0); + } + + private synchronized static void setStartTime(long millis) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(millis); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + startTime = cal.getTimeInMillis(); + cal.add(Calendar.MONTH, 1); + nextStartTime = cal.getTimeInMillis(); + } + + public static Date getNearlyTimeFromID(String msgID) { + ByteBuffer buf = ByteBuffer.allocate(8); + byte[] bytes = UtilAll.string2bytes(msgID); + buf.put((byte) 0); + buf.put((byte) 0); + buf.put((byte) 0); + buf.put((byte) 0); + buf.put(bytes, 10, 4); + buf.position(0); + long spanMS = buf.getLong(); + Calendar cal = Calendar.getInstance(); + long now = cal.getTimeInMillis(); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + long monStartTime = cal.getTimeInMillis(); + if (monStartTime + spanMS >= now) { + cal.add(Calendar.MONTH, -1); + monStartTime = cal.getTimeInMillis(); + } + cal.setTimeInMillis(monStartTime + spanMS); + return cal.getTime(); + } + + public static String getIPStrFromID(String msgID) { + byte[] ipBytes = getIPFromID(msgID); + return UtilAll.ipToIPv4Str(ipBytes); + } + + public static byte[] getIPFromID(String msgID) { + byte[] result = new byte[4]; + byte[] bytes = UtilAll.string2bytes(msgID); + System.arraycopy(bytes, 0, result, 0, 4); + return result; + } + + public static String createUniqID() { + StringBuilder sb = new StringBuilder(LEN * 2); + sb.append(FIX_STRING); + sb.append(UtilAll.bytes2string(createUniqIDBuffer())); + return sb.toString(); + } + + + private static byte[] createUniqIDBuffer() { + ByteBuffer buffer = ByteBuffer.allocate(4 + 2); + long current = System.currentTimeMillis(); + if (current >= nextStartTime) { + setStartTime(current); + } + buffer.position(0); + buffer.putInt((int) (System.currentTimeMillis() - startTime)); + buffer.putShort((short) COUNTER.getAndIncrement()); + return buffer.array(); + } + + public static void setUniqID(final Message msg) { + if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { + msg.putProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, createUniqID()); + } + } + + public static String getUniqID(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + } + + public static byte[] createFakeIP() { + ByteBuffer bb = ByteBuffer.allocate(8); + bb.putLong(System.currentTimeMillis()); + bb.position(4); + byte[] fakeIP = new byte[4]; + bb.get(fakeIP); + return fakeIP; + } +} + diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java new file mode 100644 index 0000000000..fc06d6ed7d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.util.HashSet; + + +public class MessageConst { + public static final String PROPERTY_KEYS = "KEYS"; + public static final String PROPERTY_TAGS = "TAGS"; + public static final String PROPERTY_WAIT_STORE_MSG_OK = "WAIT"; + public static final String PROPERTY_DELAY_TIME_LEVEL = "DELAY"; + public static final String PROPERTY_RETRY_TOPIC = "RETRY_TOPIC"; + public static final String PROPERTY_REAL_TOPIC = "REAL_TOPIC"; + public static final String PROPERTY_REAL_QUEUE_ID = "REAL_QID"; + public static final String PROPERTY_TRANSACTION_PREPARED = "TRAN_MSG"; + public static final String PROPERTY_PRODUCER_GROUP = "PGROUP"; + public static final String PROPERTY_MIN_OFFSET = "MIN_OFFSET"; + public static final String PROPERTY_MAX_OFFSET = "MAX_OFFSET"; + public static final String PROPERTY_BUYER_ID = "BUYER_ID"; + public static final String PROPERTY_ORIGIN_MESSAGE_ID = "ORIGIN_MESSAGE_ID"; + public static final String PROPERTY_TRANSFER_FLAG = "TRANSFER_FLAG"; + public static final String PROPERTY_CORRECTION_FLAG = "CORRECTION_FLAG"; + public static final String PROPERTY_MQ2_FLAG = "MQ2_FLAG"; + public static final String PROPERTY_RECONSUME_TIME = "RECONSUME_TIME"; + public static final String PROPERTY_MSG_REGION = "MSG_REGION"; + public static final String PROPERTY_TRACE_SWITCH = "TRACE_ON"; + public static final String PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX = "UNIQ_KEY"; + public static final String PROPERTY_MAX_RECONSUME_TIMES = "MAX_RECONSUME_TIMES"; + public static final String PROPERTY_CONSUME_START_TIMESTAMP = "CONSUME_START_TIME"; + + public static final String KEY_SEPARATOR = " "; + + public static final HashSet STRING_HASH_SET = new HashSet(); + + + static { + STRING_HASH_SET.add(PROPERTY_TRACE_SWITCH); + STRING_HASH_SET.add(PROPERTY_MSG_REGION); + STRING_HASH_SET.add(PROPERTY_KEYS); + STRING_HASH_SET.add(PROPERTY_TAGS); + STRING_HASH_SET.add(PROPERTY_WAIT_STORE_MSG_OK); + STRING_HASH_SET.add(PROPERTY_DELAY_TIME_LEVEL); + STRING_HASH_SET.add(PROPERTY_RETRY_TOPIC); + STRING_HASH_SET.add(PROPERTY_REAL_TOPIC); + STRING_HASH_SET.add(PROPERTY_REAL_QUEUE_ID); + STRING_HASH_SET.add(PROPERTY_TRANSACTION_PREPARED); + STRING_HASH_SET.add(PROPERTY_PRODUCER_GROUP); + STRING_HASH_SET.add(PROPERTY_MIN_OFFSET); + STRING_HASH_SET.add(PROPERTY_MAX_OFFSET); + STRING_HASH_SET.add(PROPERTY_BUYER_ID); + STRING_HASH_SET.add(PROPERTY_ORIGIN_MESSAGE_ID); + STRING_HASH_SET.add(PROPERTY_TRANSFER_FLAG); + STRING_HASH_SET.add(PROPERTY_CORRECTION_FLAG); + STRING_HASH_SET.add(PROPERTY_MQ2_FLAG); + STRING_HASH_SET.add(PROPERTY_RECONSUME_TIME); + STRING_HASH_SET.add(PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES); + STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java new file mode 100644 index 0000000000..e21c1cac58 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java @@ -0,0 +1,395 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * @author shijia.wxr + */ +public class MessageDecoder { + public final static int MSG_ID_LENGTH = 8 + 8; + + public final static Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + public final static int MESSAGE_MAGIC_CODE_POSTION = 4; + public final static int MESSAGE_FLAG_POSTION = 16; + public final static int MESSAGE_PHYSIC_OFFSET_POSTION = 28; + public final static int MESSAGE_STORE_TIMESTAMP_POSTION = 56; + public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; + + + public static String createMessageId(final ByteBuffer input, final ByteBuffer addr, final long offset) { + input.flip(); + input.limit(MessageDecoder.MSG_ID_LENGTH); + + input.put(addr); + input.putLong(offset); + + return UtilAll.bytes2string(input.array()); + } + + + public static String createMessageId(SocketAddress socketAddress, long transactionIdhashCode) { + ByteBuffer byteBuffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + byteBuffer.put(inetSocketAddress.getAddress().getAddress()); + byteBuffer.putInt(inetSocketAddress.getPort()); + byteBuffer.putLong(transactionIdhashCode); + byteBuffer.flip(); + return UtilAll.bytes2string(byteBuffer.array()); + } + + + public static MessageId decodeMessageId(final String msgId) throws UnknownHostException { + SocketAddress address; + long offset; + + + byte[] ip = UtilAll.string2bytes(msgId.substring(0, 8)); + byte[] port = UtilAll.string2bytes(msgId.substring(8, 16)); + ByteBuffer bb = ByteBuffer.wrap(port); + int portInt = bb.getInt(0); + address = new InetSocketAddress(InetAddress.getByAddress(ip), portInt); + + // offset + byte[] data = UtilAll.string2bytes(msgId.substring(16, 32)); + bb = ByteBuffer.wrap(data); + offset = bb.getLong(0); + + return new MessageId(address, offset); + } + + + public static MessageExt decode(java.nio.ByteBuffer byteBuffer) { + return decode(byteBuffer, true, true, false); + } + + public static MessageExt clientDecode(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + return decode(byteBuffer, readBody, true, true); + } + + public static MessageExt decode(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + return decode(byteBuffer, readBody, true, false); + } + + + public static byte[] encode(MessageExt messageExt, boolean needCompress) throws Exception { + byte[] body = messageExt.getBody(); + byte[] topics = messageExt.getTopic().getBytes(CHARSET_UTF8); + byte topicLen = (byte) topics.length; + String properties = messageProperties2String(messageExt.getProperties()); + byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); + short propertiesLength = (short) propertiesBytes.length; + int sysFlag = messageExt.getSysFlag(); + byte[] newBody = messageExt.getBody(); + if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + newBody = UtilAll.compress(body, 5); + } + int bodyLength = newBody.length; + int storeSize = messageExt.getStoreSize(); + ByteBuffer byteBuffer; + if (storeSize > 0) { + byteBuffer = ByteBuffer.allocate(storeSize); + } else { + storeSize = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + 8 // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + 8 // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8 // 14 Prepared Transaction Offset + + 4 + bodyLength // 14 BODY + + 1 + topicLen // 15 TOPIC + + 2 + propertiesLength // 16 propertiesLength + + 0; + byteBuffer = ByteBuffer.allocate(storeSize); + } + // 1 TOTALSIZE + byteBuffer.putInt(storeSize); + + // 2 MAGICCODE + byteBuffer.putInt(MESSAGE_MAGIC_CODE); + + // 3 BODYCRC + int bodyCRC = messageExt.getBodyCRC(); + byteBuffer.putInt(bodyCRC); + + // 4 QUEUEID + int queueId = messageExt.getQueueId(); + byteBuffer.putInt(queueId); + + // 5 FLAG + int flag = messageExt.getFlag(); + byteBuffer.putInt(flag); + + // 6 QUEUEOFFSET + long queueOffset = messageExt.getQueueOffset(); + byteBuffer.putLong(queueOffset); + + // 7 PHYSICALOFFSET + long physicOffset = messageExt.getCommitLogOffset(); + byteBuffer.putLong(physicOffset); + + // 8 SYSFLAG + byteBuffer.putInt(sysFlag); + + // 9 BORNTIMESTAMP + long bornTimeStamp = messageExt.getBornTimestamp(); + byteBuffer.putLong(bornTimeStamp); + + // 10 BORNHOST + InetSocketAddress bornHost = (InetSocketAddress) messageExt.getBornHost(); + byteBuffer.put(bornHost.getAddress().getAddress()); + byteBuffer.putInt(bornHost.getPort()); + + // 11 STORETIMESTAMP + long storeTimestamp = messageExt.getStoreTimestamp(); + byteBuffer.putLong(storeTimestamp); + + // 12 STOREHOST + InetSocketAddress serverHost = (InetSocketAddress) messageExt.getStoreHost(); + byteBuffer.put(serverHost.getAddress().getAddress()); + byteBuffer.putInt(serverHost.getPort()); + + // 13 RECONSUMETIMES + int reconsumeTimes = messageExt.getReconsumeTimes(); + byteBuffer.putInt(reconsumeTimes); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = messageExt.getPreparedTransactionOffset(); + byteBuffer.putLong(preparedTransactionOffset); + + // 15 BODY + byteBuffer.putInt(bodyLength); + byteBuffer.put(newBody); + + // 16 TOPIC + byteBuffer.put(topicLen); + byteBuffer.put(topics); + + // 17 properties + byteBuffer.putShort(propertiesLength); + byteBuffer.put(propertiesBytes); + + return byteBuffer.array(); + } + + public static MessageExt decode( + java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody) { + return decode(byteBuffer, readBody, deCompressBody, false); + } + + public static MessageExt decode( + java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) { + try { + + MessageExt msgExt; + if (isClient) { + msgExt = new MessageClientExt(); + } else { + msgExt = new MessageExt(); + } + + // 1 TOTALSIZE + int storeSize = byteBuffer.getInt(); + msgExt.setStoreSize(storeSize); + + // 2 MAGICCODE + byteBuffer.getInt(); + + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + msgExt.setBodyCRC(bodyCRC); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + msgExt.setQueueId(queueId); + + // 5 FLAG + int flag = byteBuffer.getInt(); + msgExt.setFlag(flag); + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + msgExt.setQueueOffset(queueOffset); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + msgExt.setCommitLogOffset(physicOffset); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + msgExt.setSysFlag(sysFlag); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + msgExt.setBornTimestamp(bornTimeStamp); + + // 10 BORNHOST + byte[] bornHost = new byte[4]; + byteBuffer.get(bornHost, 0, 4); + int port = byteBuffer.getInt(); + msgExt.setBornHost(new InetSocketAddress(InetAddress.getByAddress(bornHost), port)); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + msgExt.setStoreTimestamp(storeTimestamp); + + // 12 STOREHOST + byte[] storeHost = new byte[4]; + byteBuffer.get(storeHost, 0, 4); + port = byteBuffer.getInt(); + msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByAddress(storeHost), port)); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + msgExt.setReconsumeTimes(reconsumeTimes); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + msgExt.setPreparedTransactionOffset(preparedTransactionOffset); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + if (readBody) { + byte[] body = new byte[bodyLen]; + byteBuffer.get(body); + + // uncompress body + if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + body = UtilAll.uncompress(body); + } + + msgExt.setBody(body); + } else { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byte[] topic = new byte[(int) topicLen]; + byteBuffer.get(topic); + msgExt.setTopic(new String(topic, CHARSET_UTF8)); + + // 17 properties + short propertiesLength = byteBuffer.getShort(); + if (propertiesLength > 0) { + byte[] properties = new byte[propertiesLength]; + byteBuffer.get(properties); + String propertiesString = new String(properties, CHARSET_UTF8); + Map map = string2messageProperties(propertiesString); + msgExt.setProperties(map); + } + + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH); + String msgId = createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset()); + msgExt.setMsgId(msgId); + + if (isClient) { + ((MessageClientExt) msgExt).setOffsetMsgId(msgId); + } + + return msgExt; + } catch (UnknownHostException e) { + byteBuffer.position(byteBuffer.limit()); + } catch (BufferUnderflowException e) { + byteBuffer.position(byteBuffer.limit()); + } catch (Exception e) { + byteBuffer.position(byteBuffer.limit()); + } + + return null; + } + + + public static List decodes(java.nio.ByteBuffer byteBuffer) { + return decodes(byteBuffer, true); + } + + public static List decodes(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + List msgExts = new ArrayList(); + while (byteBuffer.hasRemaining()) { + MessageExt msgExt = clientDecode(byteBuffer, readBody); + if (null != msgExt) { + msgExts.add(msgExt); + } else { + break; + } + } + return msgExts; + } + + public static final char NAME_VALUE_SEPARATOR = 1; + public static final char PROPERTY_SEPARATOR = 2; + + + public static String messageProperties2String(Map properties) { + StringBuilder sb = new StringBuilder(); + if (properties != null) { + for (final Map.Entry entry : properties.entrySet()) { + final String name = entry.getKey(); + final String value = entry.getValue(); + + sb.append(name); + sb.append(NAME_VALUE_SEPARATOR); + sb.append(value); + sb.append(PROPERTY_SEPARATOR); + } + } + return sb.toString(); + } + + public static Map string2messageProperties(final String properties) { + Map map = new HashMap(); + if (properties != null) { + String[] items = properties.split(String.valueOf(PROPERTY_SEPARATOR)); + if (items != null) { + for (String i : items) { + String[] nv = i.split(String.valueOf(NAME_VALUE_SEPARATOR)); + if (nv != null && 2 == nv.length) { + map.put(nv[0], nv[1]); + } + } + } + } + + return map; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java new file mode 100644 index 0000000000..627935df1b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; + + +/** + * @author shijia.wxr + */ +public class MessageExt extends Message { + private static final long serialVersionUID = 5720810158625748049L; + + private int queueId; + + private int storeSize; + + private long queueOffset; + private int sysFlag; + private long bornTimestamp; + private SocketAddress bornHost; + + private long storeTimestamp; + private SocketAddress storeHost; + private String msgId; + private long commitLogOffset; + private int bodyCRC; + private int reconsumeTimes; + + private long preparedTransactionOffset; + + + public MessageExt() { + } + + + public MessageExt(int queueId, long bornTimestamp, SocketAddress bornHost, long storeTimestamp, + SocketAddress storeHost, String msgId) { + this.queueId = queueId; + this.bornTimestamp = bornTimestamp; + this.bornHost = bornHost; + this.storeTimestamp = storeTimestamp; + this.storeHost = storeHost; + this.msgId = msgId; + } + + public static TopicFilterType parseTopicFilterType(final int sysFlag) { + if ((sysFlag & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG) { + return TopicFilterType.MULTI_TAG; + } + + return TopicFilterType.SINGLE_TAG; + } + + public ByteBuffer getBornHostBytes() { + return socketAddress2ByteBuffer(this.bornHost); + } + + public ByteBuffer getBornHostBytes(ByteBuffer byteBuffer) { + return socketAddress2ByteBuffer(this.bornHost, byteBuffer); + } + + private static ByteBuffer socketAddress2ByteBuffer(final SocketAddress socketAddress, final ByteBuffer byteBuffer) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + byteBuffer.put(inetSocketAddress.getAddress().getAddress(), 0, 4); + byteBuffer.putInt(inetSocketAddress.getPort()); + byteBuffer.flip(); + return byteBuffer; + } + + public static ByteBuffer socketAddress2ByteBuffer(SocketAddress socketAddress) { + ByteBuffer byteBuffer = ByteBuffer.allocate(8); + return socketAddress2ByteBuffer(socketAddress, byteBuffer); + } + + public ByteBuffer getStoreHostBytes() { + return socketAddress2ByteBuffer(this.storeHost); + } + + public ByteBuffer getStoreHostBytes(ByteBuffer byteBuffer) { + return socketAddress2ByteBuffer(this.storeHost, byteBuffer); + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + public long getBornTimestamp() { + return bornTimestamp; + } + + public void setBornTimestamp(long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + public SocketAddress getBornHost() { + return bornHost; + } + + public void setBornHost(SocketAddress bornHost) { + this.bornHost = bornHost; + } + + public String getBornHostString() { + if (this.bornHost != null) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) this.bornHost; + return inetSocketAddress.getAddress().getHostAddress(); + } + + return null; + } + + public String getBornHostNameString() { + if (this.bornHost != null) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) this.bornHost; + return inetSocketAddress.getAddress().getHostName(); + } + + return null; + } + + public long getStoreTimestamp() { + return storeTimestamp; + } + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } + + public SocketAddress getStoreHost() { + return storeHost; + } + + public void setStoreHost(SocketAddress storeHost) { + this.storeHost = storeHost; + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + public int getSysFlag() { + return sysFlag; + } + + public void setSysFlag(int sysFlag) { + this.sysFlag = sysFlag; + } + + public int getBodyCRC() { + return bodyCRC; + } + + public void setBodyCRC(int bodyCRC) { + this.bodyCRC = bodyCRC; + } + + public long getQueueOffset() { + return queueOffset; + } + + public void setQueueOffset(long queueOffset) { + this.queueOffset = queueOffset; + } + + public long getCommitLogOffset() { + return commitLogOffset; + } + + public void setCommitLogOffset(long physicOffset) { + this.commitLogOffset = physicOffset; + } + + public int getStoreSize() { + return storeSize; + } + + public void setStoreSize(int storeSize) { + this.storeSize = storeSize; + } + + public int getReconsumeTimes() { + return reconsumeTimes; + } + + + public void setReconsumeTimes(int reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + + public long getPreparedTransactionOffset() { + return preparedTransactionOffset; + } + + + public void setPreparedTransactionOffset(long preparedTransactionOffset) { + this.preparedTransactionOffset = preparedTransactionOffset; + } + + + @Override + public String toString() { + return "MessageExt [queueId=" + queueId + ", storeSize=" + storeSize + ", queueOffset=" + queueOffset + + ", sysFlag=" + sysFlag + ", bornTimestamp=" + bornTimestamp + ", bornHost=" + bornHost + + ", storeTimestamp=" + storeTimestamp + ", storeHost=" + storeHost + ", msgId=" + msgId + + ", commitLogOffset=" + commitLogOffset + ", bodyCRC=" + bodyCRC + ", reconsumeTimes=" + + reconsumeTimes + ", preparedTransactionOffset=" + preparedTransactionOffset + + ", toString()=" + super.toString() + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java new file mode 100644 index 0000000000..d08be863db --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.net.SocketAddress; + + +/** + * @author shijia.wxr + */ +public class MessageId { + private SocketAddress address; + private long offset; + + + public MessageId(SocketAddress address, long offset) { + this.address = address; + this.offset = offset; + } + + + public SocketAddress getAddress() { + return address; + } + + + public void setAddress(SocketAddress address) { + this.address = address; + } + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java new file mode 100644 index 0000000000..35d2827c33 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.io.Serializable; + + +/** + * @author shijia.wxr + */ +public class MessageQueue implements Comparable, Serializable { + private static final long serialVersionUID = 6191200464116433425L; + private String topic; + private String brokerName; + private int queueId; + + + public MessageQueue() { + + } + + + public MessageQueue(String topic, String brokerName, int queueId) { + this.topic = topic; + this.brokerName = brokerName; + this.queueId = queueId; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + result = prime * result + queueId; + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + MessageQueue other = (MessageQueue) obj; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } else if (!brokerName.equals(other.brokerName)) + return false; + if (queueId != other.queueId) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + return true; + } + + + @Override + public String toString() { + return "MessageQueue [topic=" + topic + ", brokerName=" + brokerName + ", queueId=" + queueId + "]"; + } + + + @Override + public int compareTo(MessageQueue o) { + { + int result = this.topic.compareTo(o.topic); + if (result != 0) { + return result; + } + } + + { + int result = this.brokerName.compareTo(o.brokerName); + if (result != 0) { + return result; + } + } + + return this.queueId - o.queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueueForC.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueueForC.java new file mode 100644 index 0000000000..a905af6952 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueueForC.java @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.message; + +import java.io.Serializable; + + +/** + * @author lansheng.zj + */ +public class MessageQueueForC implements Comparable, Serializable { + + private static final long serialVersionUID = 5320967846569962104L; + private String topic; + private String brokerName; + private int queueId; + private long offset; + + + public MessageQueueForC(String topic, String brokerName, int queueId, long offset) { + this.topic = topic; + this.brokerName = brokerName; + this.queueId = queueId; + this.offset = offset; + } + + + @Override + public int compareTo(MessageQueueForC o) { + int result = this.topic.compareTo(o.topic); + if (result != 0) { + return result; + } + result = this.brokerName.compareTo(o.brokerName); + if (result != 0) { + return result; + } + result = this.queueId - o.queueId; + if (result != 0) { + return result; + } + if ((this.offset - o.offset) > 0) { + return 1; + } else if ((this.offset - o.offset) == 0) { + return 0; + } else { + return -1; + } + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + result = prime * result + queueId; + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + MessageQueueForC other = (MessageQueueForC) obj; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } else if (!brokerName.equals(other.brokerName)) + return false; + if (queueId != other.queueId) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + + if (offset != other.offset) { + return false; + } + return true; + } + + + @Override + public String toString() { + return "MessageQueueForC [topic=" + topic + ", brokerName=" + brokerName + ", queueId=" + queueId + + ", offset=" + offset + "]"; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageType.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageType.java new file mode 100644 index 0000000000..164eb87fe7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageType.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.message; + +public enum MessageType { + Normal_Msg, + Trans_Msg_Half, + Trans_msg_Commit, + Delay_Msg, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java new file mode 100644 index 0000000000..08db35794c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: NamesrvConfig.java 1839 2013-05-16 02:12:02Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.namesrv; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + + +/** + * + * @author shijia.wxr + * @author lansheng.zj + */ +public class NamesrvConfig { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + private String kvConfigPath = System.getProperty("user.home") + File.separator + "namesrv" + File.separator + "kvConfig.json"; + private String configStorePath = System.getProperty("user.home") + File.separator + "namesrv" + File.separator + "namesrv.properties"; + private String productEnvName = "center"; + private boolean clusterTest = false; + private boolean orderMessageEnable = false; + + public boolean isOrderMessageEnable() { + return orderMessageEnable; + } + + public void setOrderMessageEnable(boolean orderMessageEnable) { + this.orderMessageEnable = orderMessageEnable; + } + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getKvConfigPath() { + return kvConfigPath; + } + + + public void setKvConfigPath(String kvConfigPath) { + this.kvConfigPath = kvConfigPath; + } + + + public String getProductEnvName() { + return productEnvName; + } + + + public void setProductEnvName(String productEnvName) { + this.productEnvName = productEnvName; + } + + + public boolean isClusterTest() { + return clusterTest; + } + + + public void setClusterTest(boolean clusterTest) { + this.clusterTest = clusterTest; + } + + public String getConfigStorePath() { + return configStorePath; + } + + public void setConfigStorePath(final String configStorePath) { + this.configStorePath = configStorePath; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java new file mode 100644 index 0000000000..fcc32d995b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.namesrv; + +/** + * @author shijia.wxr + */ +public class NamesrvUtil { + public static final String NAMESPACE_ORDER_TOPIC_CONFIG = "ORDER_TOPIC_CONFIG"; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java new file mode 100644 index 0000000000..68bf44adad --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.namesrv; + +import com.alibaba.rocketmq.common.protocol.body.KVTable; + + +/** + * @author shijia.wxr + */ +public class RegisterBrokerResult { + private String haServerAddr; + private String masterAddr; + private KVTable kvTable; + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } + + + public KVTable getKvTable() { + return kvTable; + } + + + public void setKvTable(KVTable kvTable) { + this.kvTable = kvTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java new file mode 100644 index 0000000000..2e4ad87ac4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: TopAddressing.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.namesrv; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.utils.HttpTinyClient; +import com.alibaba.rocketmq.common.utils.HttpTinyClient.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + + +/** + * @author shijia.wxr + * @author manhong.yqd + */ +public class TopAddressing { + private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private String nsAddr; + private String wsAddr; + private String unitName; + + + public TopAddressing(final String wsAddr) { + this(wsAddr, null); + } + + + public TopAddressing(final String wsAddr, final String unitName) { + this.wsAddr = wsAddr; + this.unitName = unitName; + } + + public final String fetchNSAddr() { + return fetchNSAddr(true, 3000); + } + + public final String fetchNSAddr(boolean verbose, long timeoutMills) { + String url = this.wsAddr; + try { + if (!UtilAll.isBlank(this.unitName)) { + url = url + "-" + this.unitName + "?nofix=1"; + } + HttpResult result = HttpTinyClient.httpGet(url, null, null, "UTF-8", timeoutMills); + if (200 == result.code) { + String responseStr = result.content; + if (responseStr != null) { + return clearNewLine(responseStr); + } else { + log.error("fetch nameserver address is null"); + } + } else { + log.error("fetch nameserver address failed. statusCode={}", result.code); + } + } catch (IOException e) { + if (verbose) { + log.error("fetch name server address exception", e); + } + } + + if (verbose) { + String errorMsg = + "connect to " + url + " failed, maybe the domain name " + MixAll.WS_DOMAIN_NAME + " not bind in /etc/hosts"; + errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL); + + log.warn(errorMsg); + } + return null; + } + + private static String clearNewLine(final String str) { + String newString = str.trim(); + int index = newString.indexOf("\r"); + if (index != -1) { + return newString.substring(0, index); + } + + index = newString.indexOf("\n"); + if (index != -1) { + return newString.substring(0, index); + } + + return newString; + } + + public String getNsAddr() { + return nsAddr; + } + + + public void setNsAddr(String nsAddr) { + this.nsAddr = nsAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java new file mode 100644 index 0000000000..aaaa51da2f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol; + +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + */ +public class MQProtosHelper { + public static boolean registerBrokerToNameServer(final String nsaddr, final String brokerAddr, + final long timeoutMillis) { + RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); + + try { + RemotingCommand response = RemotingHelper.invokeSync(nsaddr, request, timeoutMillis); + if (response != null) { + return ResponseCode.SUCCESS == response.getCode(); + } + } catch (RemotingConnectException e) { + e.printStackTrace(); + } catch (RemotingSendRequestException e) { + e.printStackTrace(); + } catch (RemotingTimeoutException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + return false; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java new file mode 100644 index 0000000000..a8b86983b7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol; + +public class RequestCode { + + public static final int SEND_MESSAGE = 10; + + public static final int PULL_MESSAGE = 11; + + public static final int QUERY_MESSAGE = 12; + public static final int QUERY_BROKER_OFFSET = 13; + public static final int QUERY_CONSUMER_OFFSET = 14; + public static final int UPDATE_CONSUMER_OFFSET = 15; + public static final int UPDATE_AND_CREATE_TOPIC = 17; + public static final int GET_ALL_TOPIC_CONFIG = 21; + public static final int GET_TOPIC_CONFIG_LIST = 22; + + public static final int GET_TOPIC_NAME_LIST = 23; + + public static final int UPDATE_BROKER_CONFIG = 25; + + public static final int GET_BROKER_CONFIG = 26; + + public static final int TRIGGER_DELETE_FILES = 27; + + public static final int GET_BROKER_RUNTIME_INFO = 28; + public static final int SEARCH_OFFSET_BY_TIMESTAMP = 29; + public static final int GET_MAX_OFFSET = 30; + public static final int GET_MIN_OFFSET = 31; + + public static final int GET_EARLIEST_MSG_STORETIME = 32; + + public static final int VIEW_MESSAGE_BY_ID = 33; + + public static final int HEART_BEAT = 34; + + public static final int UNREGISTER_CLIENT = 35; + + public static final int CONSUMER_SEND_MSG_BACK = 36; + + public static final int END_TRANSACTION = 37; + public static final int GET_CONSUMER_LIST_BY_GROUP = 38; + + public static final int CHECK_TRANSACTION_STATE = 39; + + public static final int NOTIFY_CONSUMER_IDS_CHANGED = 40; + + public static final int LOCK_BATCH_MQ = 41; + + public static final int UNLOCK_BATCH_MQ = 42; + public static final int GET_ALL_CONSUMER_OFFSET = 43; + + public static final int GET_ALL_DELAY_OFFSET = 45; + + public static final int PUT_KV_CONFIG = 100; + + public static final int GET_KV_CONFIG = 101; + + public static final int DELETE_KV_CONFIG = 102; + + public static final int REGISTER_BROKER = 103; + + public static final int UNREGISTER_BROKER = 104; + public static final int GET_ROUTEINTO_BY_TOPIC = 105; + + public static final int GET_BROKER_CLUSTER_INFO = 106; + public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP = 200; + public static final int GET_ALL_SUBSCRIPTIONGROUP_CONFIG = 201; + public static final int GET_TOPIC_STATS_INFO = 202; + public static final int GET_CONSUMER_CONNECTION_LIST = 203; + public static final int GET_PRODUCER_CONNECTION_LIST = 204; + public static final int WIPE_WRITE_PERM_OF_BROKER = 205; + + + public static final int GET_ALL_TOPIC_LIST_FROM_NAMESERVER = 206; + + public static final int DELETE_SUBSCRIPTIONGROUP = 207; + public static final int GET_CONSUME_STATS = 208; + + public static final int SUSPEND_CONSUMER = 209; + + public static final int RESUME_CONSUMER = 210; + public static final int RESET_CONSUMER_OFFSET_IN_CONSUMER = 211; + public static final int RESET_CONSUMER_OFFSET_IN_BROKER = 212; + + public static final int ADJUST_CONSUMER_THREAD_POOL = 213; + + public static final int WHO_CONSUME_THE_MESSAGE = 214; + + + public static final int DELETE_TOPIC_IN_BROKER = 215; + + public static final int DELETE_TOPIC_IN_NAMESRV = 216; + public static final int GET_KVLIST_BY_NAMESPACE = 219; + + + public static final int RESET_CONSUMER_CLIENT_OFFSET = 220; + + public static final int GET_CONSUMER_STATUS_FROM_CLIENT = 221; + + public static final int INVOKE_BROKER_TO_RESET_OFFSET = 222; + + public static final int INVOKE_BROKER_TO_GET_CONSUMER_STATUS = 223; + + + public static final int QUERY_TOPIC_CONSUME_BY_WHO = 300; + + public static final int GET_TOPICS_BY_CLUSTER = 224; + + public static final int REGISTER_FILTER_SERVER = 301; + public static final int REGISTER_MESSAGE_FILTER_CLASS = 302; + + public static final int QUERY_CONSUME_TIME_SPAN = 303; + + public static final int GET_SYSTEM_TOPIC_LIST_FROM_NS = 304; + public static final int GET_SYSTEM_TOPIC_LIST_FROM_BROKER = 305; + + public static final int CLEAN_EXPIRED_CONSUMEQUEUE = 306; + + public static final int GET_CONSUMER_RUNNING_INFO = 307; + + public static final int QUERY_CORRECTION_OFFSET = 308; + public static final int CONSUME_MESSAGE_DIRECTLY = 309; + + public static final int SEND_MESSAGE_V2 = 310; + + public static final int GET_UNIT_TOPIC_LIST = 311; + + public static final int GET_HAS_UNIT_SUB_TOPIC_LIST = 312; + + public static final int GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST = 313; + + public static final int CLONE_GROUP_OFFSET = 314; + + public static final int VIEW_BROKER_STATS_DATA = 315; + + public static final int CLEAN_UNUSED_TOPIC = 316; + + public static final int GET_BROKER_CONSUME_STATS = 317; + + /** + * update the config of name server + */ + public static final int UPDATE_NAMESRV_CONFIG = 318; + + /** + * get config from name server + */ + public static final int GET_NAMESRV_CONFIG = 319; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java new file mode 100644 index 0000000000..3c01fad144 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; + + +public class ResponseCode extends RemotingSysResponseCode { + + public static final int FLUSH_DISK_TIMEOUT = 10; + + public static final int SLAVE_NOT_AVAILABLE = 11; + + public static final int FLUSH_SLAVE_TIMEOUT = 12; + + public static final int MESSAGE_ILLEGAL = 13; + + public static final int SERVICE_NOT_AVAILABLE = 14; + + public static final int VERSION_NOT_SUPPORTED = 15; + + public static final int NO_PERMISSION = 16; + + public static final int TOPIC_NOT_EXIST = 17; + public static final int TOPIC_EXIST_ALREADY = 18; + public static final int PULL_NOT_FOUND = 19; + + public static final int PULL_RETRY_IMMEDIATELY = 20; + + public static final int PULL_OFFSET_MOVED = 21; + + public static final int QUERY_NOT_FOUND = 22; + + public static final int SUBSCRIPTION_PARSE_FAILED = 23; + + public static final int SUBSCRIPTION_NOT_EXIST = 24; + + public static final int SUBSCRIPTION_NOT_LATEST = 25; + + public static final int SUBSCRIPTION_GROUP_NOT_EXIST = 26; + + public static final int TRANSACTION_SHOULD_COMMIT = 200; + + public static final int TRANSACTION_SHOULD_ROLLBACK = 201; + + public static final int TRANSACTION_STATE_UNKNOW = 202; + + public static final int TRANSACTION_STATE_GROUP_WRONG = 203; + public static final int NO_BUYER_ID = 204; + + + public static final int NOT_IN_CURRENT_UNIT = 205; + + + public static final int CONSUMER_NOT_ONLINE = 206; + + + public static final int CONSUME_MSG_TIMEOUT = 207; + + + public static final int NO_MESSAGE = 208; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsData.java new file mode 100644 index 0000000000..6f51b061c4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsData.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class BrokerStatsData extends RemotingSerializable { + + private BrokerStatsItem statsMinute; + + private BrokerStatsItem statsHour; + + private BrokerStatsItem statsDay; + + + public BrokerStatsItem getStatsMinute() { + return statsMinute; + } + + + public void setStatsMinute(BrokerStatsItem statsMinute) { + this.statsMinute = statsMinute; + } + + + public BrokerStatsItem getStatsHour() { + return statsHour; + } + + + public void setStatsHour(BrokerStatsItem statsHour) { + this.statsHour = statsHour; + } + + + public BrokerStatsItem getStatsDay() { + return statsDay; + } + + + public void setStatsDay(BrokerStatsItem statsDay) { + this.statsDay = statsDay; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsItem.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsItem.java new file mode 100644 index 0000000000..1cf6c3d640 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/BrokerStatsItem.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +public class BrokerStatsItem { + private long sum; + private double tps; + private double avgpt; + + + public long getSum() { + return sum; + } + + + public void setSum(long sum) { + this.sum = sum; + } + + + public double getTps() { + return tps; + } + + + public void setTps(double tps) { + this.tps = tps; + } + + + public double getAvgpt() { + return avgpt; + } + + + public void setAvgpt(double avgpt) { + this.avgpt = avgpt; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java new file mode 100644 index 0000000000..873b5489bd --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +public enum CMResult { + CR_SUCCESS, + CR_LATER, + CR_ROLLBACK, + CR_COMMIT, + CR_THROW_EXCEPTION, + CR_RETURN_NULL, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java new file mode 100644 index 0000000000..81d6447e51 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class ClusterInfo extends RemotingSerializable { + private HashMap brokerAddrTable; + private HashMap> clusterAddrTable; + + + public HashMap getBrokerAddrTable() { + return brokerAddrTable; + } + + + public void setBrokerAddrTable(HashMap brokerAddrTable) { + this.brokerAddrTable = brokerAddrTable; + } + + + public HashMap> getClusterAddrTable() { + return clusterAddrTable; + } + + + public void setClusterAddrTable(HashMap> clusterAddrTable) { + this.clusterAddrTable = clusterAddrTable; + } + + + public String[] retrieveAllAddrByCluster(String cluster) { + List addrs = new ArrayList(); + if (clusterAddrTable.containsKey(cluster)) { + Set brokerNames = clusterAddrTable.get(cluster); + for (String brokerName : brokerNames) { + BrokerData brokerData = brokerAddrTable.get(brokerName); + if (null != brokerData) { + addrs.addAll(brokerData.getBrokerAddrs().values()); + } + } + } + + return addrs.toArray(new String[]{}); + } + + + public String[] retrieveAllClusterNames() { + return clusterAddrTable.keySet().toArray(new String[]{}); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java new file mode 100644 index 0000000000..72cf6015c8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; + + +/** + * @author shijia.wxr + */ +public class Connection { + private String clientId; + private String clientAddr; + private LanguageCode language; + private int version; + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public String getClientAddr() { + return clientAddr; + } + + + public void setClientAddr(String clientAddr) { + this.clientAddr = clientAddr; + } + + + public LanguageCode getLanguage() { + return language; + } + + + public void setLanguage(LanguageCode language) { + this.language = language; + } + + + public int getVersion() { + return version; + } + + + public void setVersion(int version) { + this.version = version; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java new file mode 100644 index 0000000000..8a69352963 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; + + +/** + * @author shijia.wxr + * + */ +public class ConsumeByWho extends RemotingSerializable { + private HashSet consumedGroup = new HashSet(); + private HashSet notConsumedGroup = new HashSet(); + private String topic; + private int queueId; + private long offset; + + + public HashSet getConsumedGroup() { + return consumedGroup; + } + + + public void setConsumedGroup(HashSet consumedGroup) { + this.consumedGroup = consumedGroup; + } + + + public HashSet getNotConsumedGroup() { + return notConsumedGroup; + } + + + public void setNotConsumedGroup(HashSet notConsumedGroup) { + this.notConsumedGroup = notConsumedGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java new file mode 100644 index 0000000000..c895fe2d22 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class ConsumeMessageDirectlyResult extends RemotingSerializable { + private boolean order = false; + private boolean autoCommit = true; + private CMResult consumeResult; + private String remark; + private long spentTimeMills; + + + public boolean isOrder() { + return order; + } + + + public void setOrder(boolean order) { + this.order = order; + } + + + public boolean isAutoCommit() { + return autoCommit; + } + + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + + public String getRemark() { + return remark; + } + + + public void setRemark(String remark) { + this.remark = remark; + } + + + public CMResult getConsumeResult() { + return consumeResult; + } + + + public void setConsumeResult(CMResult consumeResult) { + this.consumeResult = consumeResult; + } + + + public long getSpentTimeMills() { + return spentTimeMills; + } + + + public void setSpentTimeMills(long spentTimeMills) { + this.spentTimeMills = spentTimeMills; + } + + + @Override + public String toString() { + return "ConsumeMessageDirectlyResult [order=" + order + ", autoCommit=" + autoCommit + + ", consumeResult=" + consumeResult + ", remark=" + remark + ", spentTimeMills=" + + spentTimeMills + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatsList.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatsList.java new file mode 100644 index 0000000000..a1c608d434 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatsList.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +/** + * @author shijia.wxr + */ +public class ConsumeStatsList extends RemotingSerializable { + private List>> consumeStatsList = new ArrayList>>(); + private String brokerAddr; + private long totalDiff; + + public List>> getConsumeStatsList() { + return consumeStatsList; + } + + public void setConsumeStatsList(List>> consumeStatsList) { + this.consumeStatsList = consumeStatsList; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + public long getTotalDiff() { + return totalDiff; + } + + public void setTotalDiff(long totalDiff) { + this.totalDiff = totalDiff; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java new file mode 100644 index 0000000000..dcb62814b3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +public class ConsumeStatus { + private double pullRT; + private double pullTPS; + private double consumeRT; + private double consumeOKTPS; + private double consumeFailedTPS; + + private long consumeFailedMsgs; + + + public double getPullRT() { + return pullRT; + } + + + public void setPullRT(double pullRT) { + this.pullRT = pullRT; + } + + + public double getPullTPS() { + return pullTPS; + } + + + public void setPullTPS(double pullTPS) { + this.pullTPS = pullTPS; + } + + + public double getConsumeRT() { + return consumeRT; + } + + + public void setConsumeRT(double consumeRT) { + this.consumeRT = consumeRT; + } + + + public double getConsumeOKTPS() { + return consumeOKTPS; + } + + + public void setConsumeOKTPS(double consumeOKTPS) { + this.consumeOKTPS = consumeOKTPS; + } + + + public double getConsumeFailedTPS() { + return consumeFailedTPS; + } + + + public void setConsumeFailedTPS(double consumeFailedTPS) { + this.consumeFailedTPS = consumeFailedTPS; + } + + + public long getConsumeFailedMsgs() { + return consumeFailedMsgs; + } + + + public void setConsumeFailedMsgs(long consumeFailedMsgs) { + this.consumeFailedMsgs = consumeFailedMsgs; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java new file mode 100644 index 0000000000..f74c6fc2e7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerConnection extends RemotingSerializable { + private HashSet connectionSet = new HashSet(); + private ConcurrentHashMap subscriptionTable = + new ConcurrentHashMap(); + private ConsumeType consumeType; + private MessageModel messageModel; + private ConsumeFromWhere consumeFromWhere; + + + public int computeMinVersion() { + int minVersion = Integer.MAX_VALUE; + for (Connection c : this.connectionSet) { + if (c.getVersion() < minVersion) { + minVersion = c.getVersion(); + } + } + + return minVersion; + } + + + public HashSet getConnectionSet() { + return connectionSet; + } + + + public void setConnectionSet(HashSet connectionSet) { + this.connectionSet = connectionSet; + } + + + public ConcurrentHashMap getSubscriptionTable() { + return subscriptionTable; + } + + + public void setSubscriptionTable(ConcurrentHashMap subscriptionTable) { + this.subscriptionTable = subscriptionTable; + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java new file mode 100644 index 0000000000..7af4e7a376 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author manhong.yqd + */ +public class ConsumerOffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap> offsetTable = + new ConcurrentHashMap>(512); + + + public ConcurrentHashMap> getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap> offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java new file mode 100644 index 0000000000..56babc2000 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java @@ -0,0 +1,315 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.*; +import java.util.Map.Entry; + +public class ConsumerRunningInfo extends RemotingSerializable { + public static final String PROP_NAMESERVER_ADDR = "PROP_NAMESERVER_ADDR"; + public static final String PROP_THREADPOOL_CORE_SIZE = "PROP_THREADPOOL_CORE_SIZE"; + public static final String PROP_CONSUME_ORDERLY = "PROP_CONSUMEORDERLY"; + public static final String PROP_CONSUME_TYPE = "PROP_CONSUME_TYPE"; + public static final String PROP_CLIENT_VERSION = "PROP_CLIENT_VERSION"; + public static final String PROP_CONSUMER_START_TIMESTAMP = "PROP_CONSUMER_START_TIMESTAMP"; + + + private Properties properties = new Properties(); + + private TreeSet subscriptionSet = new TreeSet(); + + private TreeMap mqTable = new TreeMap(); + + private TreeMap statusTable = new TreeMap(); + + private String jstack; + + public static boolean analyzeSubscription(final TreeMap criTable) { + ConsumerRunningInfo prev = criTable.firstEntry().getValue(); + + boolean push = false; + { + String property = prev.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE); + + if (property == null) { + property = ((ConsumeType) prev.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).name(); + } + push = ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY; + } + + boolean startForAWhile = false; + { + + String property = prev.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP); + if (property == null) { + property = String.valueOf(prev.getProperties().get(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP)); + } + startForAWhile = (System.currentTimeMillis() - Long.parseLong(property)) > (1000 * 60 * 2); + } + + if (push && startForAWhile) { + + { + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ConsumerRunningInfo current = next.getValue(); + boolean equals = current.getSubscriptionSet().equals(prev.getSubscriptionSet()); + + if (!equals) { + // Different subscription in the same group of consumer + return false; + } + + prev = next.getValue(); + } + + if (prev != null) { + + if (prev.getSubscriptionSet().isEmpty()) { + // Subscription empty! + return false; + } + } + } + } + + return true; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + public TreeSet getSubscriptionSet() { + return subscriptionSet; + } + + public void setSubscriptionSet(TreeSet subscriptionSet) { + this.subscriptionSet = subscriptionSet; + } + + public static boolean analyzeRebalance(final TreeMap criTable) { + return true; + } + + public static String analyzeProcessQueue(final String clientId, ConsumerRunningInfo info) { + StringBuilder sb = new StringBuilder(); + boolean push = false; + { + String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE); + + if (property == null) { + property = ((ConsumeType) info.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).name(); + } + push = ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY; + } + + boolean orderMsg = false; + { + String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_ORDERLY); + orderMsg = Boolean.parseBoolean(property); + } + + if (push) { + Iterator> it = info.getMqTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueueInfo pq = next.getValue(); + + + if (orderMsg) { + + if (!pq.isLocked()) { + sb.append(String.format("%s %s can't lock for a while, %dms%n", // + clientId, // + mq, // + System.currentTimeMillis() - pq.getLastLockTimestamp())); + } else { + if (pq.isDroped() && (pq.getTryUnlockTimes() > 0)) { + sb.append(String.format("%s %s unlock %d times, still failed%n", // + clientId, // + mq, // + pq.getTryUnlockTimes())); + } + } + + + } else { + long diff = System.currentTimeMillis() - pq.getLastConsumeTimestamp(); + + if (diff > (1000 * 60) && pq.getCachedMsgCount() > 0) { + sb.append(String.format("%s %s can't consume for a while, maybe blocked, %dms%n", // + clientId, // + mq, // + diff)); + } + } + } + } + + return sb.toString(); + } + + public TreeMap getMqTable() { + return mqTable; + } + + public void setMqTable(TreeMap mqTable) { + this.mqTable = mqTable; + } + + public TreeMap getStatusTable() { + return statusTable; + } + + public void setStatusTable(TreeMap statusTable) { + this.statusTable = statusTable; + } + + public String formatString() { + StringBuilder sb = new StringBuilder(); + + { + sb.append("#Consumer Properties#\n"); + Iterator> it = this.properties.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-40s: %s%n", next.getKey().toString(), next.getValue().toString()); + sb.append(item); + } + } + + { + sb.append("\n\n#Consumer Subscription#\n"); + + Iterator it = this.subscriptionSet.iterator(); + int i = 0; + while (it.hasNext()) { + SubscriptionData next = it.next(); + String item = String.format("%03d Topic: %-40s ClassFilter: %-8s SubExpression: %s%n", // + ++i, // + next.getTopic(), // + next.isClassFilterMode(), // + next.getSubString()); + + sb.append(item); + } + } + + { + sb.append("\n\n#Consumer Offset#\n"); + sb.append(String.format("%-32s %-32s %-4s %-20s%n", // + "#Topic", // + "#Broker Name", // + "#QID", // + "#Consumer Offset"// + )); + + Iterator> it = this.mqTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %-32s %-4d %-20d%n", // + next.getKey().getTopic(), // + next.getKey().getBrokerName(), // + next.getKey().getQueueId(), // + next.getValue().getCommitOffset()); + + sb.append(item); + } + } + + { + sb.append("\n\n#Consumer MQ Detail#\n"); + sb.append(String.format("%-32s %-32s %-4s %-20s%n", // + "#Topic", // + "#Broker Name", // + "#QID", // + "#ProcessQueueInfo"// + )); + + Iterator> it = this.mqTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %-32s %-4d %s%n", // + next.getKey().getTopic(), // + next.getKey().getBrokerName(), // + next.getKey().getQueueId(), // + next.getValue().toString()); + + sb.append(item); + } + } + + { + sb.append("\n\n#Consumer RT&TPS#\n"); + sb.append(String.format("%-32s %14s %14s %14s %14s %18s %25s%n", // + "#Topic", // + "#Pull RT", // + "#Pull TPS", // + "#Consume RT", // + "#ConsumeOK TPS", // + "#ConsumeFailed TPS", // + "#ConsumeFailedMsgsInHour"// + )); + + Iterator> it = this.statusTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %14.2f %14.2f %14.2f %14.2f %18.2f %25d%n", // + next.getKey(), // + next.getValue().getPullRT(), // + next.getValue().getPullTPS(), // + next.getValue().getConsumeRT(), // + next.getValue().getConsumeOKTPS(), // + next.getValue().getConsumeFailedTPS(), // + next.getValue().getConsumeFailedMsgs()// + ); + + sb.append(item); + } + } + + if (this.jstack != null) { + sb.append("\n\n#Consumer jstack#\n"); + sb.append(this.jstack); + } + + return sb.toString(); + } + + public String getJstack() { + return jstack; + } + + + public void setJstack(String jstack) { + this.jstack = jstack; + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java new file mode 100644 index 0000000000..ca84f212d6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; +import java.util.Map; + + +/** + * @author manhong.yqd + */ +@Deprecated +public class GetConsumerStatusBody extends RemotingSerializable { + private Map messageQueueTable = new HashMap(); + private Map> consumerTable = + new HashMap>(); + + + public Map getMessageQueueTable() { + return messageQueueTable; + } + + + public void setMessageQueueTable(Map messageQueueTable) { + this.messageQueueTable = messageQueueTable; + } + + + public Map> getConsumerTable() { + return consumerTable; + } + + + public void setConsumerTable(Map> consumerTable) { + this.consumerTable = consumerTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java new file mode 100644 index 0000000000..9f7e500df9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; + + +/** + * @author shijia.wxr + * + */ +public class GroupList extends RemotingSerializable { + private HashSet groupList = new HashSet(); + + + public HashSet getGroupList() { + return groupList; + } + + + public void setGroupList(HashSet groupList) { + this.groupList = groupList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java new file mode 100644 index 0000000000..41cfcb81c1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; + + +/** + * @author shijia.wxr + * + */ +public class KVTable extends RemotingSerializable { + private HashMap table = new HashMap(); + + + public HashMap getTable() { + return table; + } + + + public void setTable(HashMap table) { + this.table = table; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java new file mode 100644 index 0000000000..992f656eb3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + * + */ +public class LockBatchRequestBody extends RemotingSerializable { + private String consumerGroup; + private String clientId; + private Set mqSet = new HashSet(); + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public Set getMqSet() { + return mqSet; + } + + + public void setMqSet(Set mqSet) { + this.mqSet = mqSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java new file mode 100644 index 0000000000..12f6c76261 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + * + */ +public class LockBatchResponseBody extends RemotingSerializable { + + private Set lockOKMQSet = new HashSet(); + + + public Set getLockOKMQSet() { + return lockOKMQSet; + } + + + public void setLockOKMQSet(Set lockOKMQSet) { + this.lockOKMQSet = lockOKMQSet; + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java new file mode 100644 index 0000000000..6c174430a9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.UtilAll; + + +public class ProcessQueueInfo { + private long commitOffset; + + private long cachedMsgMinOffset; + private long cachedMsgMaxOffset; + private int cachedMsgCount; + + private long transactionMsgMinOffset; + private long transactionMsgMaxOffset; + private int transactionMsgCount; + + private boolean locked; + private long tryUnlockTimes; + private long lastLockTimestamp; + + private boolean droped; + private long lastPullTimestamp; + private long lastConsumeTimestamp; + + + public long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(long commitOffset) { + this.commitOffset = commitOffset; + } + + + public long getCachedMsgMinOffset() { + return cachedMsgMinOffset; + } + + + public void setCachedMsgMinOffset(long cachedMsgMinOffset) { + this.cachedMsgMinOffset = cachedMsgMinOffset; + } + + + public long getCachedMsgMaxOffset() { + return cachedMsgMaxOffset; + } + + + public void setCachedMsgMaxOffset(long cachedMsgMaxOffset) { + this.cachedMsgMaxOffset = cachedMsgMaxOffset; + } + + + public int getCachedMsgCount() { + return cachedMsgCount; + } + + + public void setCachedMsgCount(int cachedMsgCount) { + this.cachedMsgCount = cachedMsgCount; + } + + + public long getTransactionMsgMinOffset() { + return transactionMsgMinOffset; + } + + + public void setTransactionMsgMinOffset(long transactionMsgMinOffset) { + this.transactionMsgMinOffset = transactionMsgMinOffset; + } + + + public long getTransactionMsgMaxOffset() { + return transactionMsgMaxOffset; + } + + + public void setTransactionMsgMaxOffset(long transactionMsgMaxOffset) { + this.transactionMsgMaxOffset = transactionMsgMaxOffset; + } + + + public int getTransactionMsgCount() { + return transactionMsgCount; + } + + + public void setTransactionMsgCount(int transactionMsgCount) { + this.transactionMsgCount = transactionMsgCount; + } + + + public boolean isLocked() { + return locked; + } + + + public void setLocked(boolean locked) { + this.locked = locked; + } + + + public long getTryUnlockTimes() { + return tryUnlockTimes; + } + + + public void setTryUnlockTimes(long tryUnlockTimes) { + this.tryUnlockTimes = tryUnlockTimes; + } + + + public long getLastLockTimestamp() { + return lastLockTimestamp; + } + + + public void setLastLockTimestamp(long lastLockTimestamp) { + this.lastLockTimestamp = lastLockTimestamp; + } + + + public boolean isDroped() { + return droped; + } + + + public void setDroped(boolean droped) { + this.droped = droped; + } + + + public long getLastPullTimestamp() { + return lastPullTimestamp; + } + + + public void setLastPullTimestamp(long lastPullTimestamp) { + this.lastPullTimestamp = lastPullTimestamp; + } + + + public long getLastConsumeTimestamp() { + return lastConsumeTimestamp; + } + + + public void setLastConsumeTimestamp(long lastConsumeTimestamp) { + this.lastConsumeTimestamp = lastConsumeTimestamp; + } + + + @Override + public String toString() { + return "ProcessQueueInfo [commitOffset=" + commitOffset + ", cachedMsgMinOffset=" + + cachedMsgMinOffset + ", cachedMsgMaxOffset=" + cachedMsgMaxOffset + ", cachedMsgCount=" + + cachedMsgCount + ", transactionMsgMinOffset=" + transactionMsgMinOffset + + ", transactionMsgMaxOffset=" + transactionMsgMaxOffset + ", transactionMsgCount=" + + transactionMsgCount + ", locked=" + locked + ", tryUnlockTimes=" + tryUnlockTimes + + ", lastLockTimestamp=" + UtilAll.timeMillisToHumanString(lastLockTimestamp) + ", droped=" + + droped + ", lastPullTimestamp=" + UtilAll.timeMillisToHumanString(lastPullTimestamp) + + ", lastConsumeTimestamp=" + UtilAll.timeMillisToHumanString(lastConsumeTimestamp) + "]"; + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java new file mode 100644 index 0000000000..32fe1d0e33 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; + + +/** + * @author shijia.wxr + */ +public class ProducerConnection extends RemotingSerializable { + private HashSet connectionSet = new HashSet(); + + + public HashSet getConnectionSet() { + return connectionSet; + } + + + public void setConnectionSet(HashSet connectionSet) { + this.connectionSet = connectionSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java new file mode 100644 index 0000000000..2f526663ee --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.List; + + +/** + * @author manhong.yqd + */ +public class QueryConsumeTimeSpanBody extends RemotingSerializable { + List consumeTimeSpanSet = new ArrayList(); + + + public List getConsumeTimeSpanSet() { + return consumeTimeSpanSet; + } + + + public void setConsumeTimeSpanSet(List consumeTimeSpanSet) { + this.consumeTimeSpanSet = consumeTimeSpanSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java new file mode 100644 index 0000000000..225b90c594 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; +import java.util.Map; + + +/** + * @author manhong.yqd + */ +public class QueryCorrectionOffsetBody extends RemotingSerializable { + private Map correctionOffsets = new HashMap(); + + + public Map getCorrectionOffsets() { + return correctionOffsets; + } + + + public void setCorrectionOffsets(Map correctionOffsets) { + this.correctionOffsets = correctionOffsets; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java new file mode 100644 index 0000000000..14001ec272 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.Date; + + +/** + * @author manhong.yqd + */ +public class QueueTimeSpan { + private MessageQueue messageQueue; + private long minTimeStamp; + private long maxTimeStamp; + private long consumeTimeStamp; + private long delayTime; + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getMinTimeStamp() { + return minTimeStamp; + } + + + public void setMinTimeStamp(long minTimeStamp) { + this.minTimeStamp = minTimeStamp; + } + + + public long getMaxTimeStamp() { + return maxTimeStamp; + } + + + public void setMaxTimeStamp(long maxTimeStamp) { + this.maxTimeStamp = maxTimeStamp; + } + + + public long getConsumeTimeStamp() { + return consumeTimeStamp; + } + + + public void setConsumeTimeStamp(long consumeTimeStamp) { + this.consumeTimeStamp = consumeTimeStamp; + } + + + public String getMinTimeStampStr() { + return UtilAll.formatDate(new Date(minTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS); + } + + + public String getMaxTimeStampStr() { + return UtilAll.formatDate(new Date(maxTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS); + } + + + public String getConsumeTimeStampStr() { + return UtilAll.formatDate(new Date(consumeTimeStamp), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS); + } + + + public long getDelayTime() { + return delayTime; + } + + + public void setDelayTime(long delayTime) { + this.delayTime = delayTime; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java new file mode 100644 index 0000000000..364bbcb929 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.List; + + +public class RegisterBrokerBody extends RemotingSerializable { + private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + private List filterServerList = new ArrayList(); + + + public TopicConfigSerializeWrapper getTopicConfigSerializeWrapper() { + return topicConfigSerializeWrapper; + } + + + public void setTopicConfigSerializeWrapper(TopicConfigSerializeWrapper topicConfigSerializeWrapper) { + this.topicConfigSerializeWrapper = topicConfigSerializeWrapper; + } + + + public List getFilterServerList() { + return filterServerList; + } + + + public void setFilterServerList(List filterServerList) { + this.filterServerList = filterServerList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java new file mode 100644 index 0000000000..2122e61d2f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.Map; + + +/** + * @author manhong.yqd + * + */ +public class ResetOffsetBody extends RemotingSerializable { + private Map offsetTable; + + + public Map getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(Map offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBodyForC.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBodyForC.java new file mode 100644 index 0000000000..fb7360ef8b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBodyForC.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueueForC; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.List; + +public class ResetOffsetBodyForC extends RemotingSerializable { + + private List offsetTable; + + + public List getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(List offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java new file mode 100644 index 0000000000..096672c139 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author manhong.yqd + */ +public class SubscriptionGroupWrapper extends RemotingSerializable { + private ConcurrentHashMap subscriptionGroupTable = + new ConcurrentHashMap(1024); + private DataVersion dataVersion = new DataVersion(); + + + public ConcurrentHashMap getSubscriptionGroupTable() { + return subscriptionGroupTable; + } + + + public void setSubscriptionGroupTable( + ConcurrentHashMap subscriptionGroupTable) { + this.subscriptionGroupTable = subscriptionGroupTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java new file mode 100644 index 0000000000..0050762b68 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; + + +public class TopicConfigSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap topicConfigTable = + new ConcurrentHashMap(); + private DataVersion dataVersion = new DataVersion(); + + + public ConcurrentHashMap getTopicConfigTable() { + return topicConfigTable; + } + + + public void setTopicConfigTable(ConcurrentHashMap topicConfigTable) { + this.topicConfigTable = topicConfigTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java new file mode 100644 index 0000000000..84912cec60 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class TopicList extends RemotingSerializable { + private Set topicList = new HashSet(); + private String brokerAddr; + + + public Set getTopicList() { + return topicList; + } + + + public void setTopicList(Set topicList) { + this.topicList = topicList; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java new file mode 100644 index 0000000000..542b797448 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class UnlockBatchRequestBody extends RemotingSerializable { + private String consumerGroup; + private String clientId; + private Set mqSet = new HashSet(); + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public Set getMqSet() { + return mqSet; + } + + + public void setMqSet(Set mqSet) { + this.mqSet = mqSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java new file mode 100644 index 0000000000..37d6a7fe80 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CheckTransactionStateRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + private String msgId; + private String transactionId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } + + public String getMsgId() { + return msgId; + } + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java new file mode 100644 index 0000000000..76c97327ee --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CheckTransactionStateResponseHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + @CFNotNull + private Integer commitOrRollback; // TRANSACTION_COMMIT_TYPE + + + // TRANSACTION_ROLLBACK_TYPE + + @Override + public void checkFields() throws RemotingCommandException { + if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == this.commitOrRollback) { + return; + } + + throw new RemotingCommandException("commitOrRollback field wrong"); + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } + + + public Integer getCommitOrRollback() { + return commitOrRollback; + } + + + public void setCommitOrRollback(Integer commitOrRollback) { + this.commitOrRollback = commitOrRollback; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java new file mode 100644 index 0000000000..6043229eed --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + */ +public class CloneGroupOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String srcGroup; + @CFNotNull + private String destGroup; + private String topic; + private boolean offline; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getDestGroup() { + return destGroup; + } + + + public void setDestGroup(String destGroup) { + this.destGroup = destGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getSrcGroup() { + + return srcGroup; + } + + + public void setSrcGroup(String srcGroup) { + this.srcGroup = srcGroup; + } + + + public boolean isOffline() { + return offline; + } + + + public void setOffline(boolean offline) { + this.offline = offline; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java new file mode 100644 index 0000000000..3c68636e78 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNullable + private String clientId; + @CFNullable + private String msgId; + @CFNullable + private String brokerName; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java new file mode 100644 index 0000000000..c0acf88f73 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class ConsumerSendMsgBackRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + @CFNotNull + private String group; + @CFNotNull + private Integer delayLevel; + private String originMsgId; + private String originTopic; + @CFNullable + private boolean unitMode = false; + private Integer maxReconsumeTimes; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public Integer getDelayLevel() { + return delayLevel; + } + + + public void setDelayLevel(Integer delayLevel) { + this.delayLevel = delayLevel; + } + + + public String getOriginMsgId() { + return originMsgId; + } + + + public void setOriginMsgId(String originMsgId) { + this.originMsgId = originMsgId; + } + + + public String getOriginTopic() { + return originTopic; + } + + + public void setOriginTopic(String originTopic) { + this.originTopic = originTopic; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + + public Integer getMaxReconsumeTimes() { + return maxReconsumeTimes; + } + + + public void setMaxReconsumeTimes(final Integer maxReconsumeTimes) { + this.maxReconsumeTimes = maxReconsumeTimes; + } + + + @Override + public String toString() { + return "ConsumerSendMsgBackRequestHeader [group=" + group + ", originTopic=" + originTopic + ", originMsgId=" + originMsgId + + ", delayLevel=" + delayLevel + ", unitMode=" + unitMode + ", maxReconsumeTimes=" + maxReconsumeTimes + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java new file mode 100644 index 0000000000..a9d219c3c2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: CreateTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CreateTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer readQueueNums; + @CFNotNull + private Integer writeQueueNums; + @CFNotNull + private Integer perm; + @CFNotNull + private String topicFilterType; + private Integer topicSysFlag; + @CFNotNull + private Boolean order = false; + + + @Override + public void checkFields() throws RemotingCommandException { + try { + TopicFilterType.valueOf(this.topicFilterType); + } catch (Exception e) { + throw new RemotingCommandException("topicFilterType = [" + topicFilterType + "] value invalid", e); + } + } + + + public TopicFilterType getTopicFilterTypeEnum() { + return TopicFilterType.valueOf(this.topicFilterType); + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getDefaultTopic() { + return defaultTopic; + } + + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + + public Integer getReadQueueNums() { + return readQueueNums; + } + + + public void setReadQueueNums(Integer readQueueNums) { + this.readQueueNums = readQueueNums; + } + + + public Integer getWriteQueueNums() { + return writeQueueNums; + } + + + public void setWriteQueueNums(Integer writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + + public Integer getPerm() { + return perm; + } + + + public void setPerm(Integer perm) { + this.perm = perm; + } + + + public String getTopicFilterType() { + return topicFilterType; + } + + + public void setTopicFilterType(String topicFilterType) { + this.topicFilterType = topicFilterType; + } + + + public Integer getTopicSysFlag() { + return topicSysFlag; + } + + + public void setTopicSysFlag(Integer topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + + public Boolean getOrder() { + return order; + } + + + public void setOrder(Boolean order) { + this.order = order; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java new file mode 100644 index 0000000000..9307c01f26 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + */ +public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader { + @CFNotNull + private String groupName; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java new file mode 100644 index 0000000000..4b1a844497 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class DeleteTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java new file mode 100644 index 0000000000..12f052d43e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class EndTransactionRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + @CFNotNull + private Integer commitOrRollback; // TRANSACTION_COMMIT_TYPE + // TRANSACTION_ROLLBACK_TYPE + // TRANSACTION_NOT_TYPE + + @CFNullable + private Boolean fromTransactionCheck = false; + + @CFNotNull + private String msgId; + + private String transactionId; + + @Override + public void checkFields() throws RemotingCommandException { + if (MessageSysFlag.TRANSACTION_NOT_TYPE == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TRANSACTION_COMMIT_TYPE == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == this.commitOrRollback) { + return; + } + + throw new RemotingCommandException("commitOrRollback field wrong"); + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } + + + public Integer getCommitOrRollback() { + return commitOrRollback; + } + + + public void setCommitOrRollback(Integer commitOrRollback) { + this.commitOrRollback = commitOrRollback; + } + + + public Boolean getFromTransactionCheck() { + return fromTransactionCheck; + } + + + public void setFromTransactionCheck(Boolean fromTransactionCheck) { + this.fromTransactionCheck = fromTransactionCheck; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + + @Override + public String toString() { + return "EndTransactionRequestHeader [producerGroup=" + producerGroup + ", tranStateTableOffset=" + + tranStateTableOffset + ", commitLogOffset=" + commitLogOffset + ", commitOrRollback=" + + commitOrRollback + ", fromTransactionCheck=" + fromTransactionCheck + ", msgId=" + msgId + + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java new file mode 100644 index 0000000000..49b71f5c52 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class EndTransactionResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java new file mode 100644 index 0000000000..e6283b6185 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetAllTopicConfigResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetAllTopicConfigResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java new file mode 100644 index 0000000000..21a16fa78e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetBrokerConfigResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetBrokerConfigResponseHeader implements CommandCustomHeader { + @CFNotNull + private String version; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getVersion() { + return version; + } + + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsInBrokerHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsInBrokerHeader.java new file mode 100644 index 0000000000..20eff19d01 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsInBrokerHeader.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class GetConsumeStatsInBrokerHeader implements CommandCustomHeader { + @CFNotNull + private boolean isOrder; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public boolean isOrder() { + return isOrder; + } + + public void setIsOrder(boolean isOrder) { + this.isOrder = isOrder; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java new file mode 100644 index 0000000000..e2f4ca3380 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumeStatsRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java new file mode 100644 index 0000000000..96bfed5480 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * + * @author shijia.wxr + * + */ +public class GetConsumerConnectionListRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // To change body of implemented methods use File | Settings | File + // Templates. + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java new file mode 100644 index 0000000000..bf38215841 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java new file mode 100644 index 0000000000..b382ece2ea --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupResponseBody extends RemotingSerializable { + private List consumerIdList; + + + public List getConsumerIdList() { + return consumerIdList; + } + + + public void setConsumerIdList(List consumerIdList) { + this.consumerIdList = consumerIdList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java new file mode 100644 index 0000000000..33f73e5d12 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java new file mode 100644 index 0000000000..bc6ac8920c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerRunningInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String clientId; + @CFNullable + private boolean jstackEnable; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public boolean isJstackEnable() { + return jstackEnable; + } + + + public void setJstackEnable(boolean jstackEnable) { + this.jstackEnable = jstackEnable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java new file mode 100644 index 0000000000..aa138f76ec --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + * + */ +public class GetConsumerStatusRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + @CFNullable + private String clientAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public String getClientAddr() { + return clientAddr; + } + + + public void setClientAddr(String clientAddr) { + this.clientAddr = clientAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java new file mode 100644 index 0000000000..0610fc82a6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetEarliestMsgStoretimeRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java new file mode 100644 index 0000000000..710fbfbba1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetEarliestMsgStoretimeResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetEarliestMsgStoretimeResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long timestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java new file mode 100644 index 0000000000..f44aeaf7dc --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetMaxOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMaxOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java new file mode 100644 index 0000000000..a0592a1b2a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetMaxOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMaxOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java new file mode 100644 index 0000000000..0e6371e9de --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMinOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java new file mode 100644 index 0000000000..82e588b22e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetMinOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMinOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java new file mode 100644 index 0000000000..83f0e6102b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetProducerConnectionListRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // To change body of implemented methods use File | Settings | File + // Templates. + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java new file mode 100644 index 0000000000..b7085bb6fa --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class GetTopicStatsInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java new file mode 100644 index 0000000000..c0f026f061 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + * + */ +public class GetTopicsByClusterRequestHeader implements CommandCustomHeader { + @CFNotNull + private String cluster; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getCluster() { + return cluster; + } + + + public void setCluster(String cluster) { + this.cluster = cluster; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java new file mode 100644 index 0000000000..bc93fb69c4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class NotifyConsumerIdsChangedRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java new file mode 100644 index 0000000000..ce2a6cebbe --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: PullMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class PullMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long queueOffset; + @CFNotNull + private Integer maxMsgNums; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long commitOffset; + @CFNotNull + private Long suspendTimeoutMillis; + @CFNullable + private String subscription; + @CFNotNull + private Long subVersion; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } + + + public Integer getMaxMsgNums() { + return maxMsgNums; + } + + + public void setMaxMsgNums(Integer maxMsgNums) { + this.maxMsgNums = maxMsgNums; + } + + + public Integer getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + + public Long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(Long commitOffset) { + this.commitOffset = commitOffset; + } + + + public Long getSuspendTimeoutMillis() { + return suspendTimeoutMillis; + } + + + public void setSuspendTimeoutMillis(Long suspendTimeoutMillis) { + this.suspendTimeoutMillis = suspendTimeoutMillis; + } + + + public String getSubscription() { + return subscription; + } + + + public void setSubscription(String subscription) { + this.subscription = subscription; + } + + + public Long getSubVersion() { + return subVersion; + } + + + public void setSubVersion(Long subVersion) { + this.subVersion = subVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java new file mode 100644 index 0000000000..952f718545 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: PullMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class PullMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long suggestWhichBrokerId; + @CFNotNull + private Long nextBeginOffset; + @CFNotNull + private Long minOffset; + @CFNotNull + private Long maxOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getNextBeginOffset() { + return nextBeginOffset; + } + + + public void setNextBeginOffset(Long nextBeginOffset) { + this.nextBeginOffset = nextBeginOffset; + } + + + public Long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(Long minOffset) { + this.minOffset = minOffset; + } + + + public Long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(Long maxOffset) { + this.maxOffset = maxOffset; + } + + + public Long getSuggestWhichBrokerId() { + return suggestWhichBrokerId; + } + + + public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) { + this.suggestWhichBrokerId = suggestWhichBrokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java new file mode 100644 index 0000000000..58ad7410c1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + * + */ +public class QueryConsumeTimeSpanRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java new file mode 100644 index 0000000000..e5e92477c3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java new file mode 100644 index 0000000000..baca34232c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryConsumerOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java new file mode 100644 index 0000000000..85b4d0cd22 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * + * @author manhong.yqd + * + */ +public class QueryCorrectionOffsetHeader implements CommandCustomHeader { + private String filterGroups; + @CFNotNull + private String compareGroup; + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getFilterGroups() { + return filterGroups; + } + + + public void setFilterGroups(String filterGroups) { + this.filterGroups = filterGroups; + } + + + public String getCompareGroup() { + return compareGroup; + } + + + public void setCompareGroup(String compareGroup) { + this.compareGroup = compareGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java new file mode 100644 index 0000000000..828e35ceb9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String key; + @CFNotNull + private Integer maxNum; + @CFNotNull + private Long beginTimestamp; + @CFNotNull + private Long endTimestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } + + + public Integer getMaxNum() { + return maxNum; + } + + + public void setMaxNum(Integer maxNum) { + this.maxNum = maxNum; + } + + + public Long getBeginTimestamp() { + return beginTimestamp; + } + + + public void setBeginTimestamp(Long beginTimestamp) { + this.beginTimestamp = beginTimestamp; + } + + + public Long getEndTimestamp() { + return endTimestamp; + } + + + public void setEndTimestamp(Long endTimestamp) { + this.endTimestamp = endTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java new file mode 100644 index 0000000000..b05bee97a9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long indexLastUpdateTimestamp; + @CFNotNull + private Long indexLastUpdatePhyoffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public void setIndexLastUpdateTimestamp(Long indexLastUpdateTimestamp) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + } + + + public Long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } + + + public void setIndexLastUpdatePhyoffset(Long indexLastUpdatePhyoffset) { + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java new file mode 100644 index 0000000000..7ba3dc3883 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryTopicConsumeByWhoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java new file mode 100644 index 0000000000..cad1aab48f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * + * @author manhong.yqd + * + */ +public class ResetOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + @CFNotNull + private long timestamp; + @CFNotNull + private boolean isForce; + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + + public boolean isForce() { + return isForce; + } + + + public void setForce(boolean isForce) { + this.isForce = isForce; + } + + + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java new file mode 100644 index 0000000000..6e9d476b2c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SearchOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SearchOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long timestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java new file mode 100644 index 0000000000..829f360ff4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SearchOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SearchOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java new file mode 100644 index 0000000000..93b1951db1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SendMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SendMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer defaultTopicQueueNums; + @CFNotNull + private Integer queueId; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long bornTimestamp; + @CFNotNull + private Integer flag; + @CFNullable + private String properties; + @CFNullable + private Integer reconsumeTimes; + @CFNullable + private boolean unitMode = false; + private Integer maxReconsumeTimes; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getDefaultTopic() { + return defaultTopic; + } + + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + + public Integer getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Integer getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + + public Long getBornTimestamp() { + return bornTimestamp; + } + + + public void setBornTimestamp(Long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + + public Integer getFlag() { + return flag; + } + + + public void setFlag(Integer flag) { + this.flag = flag; + } + + + public String getProperties() { + return properties; + } + + + public void setProperties(String properties) { + this.properties = properties; + } + + + public Integer getReconsumeTimes() { + return reconsumeTimes; + } + + + public void setReconsumeTimes(Integer reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public Integer getMaxReconsumeTimes() { + return maxReconsumeTimes; + } + + + public void setMaxReconsumeTimes(final Integer maxReconsumeTimes) { + this.maxReconsumeTimes = maxReconsumeTimes; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java new file mode 100644 index 0000000000..dadf4bf065 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain producerGroup copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SendMessageRequestHeaderV2 implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer defaultTopicQueueNums; + @CFNotNull + private Integer queueId; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long bornTimestamp; + @CFNotNull + private Integer flag; + @CFNullable + private String properties; + @CFNullable + private Integer reconsumeTimes; + @CFNullable + private boolean unitMode; + + private Integer consumeRetryTimes; + + public static SendMessageRequestHeader createSendMessageRequestHeaderV1(final SendMessageRequestHeaderV2 v2) { + SendMessageRequestHeader v1 = new SendMessageRequestHeader(); + v1.setProducerGroup(v2.producerGroup); + v1.setTopic(v2.topic); + v1.setDefaultTopic(v2.defaultTopic); + v1.setDefaultTopicQueueNums(v2.defaultTopicQueueNums); + v1.setQueueId(v2.queueId); + v1.setSysFlag(v2.sysFlag); + v1.setBornTimestamp(v2.bornTimestamp); + v1.setFlag(v2.flag); + v1.setProperties(v2.properties); + v1.setReconsumeTimes(v2.reconsumeTimes); + v1.setUnitMode(v2.unitMode); + v1.setMaxReconsumeTimes(v2.consumeRetryTimes); + return v1; + } + + public static SendMessageRequestHeaderV2 createSendMessageRequestHeaderV2(final SendMessageRequestHeader v1) { + SendMessageRequestHeaderV2 v2 = new SendMessageRequestHeaderV2(); + v2.producerGroup = v1.getProducerGroup(); + v2.topic = v1.getTopic(); + v2.defaultTopic = v1.getDefaultTopic(); + v2.defaultTopicQueueNums = v1.getDefaultTopicQueueNums(); + v2.queueId = v1.getQueueId(); + v2.sysFlag = v1.getSysFlag(); + v2.bornTimestamp = v1.getBornTimestamp(); + v2.flag = v1.getFlag(); + v2.properties = v1.getProperties(); + v2.reconsumeTimes = v1.getReconsumeTimes(); + v2.unitMode = v1.isUnitMode(); + v2.consumeRetryTimes = v1.getMaxReconsumeTimes(); + return v2; + } + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getDefaultTopic() { + return defaultTopic; + } + + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + + public Integer getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Integer getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + + public Long getBornTimestamp() { + return bornTimestamp; + } + + + public void setBornTimestamp(Long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + + public Integer getFlag() { + return flag; + } + + + public void setFlag(Integer flag) { + this.flag = flag; + } + + + public String getProperties() { + return properties; + } + + + public void setProperties(String properties) { + this.properties = properties; + } + + + public Integer getReconsumeTimes() { + return reconsumeTimes; + } + + + public void setReconsumeTimes(Integer reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + + public Integer getConsumeRetryTimes() { + return consumeRetryTimes; + } + + + public void setConsumeRetryTimes(final Integer consumeRetryTimes) { + this.consumeRetryTimes = consumeRetryTimes; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java new file mode 100644 index 0000000000..53c233a585 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SendMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SendMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private String msgId; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long queueOffset; + private String transactionId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java new file mode 100644 index 0000000000..cf3fcca81e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UnregisterClientRequestHeader implements CommandCustomHeader { + @CFNotNull + private String clientID; + + @CFNullable + private String producerGroup; + @CFNullable + private String consumerGroup; + + + public String getClientID() { + return clientID; + } + + + public void setClientID(String clientID) { + this.clientID = clientID; + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java new file mode 100644 index 0000000000..86863bad64 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UnregisterClientResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java new file mode 100644 index 0000000000..4fd73266a1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: UpdateConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long commitOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(Long commitOffset) { + this.commitOffset = commitOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java new file mode 100644 index 0000000000..a90f41ea0f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: UpdateConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UpdateConsumerOffsetResponseHeader implements CommandCustomHeader { + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewBrokerStatsDataRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewBrokerStatsDataRequestHeader.java new file mode 100644 index 0000000000..45f3ac3432 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewBrokerStatsDataRequestHeader.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class ViewBrokerStatsDataRequestHeader implements CommandCustomHeader { + @CFNotNull + private String statsName; + @CFNotNull + private String statsKey; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getStatsName() { + return statsName; + } + + + public void setStatsName(String statsName) { + this.statsName = statsName; + } + + + public String getStatsKey() { + return statsKey; + } + + + public void setStatsKey(String statsKey) { + this.statsKey = statsKey; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java new file mode 100644 index 0000000000..d1db42f5cb --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ViewMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class ViewMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java new file mode 100644 index 0000000000..d9344a5ced --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ViewMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class ViewMessageResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java new file mode 100644 index 0000000000..ef07b43cf5 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterFilterServerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String filterServerAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getFilterServerAddr() { + return filterServerAddr; + } + + + public void setFilterServerAddr(String filterServerAddr) { + this.filterServerAddr = filterServerAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java new file mode 100644 index 0000000000..7a4a1dee86 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterFilterServerResponseHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java new file mode 100644 index 0000000000..2423d952a1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterMessageFilterClassRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String className; + @CFNotNull + private Integer classCRC; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getClassName() { + return className; + } + + + public void setClassName(String className) { + this.className = className; + } + + + public Integer getClassCRC() { + return classCRC; + } + + + public void setClassCRC(Integer classCRC) { + this.classCRC = classCRC; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java new file mode 100644 index 0000000000..c3bc110dc0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class DeleteKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java new file mode 100644 index 0000000000..0bfec69457 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class DeleteTopicInNamesrvRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java new file mode 100644 index 0000000000..595ab381b3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class GetKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java new file mode 100644 index 0000000000..f4c63a8e96 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class GetKVConfigResponseHeader implements CommandCustomHeader { + @CFNullable + private String value; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getValue() { + return value; + } + + + public void setValue(String value) { + this.value = value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java new file mode 100644 index 0000000000..82cfc6d974 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class GetKVListByNamespaceRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java new file mode 100644 index 0000000000..fce8802e3e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetRouteInfoRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetRouteInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java new file mode 100644 index 0000000000..95777d6521 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetRouteInfoResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetRouteInfoResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java new file mode 100644 index 0000000000..22abc6c029 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class PutKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + @CFNotNull + private String value; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } + + + public String getValue() { + return value; + } + + + public void setValue(String value) { + this.value = value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java new file mode 100644 index 0000000000..4ae448f0aa --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: RegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author lansheng.zj + */ +public class RegisterBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String clusterName; + @CFNotNull + private String haServerAddr; + @CFNotNull + private Long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getClusterName() { + return clusterName; + } + + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public Long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java new file mode 100644 index 0000000000..5aed1989a7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class RegisterBrokerResponseHeader implements CommandCustomHeader { + @CFNullable + private String haServerAddr; + @CFNullable + private String masterAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java new file mode 100644 index 0000000000..e5a895d65c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: RegisterOrderTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class RegisterOrderTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String orderTopicString; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getOrderTopicString() { + return orderTopicString; + } + + + public void setOrderTopicString(String orderTopicString) { + this.orderTopicString = orderTopicString; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java new file mode 100644 index 0000000000..0cde0ffb6f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: UnRegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author lansheng.zj + */ +public class UnRegisterBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String clusterName; + @CFNotNull + private Long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getClusterName() { + return clusterName; + } + + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + + public Long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java new file mode 100644 index 0000000000..ed664d5097 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class WipeWritePermOfBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java new file mode 100644 index 0000000000..9df9d3f557 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * + */ +public class WipeWritePermOfBrokerResponseHeader implements CommandCustomHeader { + @CFNotNull + private Integer wipeTopicCount; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Integer getWipeTopicCount() { + return wipeTopicCount; + } + + + public void setWipeTopicCount(Integer wipeTopicCount) { + this.wipeTopicCount = wipeTopicCount; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java new file mode 100644 index 0000000000..9144c8c962 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ConsumeType.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * @author shijia.wxr + */ +public enum ConsumeType { + + CONSUME_ACTIVELY("PULL"), + + CONSUME_PASSIVELY("PUSH"); + + private String typeCN; + + ConsumeType(String typeCN) { + this.typeCN = typeCN; + } + + + public String getTypeCN() { + return typeCN; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java new file mode 100644 index 0000000000..f1fd6f9661 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ConsumerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class ConsumerData { + private String groupName; + private ConsumeType consumeType; + private MessageModel messageModel; + private ConsumeFromWhere consumeFromWhere; + private Set subscriptionDataSet = new HashSet(); + private boolean unitMode; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } + + + public Set getSubscriptionDataSet() { + return subscriptionDataSet; + } + + + public void setSubscriptionDataSet(Set subscriptionDataSet) { + this.subscriptionDataSet = subscriptionDataSet; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + @Override + public String toString() { + return "ConsumerData [groupName=" + groupName + ", consumeType=" + consumeType + ", messageModel=" + + messageModel + ", consumeFromWhere=" + consumeFromWhere + ", unitMode=" + unitMode + + ", subscriptionDataSet=" + subscriptionDataSet + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java new file mode 100644 index 0000000000..8f11e39e97 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: HeartbeatData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class HeartbeatData extends RemotingSerializable { + private String clientID; + private Set producerDataSet = new HashSet(); + private Set consumerDataSet = new HashSet(); + + + public String getClientID() { + return clientID; + } + + + public void setClientID(String clientID) { + this.clientID = clientID; + } + + + public Set getProducerDataSet() { + return producerDataSet; + } + + + public void setProducerDataSet(Set producerDataSet) { + this.producerDataSet = producerDataSet; + } + + + public Set getConsumerDataSet() { + return consumerDataSet; + } + + + public void setConsumerDataSet(Set consumerDataSet) { + this.consumerDataSet = consumerDataSet; + } + + + @Override + public String toString() { + return "HeartbeatData [clientID=" + clientID + ", producerDataSet=" + producerDataSet + + ", consumerDataSet=" + consumerDataSet + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java new file mode 100644 index 0000000000..1ff74facb4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: MessageModel.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * Message model + * + * @author shijia.wxr + */ +public enum MessageModel { + /** + * broadcast + */ + BROADCASTING("BROADCASTING"), + /** + * clustering + */ + CLUSTERING("CLUSTERING"); + + private String modeCN; + + MessageModel(String modeCN) { + this.modeCN = modeCN; + } + + + public String getModeCN() { + return modeCN; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java new file mode 100644 index 0000000000..ee95dddf95 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ProducerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * @author shijia.wxr + */ +public class ProducerData { + private String groupName; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + @Override + public String toString() { + return "ProducerData [groupName=" + groupName + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java new file mode 100644 index 0000000000..b4b72fcd85 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SubscriptionData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import com.alibaba.fastjson.annotation.JSONField; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class SubscriptionData implements Comparable { + public final static String SUB_ALL = "*"; + private boolean classFilterMode = false; + private String topic; + private String subString; + private Set tagsSet = new HashSet(); + private Set codeSet = new HashSet(); + private long subVersion = System.currentTimeMillis(); + + @JSONField(serialize = false) + private String filterClassSource; + + + public SubscriptionData() { + + } + + + public SubscriptionData(String topic, String subString) { + super(); + this.topic = topic; + this.subString = subString; + } + + public String getFilterClassSource() { + return filterClassSource; + } + + public void setFilterClassSource(String filterClassSource) { + this.filterClassSource = filterClassSource; + } + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getSubString() { + return subString; + } + + + public void setSubString(String subString) { + this.subString = subString; + } + + + public Set getTagsSet() { + return tagsSet; + } + + + public void setTagsSet(Set tagsSet) { + this.tagsSet = tagsSet; + } + + + public long getSubVersion() { + return subVersion; + } + + + public void setSubVersion(long subVersion) { + this.subVersion = subVersion; + } + + + public Set getCodeSet() { + return codeSet; + } + + + public void setCodeSet(Set codeSet) { + this.codeSet = codeSet; + } + + + public boolean isClassFilterMode() { + return classFilterMode; + } + + + public void setClassFilterMode(boolean classFilterMode) { + this.classFilterMode = classFilterMode; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (classFilterMode ? 1231 : 1237); + result = prime * result + ((codeSet == null) ? 0 : codeSet.hashCode()); + result = prime * result + ((subString == null) ? 0 : subString.hashCode()); + result = prime * result + ((tagsSet == null) ? 0 : tagsSet.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SubscriptionData other = (SubscriptionData) obj; + if (classFilterMode != other.classFilterMode) + return false; + if (codeSet == null) { + if (other.codeSet != null) + return false; + } else if (!codeSet.equals(other.codeSet)) + return false; + if (subString == null) { + if (other.subString != null) + return false; + } else if (!subString.equals(other.subString)) + return false; + if (subVersion != other.subVersion) + return false; + if (tagsSet == null) { + if (other.tagsSet != null) + return false; + } else if (!tagsSet.equals(other.tagsSet)) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } else if (!topic.equals(other.topic)) + return false; + return true; + } + + + @Override + public String toString() { + return "SubscriptionData [classFilterMode=" + classFilterMode + ", topic=" + topic + ", subString=" + + subString + ", tagsSet=" + tagsSet + ", codeSet=" + codeSet + ", subVersion=" + subVersion + + "]"; + } + + + @Override + public int compareTo(SubscriptionData other) { + String thisValue = this.topic + "@" + this.subString; + String otherValue = other.topic + "@" + other.subString; + return thisValue.compareTo(otherValue); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java new file mode 100644 index 0000000000..322953a630 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: BrokerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +import com.alibaba.rocketmq.common.MixAll; + +import java.util.HashMap; +import java.util.Map; + + +/** + * @author shijia.wxr + * + */ +public class BrokerData implements Comparable { + private String cluster; + private String brokerName; + private HashMap brokerAddrs; + + public String selectBrokerAddr() { + String value = this.brokerAddrs.get(MixAll.MASTER_ID); + if (null == value) { + for (Map.Entry entry : this.brokerAddrs.entrySet()) { + return entry.getValue(); + } + } + + return value; + } + + public HashMap getBrokerAddrs() { + return brokerAddrs; + } + + public void setBrokerAddrs(HashMap brokerAddrs) { + this.brokerAddrs = brokerAddrs; + } + + public String getCluster() { + return cluster; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerAddrs == null) ? 0 : brokerAddrs.hashCode()); + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BrokerData other = (BrokerData) obj; + if (brokerAddrs == null) { + if (other.brokerAddrs != null) + return false; + } else if (!brokerAddrs.equals(other.brokerAddrs)) + return false; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } else if (!brokerName.equals(other.brokerName)) + return false; + return true; + } + + @Override + public String toString() { + return "BrokerData [brokerName=" + brokerName + ", brokerAddrs=" + brokerAddrs + "]"; + } + + @Override + public int compareTo(BrokerData o) { + return this.brokerName.compareTo(o.getBrokerName()); + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java new file mode 100644 index 0000000000..6f62340404 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueueData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +public class QueueData implements Comparable { + private String brokerName; + private int readQueueNums; + private int writeQueueNums; + private int perm; + private int topicSynFlag; + + public int getReadQueueNums() { + return readQueueNums; + } + + public void setReadQueueNums(int readQueueNums) { + this.readQueueNums = readQueueNums; + } + + public int getWriteQueueNums() { + return writeQueueNums; + } + + public void setWriteQueueNums(int writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + public int getPerm() { + return perm; + } + + public void setPerm(int perm) { + this.perm = perm; + } + + public int getTopicSynFlag() { + return topicSynFlag; + } + + public void setTopicSynFlag(int topicSynFlag) { + this.topicSynFlag = topicSynFlag; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + result = prime * result + perm; + result = prime * result + readQueueNums; + result = prime * result + writeQueueNums; + result = prime * result + topicSynFlag; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + QueueData other = (QueueData) obj; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } else if (!brokerName.equals(other.brokerName)) + return false; + if (perm != other.perm) + return false; + if (readQueueNums != other.readQueueNums) + return false; + if (writeQueueNums != other.writeQueueNums) + return false; + if (topicSynFlag != other.topicSynFlag) + return false; + return true; + } + + @Override + public String toString() { + return "QueueData [brokerName=" + brokerName + ", readQueueNums=" + readQueueNums + + ", writeQueueNums=" + writeQueueNums + ", perm=" + perm + ", topicSynFlag=" + topicSynFlag + + "]"; + } + + @Override + public int compareTo(QueueData o) { + return this.brokerName.compareTo(o.getBrokerName()); + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java new file mode 100644 index 0000000000..72e1b96b46 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: TopicRouteData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class TopicRouteData extends RemotingSerializable { + private String orderTopicConf; + private List queueDatas; + private List brokerDatas; + private HashMap/* Filter Server */> filterServerTable; + + + public TopicRouteData cloneTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setQueueDatas(new ArrayList()); + topicRouteData.setBrokerDatas(new ArrayList()); + topicRouteData.setFilterServerTable(new HashMap>()); + topicRouteData.setOrderTopicConf(this.orderTopicConf); + + if (this.queueDatas != null) { + topicRouteData.getQueueDatas().addAll(this.queueDatas); + } + + if (this.brokerDatas != null) { + topicRouteData.getBrokerDatas().addAll(this.brokerDatas); + } + + if (this.filterServerTable != null) { + topicRouteData.getFilterServerTable().putAll(this.filterServerTable); + } + + return topicRouteData; + } + + + public List getQueueDatas() { + return queueDatas; + } + + + public void setQueueDatas(List queueDatas) { + this.queueDatas = queueDatas; + } + + + public List getBrokerDatas() { + return brokerDatas; + } + + + public void setBrokerDatas(List brokerDatas) { + this.brokerDatas = brokerDatas; + } + + public HashMap> getFilterServerTable() { + return filterServerTable; + } + + public void setFilterServerTable(HashMap> filterServerTable) { + this.filterServerTable = filterServerTable; + } + + public String getOrderTopicConf() { + return orderTopicConf; + } + + public void setOrderTopicConf(String orderTopicConf) { + this.orderTopicConf = orderTopicConf; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerDatas == null) ? 0 : brokerDatas.hashCode()); + result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode()); + result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode()); + result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TopicRouteData other = (TopicRouteData) obj; + if (brokerDatas == null) { + if (other.brokerDatas != null) + return false; + } else if (!brokerDatas.equals(other.brokerDatas)) + return false; + if (orderTopicConf == null) { + if (other.orderTopicConf != null) + return false; + } else if (!orderTopicConf.equals(other.orderTopicConf)) + return false; + if (queueDatas == null) { + if (other.queueDatas != null) + return false; + } else if (!queueDatas.equals(other.queueDatas)) + return false; + if (filterServerTable == null) { + if (other.filterServerTable != null) + return false; + } else if (!filterServerTable.equals(other.filterServerTable)) + return false; + return true; + } + + @Override + public String toString() { + return "TopicRouteData [orderTopicConf=" + orderTopicConf + ", queueDatas=" + queueDatas + + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java new file mode 100644 index 0000000000..86bdd3d141 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol.topic; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class OffsetMovedEvent extends RemotingSerializable { + private String consumerGroup; + private MessageQueue messageQueue; + private long offsetRequest; + private long offsetNew; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getOffsetRequest() { + return offsetRequest; + } + + + public void setOffsetRequest(long offsetRequest) { + this.offsetRequest = offsetRequest; + } + + + public long getOffsetNew() { + return offsetNew; + } + + + public void setOffsetNew(long offsetNew) { + this.offsetNew = offsetNew; + } + + + @Override + public String toString() { + return "OffsetMovedEvent [consumerGroup=" + consumerGroup + ", messageQueue=" + messageQueue + + ", offsetRequest=" + offsetRequest + ", offsetNew=" + offsetNew + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/ConcurrentTreeMap.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/ConcurrentTreeMap.java new file mode 100644 index 0000000000..8fc4e76f72 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/ConcurrentTreeMap.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.queue; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * thread safe + * + * @author lansheng.zj + */ +public class ConcurrentTreeMap { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final ReentrantLock lock; + private TreeMap tree; + private RoundQueue roundQueue; + + + public ConcurrentTreeMap(int capacity, Comparator comparator) { + tree = new TreeMap(comparator); + roundQueue = new RoundQueue(capacity); + lock = new ReentrantLock(true); + } + + + public Map.Entry pollFirstEntry() { + lock.lock(); + try { + return tree.pollFirstEntry(); + } finally { + lock.unlock(); + } + } + + + public V putIfAbsentAndRetExsit(K key, V value) { + lock.lock(); + try { + if (roundQueue.put(key)) { + V exsit = tree.get(key); + if (null == exsit) { + tree.put(key, value); + exsit = value; + } + log.warn("putIfAbsentAndRetExsit success. {}", key); + return exsit; + } + + else { + V exsit = tree.get(key); + return exsit; + } + } finally { + lock.unlock(); + } + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/RoundQueue.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/RoundQueue.java new file mode 100644 index 0000000000..a3783bad7f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/queue/RoundQueue.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.queue; + +import java.util.LinkedList; +import java.util.Queue; + + +/** + * not thread safe + * + * @author lansheng.zj + */ +public class RoundQueue { + + private Queue queue; + private int capacity; + + + public RoundQueue(int capacity) { + this.capacity = capacity; + queue = new LinkedList(); + } + + + public boolean put(E e) { + boolean ok = false; + if (!queue.contains(e)) { + if (queue.size() >= capacity) { + queue.poll(); + } + queue.add(e); + ok = true; + } + + return ok; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java new file mode 100644 index 0000000000..aa0bc549c7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.running; + +public enum RunningStats { + commitLogMaxOffset, + commitLogMinOffset, + commitLogDiskRatio, + consumeQueueDiskRatio, + scheduleMessageOffset, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItem.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItem.java new file mode 100644 index 0000000000..89eefa5b3f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItem.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.stats; + +import com.alibaba.rocketmq.common.UtilAll; +import org.slf4j.Logger; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + + +public class MomentStatsItem { + + private final AtomicLong value = new AtomicLong(0); + + private final String statsName; + private final String statsKey; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + public MomentStatsItem(String statsName, String statsKey, + ScheduledExecutorService scheduledExecutorService, Logger log) { + this.statsName = statsName; + this.statsKey = statsKey; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + } + + + public void init() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + + MomentStatsItem.this.value.set(0); + } catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 5, TimeUnit.MILLISECONDS); + } + + + public void printAtMinutes() { + log.info(String.format("[%s] [%s] Stats Every 5 Minutes, Value: %d", + this.statsName, + this.statsKey, + this.value.get())); + } + + public AtomicLong getValue() { + return value; + } + + + public String getStatsKey() { + return statsKey; + } + + + public String getStatsName() { + return statsName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItemSet.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItemSet.java new file mode 100644 index 0000000000..fde88cdd14 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/MomentStatsItemSet.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.stats; + +import com.alibaba.rocketmq.common.UtilAll; +import org.slf4j.Logger; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class MomentStatsItemSet { + private final ConcurrentHashMap statsItemTable = + new ConcurrentHashMap(128); + private final String statsName; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + public MomentStatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) { + this.statsName = statsName; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + this.init(); + } + + public ConcurrentHashMap getStatsItemTable() { + return statsItemTable; + } + + public String getStatsName() { + return statsName; + } + + public void init() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + } catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 5, TimeUnit.MILLISECONDS); + } + + private void printAtMinutes() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtMinutes(); + } + } + + public void setValue(final String statsKey, final int value) { + MomentStatsItem statsItem = this.getAndCreateStatsItem(statsKey); + statsItem.getValue().set(value); + } + + public MomentStatsItem getAndCreateStatsItem(final String statsKey) { + MomentStatsItem statsItem = this.statsItemTable.get(statsKey); + if (null == statsItem) { + statsItem = + new MomentStatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + MomentStatsItem prev = this.statsItemTable.put(statsKey, statsItem); + + if (null == prev) { + + // statsItem.init(); + } + } + + return statsItem; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java new file mode 100644 index 0000000000..1c99699231 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.stats; + +import com.alibaba.rocketmq.common.UtilAll; +import org.slf4j.Logger; + +import java.util.LinkedList; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + + +public class StatsItem { + + private final AtomicLong value = new AtomicLong(0); + + private final AtomicLong times = new AtomicLong(0); + + private final LinkedList csListMinute = new LinkedList(); + + + private final LinkedList csListHour = new LinkedList(); + + + private final LinkedList csListDay = new LinkedList(); + + private final String statsName; + private final String statsKey; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + public StatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService, + Logger log) { + this.statsName = statsName; + this.statsKey = statsKey; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + } + + public StatsSnapshot getStatsDataInMinute() { + return computeStatsData(this.csListMinute); + } + + private static StatsSnapshot computeStatsData(final LinkedList csList) { + StatsSnapshot statsSnapshot = new StatsSnapshot(); + synchronized (csList) { + double tps = 0; + double avgpt = 0; + long sum = 0; + if (!csList.isEmpty()) { + CallSnapshot first = csList.getFirst(); + CallSnapshot last = csList.getLast(); + sum = last.getValue() - first.getValue(); + tps = (sum * 1000.0d) / (last.getTimestamp() - first.getTimestamp()); + + long timesDiff = last.getTimes() - first.getTimes(); + if (timesDiff > 0) { + avgpt = (sum * 1.0d) / timesDiff; + } + } + + statsSnapshot.setSum(sum); + statsSnapshot.setTps(tps); + statsSnapshot.setAvgpt(avgpt); + } + + return statsSnapshot; + } + + public StatsSnapshot getStatsDataInHour() { + return computeStatsData(this.csListHour); + } + + public StatsSnapshot getStatsDataInDay() { + return computeStatsData(this.csListDay); + } + + public void init() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInSeconds(); + } catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.SECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInMinutes(); + } catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.MINUTES); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInHour(); + } catch (Throwable e) { + } + } + }, 0, 1, TimeUnit.HOURS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + } catch (Throwable ignored) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60, TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtHour(); + } catch (Throwable ignored) { + } + } + }, Math.abs(UtilAll.computNextHourTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60, TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtDay(); + } catch (Throwable ignored) { + } + } + }, Math.abs(UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis()) - 2000, 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS); + } + + public void samplingInSeconds() { + synchronized (this.csListMinute) { + this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListMinute.size() > 7) { + this.csListMinute.removeFirst(); + } + } + } + + public void samplingInMinutes() { + synchronized (this.csListHour) { + this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListHour.size() > 7) { + this.csListHour.removeFirst(); + } + } + } + + public void samplingInHour() { + synchronized (this.csListDay) { + this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListDay.size() > 25) { + this.csListDay.removeFirst(); + } + } + } + + public void printAtMinutes() { + StatsSnapshot ss = computeStatsData(this.csListMinute); + log.info(String.format("[%s] [%s] Stats In One Minute, SUM: %d TPS: %.2f AVGPT: %.2f", + this.statsName, + this.statsKey, + ss.getSum(), + ss.getTps(), + ss.getAvgpt())); + } + + public void printAtHour() { + StatsSnapshot ss = computeStatsData(this.csListHour); + log.info(String.format("[%s] [%s] Stats In One Hour, SUM: %d TPS: %.2f AVGPT: %.2f", + this.statsName, + this.statsKey, + ss.getSum(), + ss.getTps(), + ss.getAvgpt())); + } + + public void printAtDay() { + StatsSnapshot ss = computeStatsData(this.csListDay); + log.info(String.format("[%s] [%s] Stats In One Day, SUM: %d TPS: %.2f AVGPT: %.2f", + this.statsName, + this.statsKey, + ss.getSum(), + ss.getTps(), + ss.getAvgpt())); + } + + public AtomicLong getValue() { + return value; + } + + + public String getStatsKey() { + return statsKey; + } + + + public String getStatsName() { + return statsName; + } + + + public AtomicLong getTimes() { + return times; + } +} + + +class CallSnapshot { + private final long timestamp; + private final long times; + + private final long value; + + + public CallSnapshot(long timestamp, long times, long value) { + super(); + this.timestamp = timestamp; + this.times = times; + this.value = value; + } + + + public long getTimestamp() { + return timestamp; + } + + + public long getTimes() { + return times; + } + + + public long getValue() { + return value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java new file mode 100644 index 0000000000..8a2b2a14c2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.stats; + +import com.alibaba.rocketmq.common.UtilAll; +import org.slf4j.Logger; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class StatsItemSet { + private final ConcurrentHashMap statsItemTable = + new ConcurrentHashMap(128); + + private final String statsName; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + public StatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) { + this.statsName = statsName; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + this.init(); + } + + public void init() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInSeconds(); + } catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.SECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInMinutes(); + } catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.MINUTES); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInHour(); + } catch (Throwable e) { + } + } + }, 0, 1, TimeUnit.HOURS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + } catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), 1000 * 60, TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtHour(); + } catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextHourTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60, TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtDay(); + } catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis()), 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS); + } + + private void samplingInSeconds() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInSeconds(); + } + } + + private void samplingInMinutes() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInMinutes(); + } + } + + private void samplingInHour() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInHour(); + } + } + + private void printAtMinutes() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtMinutes(); + } + } + + private void printAtHour() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtHour(); + } + } + + private void printAtDay() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtDay(); + } + } + + public void addValue(final String statsKey, final int incValue, final int incTimes) { + StatsItem statsItem = this.getAndCreateStatsItem(statsKey); + statsItem.getValue().addAndGet(incValue); + statsItem.getTimes().addAndGet(incTimes); + } + + public StatsItem getAndCreateStatsItem(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null == statsItem) { + statsItem = new StatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + StatsItem prev = this.statsItemTable.put(statsKey, statsItem); + + if (null == prev) { + + // statsItem.init(); + } + } + + return statsItem; + } + + public StatsSnapshot getStatsDataInMinute(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInMinute(); + } + return new StatsSnapshot(); + } + + public StatsSnapshot getStatsDataInHour(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInHour(); + } + return new StatsSnapshot(); + } + + public StatsSnapshot getStatsDataInDay(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInDay(); + } + return new StatsSnapshot(); + } + + public StatsItem getStatsItem(final String statsKey) { + return this.statsItemTable.get(statsKey); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java new file mode 100644 index 0000000000..4092a2b846 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.stats; + +public class StatsSnapshot { + private long sum; + private double tps; + private double avgpt; + + + public long getSum() { + return sum; + } + + + public void setSum(long sum) { + this.sum = sum; + } + + + public double getTps() { + return tps; + } + + + public void setTps(double tps) { + this.tps = tps; + } + + + public double getAvgpt() { + return avgpt; + } + + + public void setAvgpt(double avgpt) { + this.avgpt = avgpt; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java new file mode 100644 index 0000000000..cf8baf29da --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.subscription; + +import com.alibaba.rocketmq.common.MixAll; + + +/** + * @author shijia.wxr + */ +public class SubscriptionGroupConfig { + + private String groupName; + + private boolean consumeEnable = true; + private boolean consumeFromMinEnable = true; + + private boolean consumeBroadcastEnable = true; + + private int retryQueueNums = 1; + + private int retryMaxTimes = 16; + + private long brokerId = MixAll.MASTER_ID; + + private long whichBrokerWhenConsumeSlowly = 1; + + private boolean notifyConsumerIdsChangedEnable = true; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + public boolean isConsumeEnable() { + return consumeEnable; + } + + + public void setConsumeEnable(boolean consumeEnable) { + this.consumeEnable = consumeEnable; + } + + + public boolean isConsumeFromMinEnable() { + return consumeFromMinEnable; + } + + + public void setConsumeFromMinEnable(boolean consumeFromMinEnable) { + this.consumeFromMinEnable = consumeFromMinEnable; + } + + + public boolean isConsumeBroadcastEnable() { + return consumeBroadcastEnable; + } + + + public void setConsumeBroadcastEnable(boolean consumeBroadcastEnable) { + this.consumeBroadcastEnable = consumeBroadcastEnable; + } + + + public int getRetryQueueNums() { + return retryQueueNums; + } + + + public void setRetryQueueNums(int retryQueueNums) { + this.retryQueueNums = retryQueueNums; + } + + + public int getRetryMaxTimes() { + return retryMaxTimes; + } + + + public void setRetryMaxTimes(int retryMaxTimes) { + this.retryMaxTimes = retryMaxTimes; + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public long getWhichBrokerWhenConsumeSlowly() { + return whichBrokerWhenConsumeSlowly; + } + + + public void setWhichBrokerWhenConsumeSlowly(long whichBrokerWhenConsumeSlowly) { + this.whichBrokerWhenConsumeSlowly = whichBrokerWhenConsumeSlowly; + } + + public boolean isNotifyConsumerIdsChangedEnable() { + return notifyConsumerIdsChangedEnable; + } + + public void setNotifyConsumerIdsChangedEnable(final boolean notifyConsumerIdsChangedEnable) { + this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (brokerId ^ (brokerId >>> 32)); + result = prime * result + (consumeBroadcastEnable ? 1231 : 1237); + result = prime * result + (consumeEnable ? 1231 : 1237); + result = prime * result + (consumeFromMinEnable ? 1231 : 1237); + result = prime * result + (notifyConsumerIdsChangedEnable ? 1231 : 1237); + result = prime * result + ((groupName == null) ? 0 : groupName.hashCode()); + result = prime * result + retryMaxTimes; + result = prime * result + retryQueueNums; + result = + prime * result + (int) (whichBrokerWhenConsumeSlowly ^ (whichBrokerWhenConsumeSlowly >>> 32)); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SubscriptionGroupConfig other = (SubscriptionGroupConfig) obj; + if (brokerId != other.brokerId) + return false; + if (consumeBroadcastEnable != other.consumeBroadcastEnable) + return false; + if (consumeEnable != other.consumeEnable) + return false; + if (consumeFromMinEnable != other.consumeFromMinEnable) + return false; + if (groupName == null) { + if (other.groupName != null) + return false; + } else if (!groupName.equals(other.groupName)) + return false; + if (retryMaxTimes != other.retryMaxTimes) + return false; + if (retryQueueNums != other.retryQueueNums) + return false; + if (whichBrokerWhenConsumeSlowly != other.whichBrokerWhenConsumeSlowly) + return false; + if (notifyConsumerIdsChangedEnable != other.notifyConsumerIdsChangedEnable) + return false; + return true; + } + + + @Override + public String toString() { + return "SubscriptionGroupConfig [groupName=" + groupName + ", consumeEnable=" + consumeEnable + + ", consumeFromMinEnable=" + consumeFromMinEnable + ", consumeBroadcastEnable=" + + consumeBroadcastEnable + ", retryQueueNums=" + retryQueueNums + ", retryMaxTimes=" + + retryMaxTimes + ", brokerId=" + brokerId + ", whichBrokerWhenConsumeSlowly=" + + whichBrokerWhenConsumeSlowly + ", notifyConsumerIdsChangedEnable=" + + notifyConsumerIdsChangedEnable + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java new file mode 100644 index 0000000000..2f9d057f61 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * @author shijia.wxr + */ +public class MessageSysFlag { + public final static int COMPRESSED_FLAG = 0x1; + public final static int MULTI_TAGS_FLAG = 0x1 << 1; + public final static int TRANSACTION_NOT_TYPE = 0; + public final static int TRANSACTION_PREPARED_TYPE = 0x1 << 2; + public final static int TRANSACTION_COMMIT_TYPE = 0x2 << 2; + public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2; + + + public static int getTransactionValue(final int flag) { + return flag & TRANSACTION_ROLLBACK_TYPE; + } + + + public static int resetTransactionValue(final int flag, final int type) { + return (flag & (~TRANSACTION_ROLLBACK_TYPE)) | type; + } + + + public static int clearCompressedFlag(final int flag) { + return flag & (~COMPRESSED_FLAG); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java new file mode 100644 index 0000000000..d0f7287c6c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * @author shijia.wxr + */ +public class PullSysFlag { + private final static int FLAG_COMMIT_OFFSET = 0x1 << 0; + private final static int FLAG_SUSPEND = 0x1 << 1; + private final static int FLAG_SUBSCRIPTION = 0x1 << 2; + private final static int FLAG_CLASS_FILTER = 0x1 << 3; + + + public static int buildSysFlag(final boolean commitOffset, final boolean suspend, + final boolean subscription, final boolean classFilter) { + int flag = 0; + + if (commitOffset) { + flag |= FLAG_COMMIT_OFFSET; + } + + if (suspend) { + flag |= FLAG_SUSPEND; + } + + if (subscription) { + flag |= FLAG_SUBSCRIPTION; + } + + if (classFilter) { + flag |= FLAG_CLASS_FILTER; + } + + return flag; + } + + + public static int clearCommitOffsetFlag(final int sysFlag) { + return sysFlag & (~FLAG_COMMIT_OFFSET); + } + + + public static boolean hasCommitOffsetFlag(final int sysFlag) { + return (sysFlag & FLAG_COMMIT_OFFSET) == FLAG_COMMIT_OFFSET; + } + + + public static boolean hasSuspendFlag(final int sysFlag) { + return (sysFlag & FLAG_SUSPEND) == FLAG_SUSPEND; + } + + + public static boolean hasSubscriptionFlag(final int sysFlag) { + return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION; + } + + + public static boolean hasClassFilterFlag(final int sysFlag) { + return (sysFlag & FLAG_CLASS_FILTER) == FLAG_CLASS_FILTER; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java new file mode 100644 index 0000000000..65e3115369 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * @author manhong.yqd + */ +public class SubscriptionSysFlag { + + private final static int FLAG_UNIT = 0x1 << 0; + + + public static int buildSysFlag(final boolean unit) { + int sysFlag = 0; + + if (unit) { + sysFlag |= FLAG_UNIT; + } + + return sysFlag; + } + + + public static int setUnitFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT; + } + + + public static int clearUnitFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT); + } + + + public static boolean hasUnitFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT) == FLAG_UNIT; + } + + + public static void main(String[] args) { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java new file mode 100644 index 0000000000..90d48f4056 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + + * + * @author manhong.yqd + * + */ +public class TopicSysFlag { + + private final static int FLAG_UNIT = 0x1 << 0; + + private final static int FLAG_UNIT_SUB = 0x1 << 1; + + + public static int buildSysFlag(final boolean unit, final boolean hasUnitSub) { + int sysFlag = 0; + + if (unit) { + sysFlag |= FLAG_UNIT; + } + + if (hasUnitSub) { + sysFlag |= FLAG_UNIT_SUB; + } + + return sysFlag; + } + + + public static int setUnitFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT; + } + + + public static int clearUnitFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT); + } + + + public static boolean hasUnitFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT) == FLAG_UNIT; + } + + + public static int setUnitSubFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT_SUB; + } + + + public static int clearUnitSubFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT_SUB); + } + + + public static boolean hasUnitSubFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT_SUB) == FLAG_UNIT_SUB; + } + + + public static void main(String[] args) { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/ChannelUtil.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/ChannelUtil.java new file mode 100644 index 0000000000..444928f54c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/ChannelUtil.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.utils; + +import io.netty.channel.Channel; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +public class ChannelUtil { + public static String getRemoteIp(Channel channel) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) channel.remoteAddress(); + if (inetSocketAddress == null) { + return ""; + } + final InetAddress inetAddr = inetSocketAddress.getAddress(); + return inetAddr != null ? inetAddr.getHostAddress() : inetSocketAddress.getHostName(); + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java new file mode 100755 index 0000000000..dadac46a33 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.utils; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.Iterator; +import java.util.List; + + +public class HttpTinyClient { + + static public HttpResult httpGet(String url, List headers, List paramValues, + String encoding, long readTimeoutMs) throws IOException { + String encodedContent = encodingParams(paramValues, encoding); + url += (null == encodedContent) ? "" : ("?" + encodedContent); + + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout((int) readTimeoutMs); + conn.setReadTimeout((int) readTimeoutMs); + setHeaders(conn, headers, encoding); + + conn.connect(); + int respCode = conn.getResponseCode(); + String resp = null; + + if (HttpURLConnection.HTTP_OK == respCode) { + resp = IOTinyUtils.toString(conn.getInputStream(), encoding); + } else { + resp = IOTinyUtils.toString(conn.getErrorStream(), encoding); + } + return new HttpResult(respCode, resp); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + static private String encodingParams(List paramValues, String encoding) + throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(); + if (null == paramValues) { + return null; + } + + for (Iterator iter = paramValues.iterator(); iter.hasNext(); ) { + sb.append(iter.next()).append("="); + sb.append(URLEncoder.encode(iter.next(), encoding)); + if (iter.hasNext()) { + sb.append("&"); + } + } + return sb.toString(); + } + + static private void setHeaders(HttpURLConnection conn, List headers, String encoding) { + if (null != headers) { + for (Iterator iter = headers.iterator(); iter.hasNext(); ) { + conn.addRequestProperty(iter.next(), iter.next()); + } + } + conn.addRequestProperty("Client-Version", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)); + conn.addRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + encoding); + + + String ts = String.valueOf(System.currentTimeMillis()); + conn.addRequestProperty("Metaq-Client-RequestTS", ts); + } + + /** + + * + * @param url + * @param headers + + * @param paramValues + + * @param encoding + + * @param readTimeoutMs + + * + * @return the http response of given http post request + * + * @throws java.io.IOException + */ + static public HttpResult httpPost(String url, List headers, List paramValues, + String encoding, long readTimeoutMs) throws IOException { + String encodedContent = encodingParams(paramValues, encoding); + + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(3000); + conn.setReadTimeout((int) readTimeoutMs); + conn.setDoOutput(true); + conn.setDoInput(true); + setHeaders(conn, headers, encoding); + + conn.getOutputStream().write(encodedContent.getBytes(MixAll.DEFAULT_CHARSET)); + + int respCode = conn.getResponseCode(); + String resp = null; + + if (HttpURLConnection.HTTP_OK == respCode) { + resp = IOTinyUtils.toString(conn.getInputStream(), encoding); + } else { + resp = IOTinyUtils.toString(conn.getErrorStream(), encoding); + } + return new HttpResult(respCode, resp); + } finally { + if (null != conn) { + conn.disconnect(); + } + } + } + + static public class HttpResult { + final public int code; + final public String content; + + + public HttpResult(int code, String content) { + this.code = code; + this.content = content; + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java new file mode 100644 index 0000000000..ced2fae7d2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.utils; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +import java.io.*; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author manhong.yqd + */ +public class IOTinyUtils { + + static public String toString(InputStream input, String encoding) throws IOException { + return (null == encoding) ? toString(new InputStreamReader(input, RemotingHelper.DEFAULT_CHARSET)) : toString(new InputStreamReader( + input, encoding)); + } + + + static public String toString(Reader reader) throws IOException { + CharArrayWriter sw = new CharArrayWriter(); + copy(reader, sw); + return sw.toString(); + } + + + static public long copy(Reader input, Writer output) throws IOException { + char[] buffer = new char[1 << 12]; + long count = 0; + for (int n = 0; (n = input.read(buffer)) >= 0; ) { + output.write(buffer, 0, n); + count += n; + } + return count; + } + + + /** + + */ + static public List readLines(Reader input) throws IOException { + BufferedReader reader = toBufferedReader(input); + List list = new ArrayList(); + String line = null; + for (;;) { + line = reader.readLine(); + if (null != line) { + list.add(line); + } else { + break; + } + } + return list; + } + + + static private BufferedReader toBufferedReader(Reader reader) { + return reader instanceof BufferedReader ? (BufferedReader) reader : new BufferedReader(reader); + } + + + static public void copyFile(String source, String target) throws IOException { + File sf = new File(source); + if (!sf.exists()) { + throw new IllegalArgumentException("source file does not exist."); + } + File tf = new File(target); + tf.getParentFile().mkdirs(); + if (!tf.exists() && !tf.createNewFile()) { + throw new RuntimeException("failed to create target file."); + } + + FileChannel sc = null; + FileChannel tc = null; + try { + tc = new FileOutputStream(tf).getChannel(); + sc = new FileInputStream(sf).getChannel(); + sc.transferTo(0, sc.size(), tc); + } finally { + if (null != sc) { + sc.close(); + } + if (null != tc) { + tc.close(); + } + } + } + + + public static void delete(File fileOrDir) throws IOException { + if (fileOrDir == null) { + return; + } + + if (fileOrDir.isDirectory()) { + cleanDirectory(fileOrDir); + } + + fileOrDir.delete(); + } + + + /** + + */ + public static void cleanDirectory(File directory) throws IOException { + if (!directory.exists()) { + String message = directory + " does not exist"; + throw new IllegalArgumentException(message); + } + + if (!directory.isDirectory()) { + String message = directory + " is not a directory"; + throw new IllegalArgumentException(message); + } + + File[] files = directory.listFiles(); + if (files == null) { // null if security restricted + throw new IOException("Failed to list contents of " + directory); + } + + IOException exception = null; + for (File file : files) { + try { + delete(file); + } catch (IOException ioe) { + exception = ioe; + } + } + + if (null != exception) { + throw exception; + } + } + + + public static void writeStringToFile(File file, String data, String encoding) throws IOException { + OutputStream os = null; + try { + os = new FileOutputStream(file); + os.write(data.getBytes(encoding)); + } finally { + if (null != os) { + os.close(); + } + } + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java new file mode 100644 index 0000000000..72e02d0fd4 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import junit.framework.Assert; +import org.junit.Test; + +import java.net.InetAddress; +import java.util.List; + + +/** + * @author lansheng.zj + */ +public class MixAllTest { + + @Test + public void test() throws Exception { + List localInetAddress = MixAll.getLocalInetAddress(); + String local = InetAddress.getLocalHost().getHostAddress(); + Assert.assertTrue(localInetAddress.contains("127.0.0.1")); + Assert.assertTrue(localInetAddress.contains(local)); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java new file mode 100644 index 0000000000..e6468b9719 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import org.junit.Test; + + +public class RemotingUtilTest { + @Test + public void test() throws Exception { + String a = RemotingUtil.getLocalAddress(); + System.out.println(a); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java new file mode 100644 index 0000000000..7764fcccfc --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common; + +import org.junit.Test; + +import java.net.URL; +import java.util.Properties; + +import static org.junit.Assert.assertTrue; + + +public class UtilAllTest { + + @Test + public void test_currentStackTrace() { + System.out.println(UtilAll.currentStackTrace()); + } + + + @Test + public void test_a() { + URL url = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + System.out.println(url); + System.out.println(url.getPath()); + } + + + @Test + public void test_resetClassProperties() { + DemoConfig demoConfig = new DemoConfig(); + MixAll.properties2Object(new Properties(), demoConfig); + } + + + @Test + public void test_properties2String() { + DemoConfig demoConfig = new DemoConfig(); + Properties properties = MixAll.object2Properties(demoConfig); + System.out.println(MixAll.properties2String(properties)); + } + + + @Test + public void test_timeMillisToHumanString() { + System.out.println(UtilAll.timeMillisToHumanString()); + } + + + @Test + public void test_isPropertiesEqual() { + final Properties p1 = new Properties(); + final Properties p2 = new Properties(); + + p1.setProperty("a", "1"); + p1.setProperty("b", "2"); + + p2.setProperty("a", "1"); + p2.setProperty("b", "2"); + // p2.setProperty("c", "3"); + + assertTrue(MixAll.isPropertiesEqual(p1, p2)); + } + + + @Test + public void test_getpid() { + int pid = UtilAll.getPid(); + + System.out.println("PID = " + pid); + assertTrue(pid > 0); + } + + + @Test + public void test_isBlank() { + { + boolean result = UtilAll.isBlank("Hello "); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank(" Hello"); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank("He llo"); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank(" "); + assertTrue(result); + } + + { + boolean result = UtilAll.isBlank("Hello"); + assertTrue(!result); + } + } + + static class DemoConfig { + private int demoWidth = 0; + private int demoLength = 0; + private boolean demoOK = false; + private String demoName = "haha"; + + + public int getDemoWidth() { + return demoWidth; + } + + + public void setDemoWidth(int demoWidth) { + this.demoWidth = demoWidth; + } + + + public int getDemoLength() { + return demoLength; + } + + + public void setDemoLength(int demoLength) { + this.demoLength = demoLength; + } + + + public boolean isDemoOK() { + return demoOK; + } + + + public void setDemoOK(boolean demoOK) { + this.demoOK = demoOK; + } + + + public String getDemoName() { + return demoName; + } + + + public void setDemoNfieldame(String demoName) { + this.demoName = demoName; + } + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java new file mode 100644 index 0000000000..e45873bcd6 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import org.junit.Test; + + +/** + * @author shijia.wxr + * + */ +public class FilterAPITest { + + @Test + public void testBuildSubscriptionData() throws Exception { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData("ConsumerGroup1", "TestTopic", "TAG1 || Tag2 || tag3"); + System.out.println(subscriptionData); + } + + @Test + public void testSubscriptionData() throws Exception { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData("ConsumerGroup1", "TestTopic", "TAG1 || Tag2 || tag3"); + subscriptionData.setFilterClassSource("java hello"); + String json = RemotingSerializable.toJson(subscriptionData, true); + System.out.println(json); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java new file mode 100644 index 0000000000..612df6964a --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.filter; + +import com.alibaba.rocketmq.common.filter.impl.Op; +import com.alibaba.rocketmq.common.filter.impl.PolishExpr; +import junit.framework.Assert; +import org.junit.Test; + +import java.util.List; + + +/** + * @author lansheng.zj + */ +public class PolishExprTest { + + private String expression = "tag1||(tag2&&tag3)&&tag4||tag5&&(tag6 && tag7)|| tag8 && tag9"; + private PolishExpr polishExpr; + + + public void init() { + polishExpr = new PolishExpr(); + } + + + @Test + public void testReversePolish() { + List antiPolishExpression = polishExpr.reversePolish(expression); + System.out.println(antiPolishExpression); + } + + + @Test + public void testReversePolish_Performance() { + // prepare + for (int i = 0; i < 100000; i++) { + polishExpr.reversePolish(expression); + } + + long start = System.currentTimeMillis(); + for (int i = 0; i < 100000; i++) { + polishExpr.reversePolish(expression); + } + long cost = System.currentTimeMillis() - start; + System.out.println(cost); + // System.out.println(cost / 100000F); + + Assert.assertTrue(cost < 500); + } + +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java new file mode 100644 index 0000000000..32e3d98c07 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol; + +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import org.junit.Test; + + +public class ConsumeStatusTest { + + @Test + public void decode_test() throws Exception { + ConsumeStatus cs = new ConsumeStatus(); + cs.setConsumeFailedTPS(0L); + String json = RemotingSerializable.toJson(cs, true); + System.out.println(json); + ConsumeStatus fromJson = RemotingSerializable.fromJson(json, ConsumeStatus.class); + } + +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java new file mode 100644 index 0000000000..749e7dfc09 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.common.protocol; + +/** + * @author shijia.wxr + */ +public class MQProtosHelperTest { + +} diff --git a/rocketmq-example/pom.xml b/rocketmq-example/pom.xml new file mode 100644 index 0000000000..8e68a58ecd --- /dev/null +++ b/rocketmq-example/pom.xml @@ -0,0 +1,58 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-example + rocketmq-example ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + org.javassist + javassist + + + diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java new file mode 100644 index 0000000000..71505130f0 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.example.benchmark; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; + +import java.util.LinkedList; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicLong; + +public class Consumer { + + public static void main(String[] args) throws MQClientException { + Options options = ServerUtil.buildCommandlineOptions(new Options()); + CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkConsumer", args, buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + System.exit(-1); + } + + final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; + final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer"; + final String isPrefixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; + String group = groupPrefix; + if (Boolean.parseBoolean(isPrefixEnable)) { + group = groupPrefix + "_" + Long.toString(System.currentTimeMillis() % 100); + } + + System.out.printf("topic %s group %s prefix %s%n", topic, group, isPrefixEnable); + + final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmarkConsumer.createSnapshot()); + if (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long consumeTps = + (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L); + final double averageB2CRT = (end[2] - begin[2]) / (double) (end[1] - begin[1]); + final double averageS2CRT = (end[3] - begin[3]) / (double) (end[1] - begin[1]); + + System.out.printf("Consume TPS: %d Average(B2C) RT: %7.3f Average(S2C) RT: %7.3f MAX(B2C) RT: %d MAX(S2C) RT: %d%n", + consumeTps, averageB2CRT, averageS2CRT, end[4], end[5] + ); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + consumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + consumer.subscribe(topic, "*"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + MessageExt msg = msgs.get(0); + long now = System.currentTimeMillis(); + + statsBenchmarkConsumer.getReceiveMessageTotalCount().incrementAndGet(); + + long born2ConsumerRT = now - msg.getBornTimestamp(); + statsBenchmarkConsumer.getBorn2ConsumerTotalRT().addAndGet(born2ConsumerRT); + + long store2ConsumerRT = now - msg.getStoreTimestamp(); + statsBenchmarkConsumer.getStore2ConsumerTotalRT().addAndGet(store2ConsumerRT); + + compareAndSetMax(statsBenchmarkConsumer.getBorn2ConsumerMaxRT(), born2ConsumerRT); + + compareAndSetMax(statsBenchmarkConsumer.getStore2ConsumerMaxRT(), store2ConsumerRT); + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.printf("Consumer Started.%n"); + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "group", true, "Consumer group name, Default: benchmark_consumer"); + opt.setRequired(false); + options.addOption(opt); + + + opt = new Option("p", "group prefix enable", true, "Consumer group name, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void compareAndSetMax(final AtomicLong target, final long value) { + long prev = target.get(); + while (value > prev) { + boolean updated = target.compareAndSet(prev, value); + if (updated) + break; + + prev = target.get(); + } + } +} + + +class StatsBenchmarkConsumer { + private final AtomicLong receiveMessageTotalCount = new AtomicLong(0L); + + private final AtomicLong born2ConsumerTotalRT = new AtomicLong(0L); + + private final AtomicLong store2ConsumerTotalRT = new AtomicLong(0L); + + private final AtomicLong born2ConsumerMaxRT = new AtomicLong(0L); + + private final AtomicLong store2ConsumerMaxRT = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[]{ + System.currentTimeMillis(), + this.receiveMessageTotalCount.get(), + this.born2ConsumerTotalRT.get(), + this.store2ConsumerTotalRT.get(), + this.born2ConsumerMaxRT.get(), + this.store2ConsumerMaxRT.get(), + }; + + return snap; + } + + + public AtomicLong getReceiveMessageTotalCount() { + return receiveMessageTotalCount; + } + + + public AtomicLong getBorn2ConsumerTotalRT() { + return born2ConsumerTotalRT; + } + + + public AtomicLong getStore2ConsumerTotalRT() { + return store2ConsumerTotalRT; + } + + + public AtomicLong getBorn2ConsumerMaxRT() { + return born2ConsumerMaxRT; + } + + + public AtomicLong getStore2ConsumerMaxRT() { + return store2ConsumerMaxRT; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java new file mode 100644 index 0000000000..b0351c62a7 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.benchmark; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +public class Producer { + public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { + + Options options = ServerUtil.buildCommandlineOptions(new Options()); + CommandLine commandLine = ServerUtil.parseCmdLine("benchmarkProducer", args, buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + System.exit(-1); + } + + final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; + final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 64; + final int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 128; + final boolean keyEnable = commandLine.hasOption('k') ? Boolean.parseBoolean(commandLine.getOptionValue('k')) : false; + + System.out.printf("topic %s threadCount %d messageSize %d keyEnable %s%n", topic, threadCount, messageSize, keyEnable); + + final Logger log = ClientLogger.getLog(); + + final Message msg = buildMessage(messageSize, topic); + + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + + final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmark.createSnapshot()); + if (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + System.out.printf("Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d%n", + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + final DefaultMQProducer producer = new DefaultMQProducer("benchmark_producer"); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + + if (commandLine.hasOption('n')) { + String ns = commandLine.getOptionValue('n'); + producer.setNamesrvAddr(ns); + } + + producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); + + producer.start(); + + for (int i = 0; i < threadCount; i++) { + sendThreadPool.execute(new Runnable() { + @Override + public void run() { + while (true) { + try { + final long beginTimestamp = System.currentTimeMillis(); + if (keyEnable) { + msg.setKeys(String.valueOf(beginTimestamp / 1000)); + } + producer.send(msg); + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); + final long currentRT = System.currentTimeMillis() - beginTimestamp; + statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + while (currentRT > prevMaxRT) { + boolean updated = statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, currentRT); + if (updated) + break; + + prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + } + } catch (RemotingException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + } + } catch (InterruptedException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + } + } catch (MQClientException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + } catch (MQBrokerException e) { + statsBenchmark.getReceiveResponseFailedCount().incrementAndGet(); + log.error("[BENCHMARK_PRODUCER] Send Exception", e); + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + } + } + } + } + }); + } + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("w", "threadCount", true, "Thread count, Default: 64"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "messageSize", true, "Message Size, Default: 128"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("k", "keyEnable", true, "Message Key Enable, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + private static Message buildMessage(final int messageSize, final String topic) throws UnsupportedEncodingException { + Message msg = new Message(); + msg.setTopic(topic); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 10) { + sb.append("hello baby"); + } + + msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + + return msg; + } +} + + +class StatsBenchmarkProducer { + private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); + + private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); + + private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L); + + private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L); + + private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + + private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[]{ + System.currentTimeMillis(), + this.sendRequestSuccessCount.get(), + this.sendRequestFailedCount.get(), + this.receiveResponseSuccessCount.get(), + this.receiveResponseFailedCount.get(), + this.sendMessageSuccessTimeTotal.get(), + }; + + return snap; + } + + + public AtomicLong getSendRequestSuccessCount() { + return sendRequestSuccessCount; + } + + + public AtomicLong getSendRequestFailedCount() { + return sendRequestFailedCount; + } + + + public AtomicLong getReceiveResponseSuccessCount() { + return receiveResponseSuccessCount; + } + + + public AtomicLong getReceiveResponseFailedCount() { + return receiveResponseFailedCount; + } + + + public AtomicLong getSendMessageSuccessTimeTotal() { + return sendMessageSuccessTimeTotal; + } + + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java new file mode 100644 index 0000000000..3dffd2fc4a --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.example.benchmark; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.*; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +import java.io.UnsupportedEncodingException; +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +public class TransactionProducer { + private static int threadCount; + private static int messageSize; + private static boolean ischeck; + private static boolean ischeckffalse; + + + public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { + threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; + messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; + ischeck = args.length >= 3 ? Boolean.parseBoolean(args[2]) : false; + ischeckffalse = args.length >= 4 ? Boolean.parseBoolean(args[3]) : false; + + final Message msg = buildMessage(messageSize); + + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + + final StatsBenchmarkTProducer statsBenchmark = new StatsBenchmarkTProducer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmark.createSnapshot()); + while (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = + (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + System.out.printf( + "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d transaction checkCount: %d %n", + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4], end[6]); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + final TransactionCheckListener transactionCheckListener = + new TransactionCheckListenerBImpl(ischeckffalse, statsBenchmark); + final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + producer.setTransactionCheckListener(transactionCheckListener); + producer.setDefaultTopicQueueNums(1000); + producer.start(); + + final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); + + for (int i = 0; i < threadCount; i++) { + sendThreadPool.execute(new Runnable() { + @Override + public void run() { + while (true) { + try { + // Thread.sleep(1000); + final long beginTimestamp = System.currentTimeMillis(); + SendResult sendResult = + producer.sendMessageInTransaction(msg, tranExecuter, null); + if (sendResult != null) { + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); + } + + final long currentRT = System.currentTimeMillis() - beginTimestamp; + statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + while (currentRT > prevMaxRT) { + boolean updated = + statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, + currentRT); + if (updated) + break; + + prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + } + } catch (MQClientException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + } + } + } + }); + } + } + + + private static Message buildMessage(final int messageSize) throws UnsupportedEncodingException { + Message msg = new Message(); + msg.setTopic("BenchmarkTest"); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 10) { + sb.append("hello baby"); + } + + msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + + return msg; + } +} + + +class TransactionExecuterBImpl implements LocalTransactionExecuter { + + private boolean ischeck; + + + public TransactionExecuterBImpl(boolean ischeck) { + this.ischeck = ischeck; + } + + + @Override + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { + if (ischeck) { + return LocalTransactionState.UNKNOW; + } + return LocalTransactionState.COMMIT_MESSAGE; + } +} + + +class TransactionCheckListenerBImpl implements TransactionCheckListener { + private boolean ischeckffalse; + private StatsBenchmarkTProducer statsBenchmarkTProducer; + + + public TransactionCheckListenerBImpl(boolean ischeckffalse, + StatsBenchmarkTProducer statsBenchmarkTProducer) { + this.ischeckffalse = ischeckffalse; + this.statsBenchmarkTProducer = statsBenchmarkTProducer; + } + + + @Override + public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); + if (ischeckffalse) { + + return LocalTransactionState.ROLLBACK_MESSAGE; + } + + return LocalTransactionState.COMMIT_MESSAGE; + } +} + + +class StatsBenchmarkTProducer { + private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); + + private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); + + private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L); + + private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L); + + private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + + private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); + + private final AtomicLong checkRequestSuccessCount = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[]{ + System.currentTimeMillis(), + this.sendRequestSuccessCount.get(), + this.sendRequestFailedCount.get(), + this.receiveResponseSuccessCount.get(), + this.receiveResponseFailedCount.get(), + this.sendMessageSuccessTimeTotal.get(), + this.checkRequestSuccessCount.get()}; + + return snap; + } + + + public AtomicLong getSendRequestSuccessCount() { + return sendRequestSuccessCount; + } + + + public AtomicLong getSendRequestFailedCount() { + return sendRequestFailedCount; + } + + + public AtomicLong getReceiveResponseSuccessCount() { + return receiveResponseSuccessCount; + } + + + public AtomicLong getReceiveResponseFailedCount() { + return receiveResponseFailedCount; + } + + + public AtomicLong getSendMessageSuccessTimeTotal() { + return sendMessageSuccessTimeTotal; + } + + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; + } + + + public AtomicLong getCheckRequestSuccessCount() { + return checkRequestSuccessCount; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java new file mode 100644 index 0000000000..6cc6238a5a --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.broadcast; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + +import java.util.List; + +public class PushConsumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1"); + + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.setMessageModel(MessageModel.BROADCASTING); + + consumer.subscribe("TopicTest", "TagA || TagC || TagD"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n"); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + System.out.printf("Broadcast Consumer Started.%n"); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java new file mode 100644 index 0000000000..104e6d900b --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ConsumerGroupNamecc4"); + + String filterCode = MixAll.file2String("/home/admin/MessageFilterImpl.java"); + consumer.subscribe("TopicFilter7", "com.alibaba.rocketmq.example.filter.MessageFilterImpl", + filterCode); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n"); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.printf("Consumer Started.%n"); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java new file mode 100644 index 0000000000..04251face6 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + producer.start(); + + try { + for (int i = 0; i < 6000000; i++) { + Message msg = new Message("TopicFilter7", + "TagA", + "OrderID001", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + msg.putUserProperty("SequenceId", String.valueOf(i)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } + } catch (Exception e) { + e.printStackTrace(); + } + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java new file mode 100644 index 0000000000..f6ba0675c3 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.operation; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + + +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + CommandLine commandLine = buildCommandline(args); + if (commandLine != null) { + String group = commandLine.getOptionValue('g'); + String topic = commandLine.getOptionValue('t'); + String subscription = commandLine.getOptionValue('s'); + final String returnFailedHalf = commandLine.getOptionValue('f'); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + consumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + consumer.subscribe(topic, subscription); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + AtomicLong consumeTimes = new AtomicLong(0); + + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + long currentTimes = this.consumeTimes.incrementAndGet(); + System.out.printf("%-8d %s%n", currentTimes, msgs); + if (Boolean.parseBoolean(returnFailedHalf)) { + if ((currentTimes % 2) == 0) { + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.printf("Consumer Started.%n"); + } + } + + public static CommandLine buildCommandline(String[] args) { + final Options options = new Options(); + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "Consumer Group Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "subscription", true, "subscription"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("f", "returnFailedHalf", true, "return failed result, for half message"); + opt.setRequired(true); + options.addOption(opt); + + PosixParser parser = new PosixParser(); + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp("producer", options, true); + return null; + } + } catch (ParseException e) { + hf.printHelp("producer", options, true); + return null; + } + + return commandLine; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java new file mode 100644 index 0000000000..816e3e86b7 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.operation; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import org.apache.commons.cli.*; + +public class Producer { + + public static void main(String[] args) throws MQClientException, InterruptedException { + CommandLine commandLine = buildCommandline(args); + if (commandLine != null) { + String group = commandLine.getOptionValue('g'); + String topic = commandLine.getOptionValue('t'); + String tags = commandLine.getOptionValue('a'); + String keys = commandLine.getOptionValue('k'); + String msgCount = commandLine.getOptionValue('c'); + + DefaultMQProducer producer = new DefaultMQProducer(group); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + + producer.start(); + + for (int i = 0; i < Integer.parseInt(msgCount); i++) { + try { + Message msg = new Message( + topic, + tags, + keys, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%-8d %s%n", i, sendResult); + } catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000); + } + } + + producer.shutdown(); + } + } + + public static CommandLine buildCommandline(String[] args) { + final Options options = new Options(); + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "producerGroup", true, "Producer Group Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("a", "tags", true, "Tags Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "keys", true, "Keys Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "msgCount", true, "Message Count"); + opt.setRequired(true); + options.addOption(opt); + + PosixParser parser = new PosixParser(); + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp("producer", options, true); + return null; + } + } catch (ParseException e) { + hf.printHelp("producer", options, true); + return null; + } + + return commandLine; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java new file mode 100644 index 0000000000..7b5f6575a8 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.ordermessage; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + + +public class Consumer { + + public static void main(String[] args) throws MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_3"); + + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.subscribe("TopicTest", "TagA || TagC || TagD"); + + consumer.registerMessageListener(new MessageListenerOrderly() { + AtomicLong consumeTimes = new AtomicLong(0); + + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + context.setAutoCommit(false); + System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n"); + this.consumeTimes.incrementAndGet(); + if ((this.consumeTimes.get() % 2) == 0) { + return ConsumeOrderlyStatus.SUCCESS; + } else if ((this.consumeTimes.get() % 3) == 0) { + return ConsumeOrderlyStatus.ROLLBACK; + } else if ((this.consumeTimes.get() % 4) == 0) { + return ConsumeOrderlyStatus.COMMIT; + } else if ((this.consumeTimes.get() % 5) == 0) { + context.setSuspendCurrentQueueTimeMillis(3000); + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + return ConsumeOrderlyStatus.SUCCESS; + } + }); + + consumer.start(); + System.out.printf("Consumer Started.%n"); + } + +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java new file mode 100644 index 0000000000..609aa62223 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.ordermessage; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.MQProducer; +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import java.io.UnsupportedEncodingException; +import java.util.List; + +public class Producer { + public static void main(String[] args) throws UnsupportedEncodingException { + try { + MQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + producer.start(); + + String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"}; + for (int i = 0; i < 100; i++) { + int orderId = i % 10; + Message msg = + new Message("TopicTestjjj", tags[i % tags.length], "KEY" + i, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg, new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Integer id = (Integer) arg; + int index = id % mqs.size(); + return mqs.get(index); + } + }, orderId); + + System.out.printf("%s%n", sendResult); + } + + producer.shutdown(); + } catch (MQClientException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } catch (MQBrokerException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java new file mode 100644 index 0000000000..adac497142 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.quickstart; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4"); + + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.subscribe("TopicTest", "*"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n"); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java new file mode 100644 index 0000000000..fb5dbea134 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.quickstart; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.LocalTransactionExecuter; +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + producer.start(); + + for (int i = 0; i < 1000; i++) { + try { + Message msg = new Message("TopicTest", + "TagA", + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) + ); + SendResult sendResult = producer.send(msg); + LocalTransactionExecuter tranExecuter = new LocalTransactionExecuter() { + @Override + public LocalTransactionState executeLocalTransactionBranch(Message msg, Object arg) { + return null; + } + }; + System.out.printf("%s%n", sendResult); + } catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000); + } + } + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/AsyncProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/AsyncProducer.java new file mode 100644 index 0000000000..1a8f07ea79 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/AsyncProducer.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendCallback; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +import java.io.UnsupportedEncodingException; + + +public class AsyncProducer { + public static void main(String[] args) throws MQClientException, InterruptedException, UnsupportedEncodingException { + + DefaultMQProducer producer = new DefaultMQProducer("Jodie_Daily_test"); + producer.start(); + producer.setRetryTimesWhenSendAsyncFailed(0); + + for (int i = 0; i < 10000000; i++) { + try { + final int index = i; + Message msg = new Message("Jodie_topic_1023", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + producer.send(msg, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + System.out.printf("%-10d OK %s %n", index, sendResult.getMsgId()); + } + + @Override + public void onException(Throwable e) { + System.out.printf("%-10d Exception %s %n", index, e); + e.printStackTrace(); + } + }); + } catch (Exception e) { + e.printStackTrace(); + } + } + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java new file mode 100644 index 0000000000..7beb064c20 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.TreeMap; + + +public class CachedQueue { + private final TreeMap msgCachedTable = new TreeMap(); + + + public TreeMap getMsgCachedTable() { + return msgCachedTable; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java new file mode 100644 index 0000000000..e0010d4fff --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + + +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + + producer.start(); + + for (int i = 0; i < 10000000; i++) + try { + { + Message msg = new Message("TopicTest", + "TagA", + "OrderID188", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + } + + } catch (Exception e) { + e.printStackTrace(); + } + + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java new file mode 100644 index 0000000000..624576962f --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class PullConsumer { + private static final Map OFFSE_TABLE = new HashMap(); + + + public static void main(String[] args) throws MQClientException { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); + + consumer.start(); + + Set mqs = consumer.fetchSubscribeMessageQueues("TopicTest1"); + for (MessageQueue mq : mqs) { + System.out.printf("Consume from the queue: " + mq + "%n"); + SINGLE_MQ: + while (true) { + try { + PullResult pullResult = + consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32); + System.out.printf("%s%n", pullResult); + putMessageQueueOffset(mq, pullResult.getNextBeginOffset()); + switch (pullResult.getPullStatus()) { + case FOUND: + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break SINGLE_MQ; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + consumer.shutdown(); + } + + private static long getMessageQueueOffset(MessageQueue mq) { + Long offset = OFFSE_TABLE.get(mq); + if (offset != null) + return offset; + + return 0; + } + + private static void putMessageQueueOffset(MessageQueue mq, long offset) { + OFFSE_TABLE.put(mq, offset); + } + +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java new file mode 100644 index 0000000000..25d668c680 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; + +public class PullConsumerTest { + public static void main(String[] args) throws MQClientException { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); + consumer.start(); + + try { + MessageQueue mq = new MessageQueue(); + mq.setQueueId(0); + mq.setTopic("TopicTest3"); + mq.setBrokerName("vivedeMacBook-Pro.local"); + + long offset = 26; + + long beginTime = System.currentTimeMillis(); + PullResult pullResult = consumer.pullBlockIfNotFound(mq, null, offset, 32); + System.out.printf("%s%n", System.currentTimeMillis() - beginTime); + System.out.printf("%s%n", pullResult); + } catch (Exception e) { + e.printStackTrace(); + } + + consumer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java new file mode 100644 index 0000000000..0c86cf8156 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.MQPullConsumer; +import com.alibaba.rocketmq.client.consumer.MQPullConsumerScheduleService; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullTaskCallback; +import com.alibaba.rocketmq.client.consumer.PullTaskContext; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +public class PullScheduleService { + + public static void main(String[] args) throws MQClientException { + final MQPullConsumerScheduleService scheduleService = new MQPullConsumerScheduleService("GroupName1"); + + scheduleService.setMessageModel(MessageModel.CLUSTERING); + scheduleService.registerPullTaskCallback("TopicTest1", new PullTaskCallback() { + + @Override + public void doPullTask(MessageQueue mq, PullTaskContext context) { + MQPullConsumer consumer = context.getPullConsumer(); + try { + + long offset = consumer.fetchConsumeOffset(mq, false); + if (offset < 0) + offset = 0; + + PullResult pullResult = consumer.pull(mq, "*", offset, 32); + System.out.printf("%s%n", offset + "\t" + mq + "\t" + pullResult); + switch (pullResult.getPullStatus()) { + case FOUND: + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break; + default: + break; + } + consumer.updateConsumeOffset(mq, pullResult.getNextBeginOffset()); + + + context.setPullNextDelayTimeMillis(100); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + scheduleService.start(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java new file mode 100644 index 0000000000..5628cedfa1 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.List; + + +public class PushConsumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1"); + consumer.subscribe("Jodie_topic_1023", "*"); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + consumer.registerMessageListener(new MessageListenerConcurrently() { + + /** + + */ + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n"); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java new file mode 100644 index 0000000000..fc6bacda00 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + + +public class RandomAsyncCommit { + private final ConcurrentHashMap mqCachedTable = + new ConcurrentHashMap(); + + + public void putMessages(final MessageQueue mq, final List msgs) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null == cachedQueue) { + cachedQueue = new CachedQueue(); + this.mqCachedTable.put(mq, cachedQueue); + } + for (MessageExt msg : msgs) { + cachedQueue.getMsgCachedTable().put(msg.getQueueOffset(), msg); + } + } + + + public void removeMessage(final MessageQueue mq, long offset) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null != cachedQueue) { + cachedQueue.getMsgCachedTable().remove(offset); + } + } + + + public long commitableOffset(final MessageQueue mq) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null != cachedQueue) { + return cachedQueue.getMsgCachedTable().firstKey(); + } + + return -1; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java new file mode 100644 index 0000000000..68347a6b7a --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +public class TestProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + producer.start(); + + for (int i = 0; i < 1; i++) + try { + { + Message msg = new Message("TopicTest1", + "TagA", + "key113", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.send(msg); + System.out.printf("%s%n", sendResult); + + QueryResult queryMessage = + producer.queryMessage("TopicTest1", "key113", 10, 0, System.currentTimeMillis()); + for (MessageExt m : queryMessage.getMessageList()) { + System.out.printf("%s%n", m); + } + } + + } catch (Exception e) { + e.printStackTrace(); + } + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java new file mode 100644 index 0000000000..2e91e34b8c --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.common.message.MessageExt; + +import java.util.concurrent.atomic.AtomicInteger; + + +public class TransactionCheckListenerImpl implements TransactionCheckListener { + private AtomicInteger transactionIndex = new AtomicInteger(0); + + + @Override + public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + System.out.printf("server checking TrMsg " + msg.toString() + "%n"); + + int value = transactionIndex.getAndIncrement(); + if ((value % 6) == 0) { + throw new RuntimeException("Could not find db"); + } else if ((value % 5) == 0) { + return LocalTransactionState.ROLLBACK_MESSAGE; + } else if ((value % 4) == 0) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + return LocalTransactionState.UNKNOW; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java new file mode 100644 index 0000000000..cda523af9e --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import com.alibaba.rocketmq.client.producer.LocalTransactionExecuter; +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.common.message.Message; + +import java.util.concurrent.atomic.AtomicInteger; + +public class TransactionExecuterImpl implements LocalTransactionExecuter { + private AtomicInteger transactionIndex = new AtomicInteger(1); + + + @Override + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { + int value = transactionIndex.getAndIncrement(); + + if (value == 0) { + throw new RuntimeException("Could not find db"); + } else if ((value % 5) == 0) { + return LocalTransactionState.ROLLBACK_MESSAGE; + } else if ((value % 4) == 0) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + return LocalTransactionState.UNKNOW; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java new file mode 100644 index 0000000000..2c4745f267 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.client.producer.TransactionMQProducer; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + +import java.io.UnsupportedEncodingException; + +public class TransactionProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + TransactionCheckListener transactionCheckListener = new TransactionCheckListenerImpl(); + TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); + producer.setCheckThreadPoolMinSize(2); + producer.setCheckThreadPoolMaxSize(2); + producer.setCheckRequestHoldMax(2000); + producer.setTransactionCheckListener(transactionCheckListener); + producer.start(); + + String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"}; + TransactionExecuterImpl tranExecuter = new TransactionExecuterImpl(); + for (int i = 0; i < 100; i++) { + try { + Message msg = + new Message("TopicTest", tags[i % tags.length], "KEY" + i, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); + SendResult sendResult = producer.sendMessageInTransaction(msg, tranExecuter, null); + System.out.printf("%s%n", sendResult); + + Thread.sleep(10); + } catch (MQClientException e) { + e.printStackTrace(); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < 100000; i++) { + Thread.sleep(1000); + } + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/resources/MessageFilterImpl.java b/rocketmq-example/src/main/resources/MessageFilterImpl.java new file mode 100644 index 0000000000..ea6f3d8917 --- /dev/null +++ b/rocketmq-example/src/main/resources/MessageFilterImpl.java @@ -0,0 +1,22 @@ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.common.filter.MessageFilter; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class MessageFilterImpl implements MessageFilter { + + @Override + public boolean match(MessageExt msg) { + String property = msg.getProperty("SequenceId"); + if (property != null) { + int id = Integer.parseInt(property); + if (((id % 10) == 0) && // + (id > 100)) { + return true; + } + } + + return false; + } +} diff --git a/rocketmq-filtersrv/pom.xml b/rocketmq-filtersrv/pom.xml new file mode 100644 index 0000000000..28c360b386 --- /dev/null +++ b/rocketmq-filtersrv/pom.xml @@ -0,0 +1,62 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-filtersrv + rocketmq-filtersrv ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + org.apache.commons + commons-lang3 + + + diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java new file mode 100644 index 0000000000..b469b3fb93 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + */ +public class FilterServerOuterAPI { + private final RemotingClient remotingClient; + + + public FilterServerOuterAPI() { + this.remotingClient = new NettyRemotingClient(new NettyClientConfig()); + } + + + public void start() { + this.remotingClient.start(); + } + + + public void shutdown() { + this.remotingClient.shutdown(); + } + + + public RegisterFilterServerResponseHeader registerFilterServerToBroker( + final String brokerAddr, + final String filterServerAddr + ) throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + RegisterFilterServerRequestHeader requestHeader = new RegisterFilterServerRequestHeader(); + requestHeader.setFilterServerAddr(filterServerAddr); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.REGISTER_FILTER_SERVER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + RegisterFilterServerResponseHeader responseHeader = + (RegisterFilterServerResponseHeader) response + .decodeCommandCustomHeader(RegisterFilterServerResponseHeader.class); + + return responseHeader; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java new file mode 100644 index 0000000000..fac620f214 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +public class FiltersrvConfig { + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + @ImportantField + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + + private String connectWhichBroker = "127.0.0.1:10911"; + private String filterServerIP = RemotingUtil.getLocalAddress(); + + private int compressMsgBodyOverHowmuch = 1024 * 8; + private int zipCompressLevel = 5; + + + private boolean clientUploadFilterClassEnable = true; + + + private String filterClassRepertoryUrl = "http://fsrep.tbsite.net/filterclass"; + + private int fsServerAsyncSemaphoreValue = 2048; + private int fsServerCallbackExecutorThreads = 64; + private int fsServerWorkerThreads = 64; + + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public String getConnectWhichBroker() { + return connectWhichBroker; + } + + + public void setConnectWhichBroker(String connectWhichBroker) { + this.connectWhichBroker = connectWhichBroker; + } + + + public String getFilterServerIP() { + return filterServerIP; + } + + + public void setFilterServerIP(String filterServerIP) { + this.filterServerIP = filterServerIP; + } + + + public int getCompressMsgBodyOverHowmuch() { + return compressMsgBodyOverHowmuch; + } + + + public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) { + this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch; + } + + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + + public boolean isClientUploadFilterClassEnable() { + return clientUploadFilterClassEnable; + } + + + public void setClientUploadFilterClassEnable(boolean clientUploadFilterClassEnable) { + this.clientUploadFilterClassEnable = clientUploadFilterClassEnable; + } + + + public String getFilterClassRepertoryUrl() { + return filterClassRepertoryUrl; + } + + + public void setFilterClassRepertoryUrl(String filterClassRepertoryUrl) { + this.filterClassRepertoryUrl = filterClassRepertoryUrl; + } + + + public int getFsServerAsyncSemaphoreValue() { + return fsServerAsyncSemaphoreValue; + } + + + public void setFsServerAsyncSemaphoreValue(int fsServerAsyncSemaphoreValue) { + this.fsServerAsyncSemaphoreValue = fsServerAsyncSemaphoreValue; + } + + + public int getFsServerCallbackExecutorThreads() { + return fsServerCallbackExecutorThreads; + } + + + public void setFsServerCallbackExecutorThreads(int fsServerCallbackExecutorThreads) { + this.fsServerCallbackExecutorThreads = fsServerCallbackExecutorThreads; + } + + + public int getFsServerWorkerThreads() { + return fsServerWorkerThreads; + } + + + public void setFsServerWorkerThreads(int fsServerWorkerThreads) { + this.fsServerWorkerThreads = fsServerWorkerThreads; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java new file mode 100644 index 0000000000..0e3f6968bb --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.filtersrv.filter.FilterClassManager; +import com.alibaba.rocketmq.filtersrv.processor.DefaultRequestProcessor; +import com.alibaba.rocketmq.filtersrv.stats.FilterServerStatsManager; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class FiltersrvController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + + private final FiltersrvConfig filtersrvConfig; + + private final NettyServerConfig nettyServerConfig; + private final FilterClassManager filterClassManager; + + private final FilterServerOuterAPI filterServerOuterAPI = new FilterServerOuterAPI(); + private final DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer( + MixAll.FILTERSRV_CONSUMER_GROUP); + + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSScheduledThread")); + private final FilterServerStatsManager filterServerStatsManager = new FilterServerStatsManager(); + + private RemotingServer remotingServer; + + private ExecutorService remotingExecutor; + private volatile String brokerName = null; + + + public FiltersrvController(FiltersrvConfig filtersrvConfig, NettyServerConfig nettyServerConfig) { + this.filtersrvConfig = filtersrvConfig; + this.nettyServerConfig = nettyServerConfig; + this.filterClassManager = new FilterClassManager(this); + } + + + public boolean initialize() { + + MixAll.printObjectProperties(log, this.filtersrvConfig); + + + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig); + + + this.remotingExecutor = + Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), + new ThreadFactoryImpl("RemotingExecutorThread_")); + + this.registerProcessor(); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + FiltersrvController.this.registerFilterServerToBroker(); + } + }, 3, 10, TimeUnit.SECONDS); + + this.defaultMQPullConsumer.setBrokerSuspendMaxTimeMillis(this.defaultMQPullConsumer + .getBrokerSuspendMaxTimeMillis() - 1000); + this.defaultMQPullConsumer.setConsumerTimeoutMillisWhenSuspend(this.defaultMQPullConsumer + .getConsumerTimeoutMillisWhenSuspend() - 1000); + + this.defaultMQPullConsumer.setNamesrvAddr(this.filtersrvConfig.getNamesrvAddr()); + this.defaultMQPullConsumer.setInstanceName(String.valueOf(UtilAll.getPid())); + + return true; + } + + private void registerProcessor() { + this.remotingServer + .registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); + } + + public void registerFilterServerToBroker() { + try { + RegisterFilterServerResponseHeader responseHeader = + this.filterServerOuterAPI.registerFilterServerToBroker( + this.filtersrvConfig.getConnectWhichBroker(), this.localAddr()); + this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() + .setDefaultBrokerId(responseHeader.getBrokerId()); + + if (null == this.brokerName) { + this.brokerName = responseHeader.getBrokerName(); + } + + log.info("register filter server<{}> to broker<{}> OK, Return: {} {}", + this.localAddr(), + this.filtersrvConfig.getConnectWhichBroker(), + responseHeader.getBrokerName(), + responseHeader.getBrokerId()); + } catch (Exception e) { + log.warn("register filter server Exception", e); + + log.warn("access broker failed, kill oneself"); + System.exit(-1); + } + } + + public String localAddr() { + return String.format("%s:%d", this.filtersrvConfig.getFilterServerIP(), + this.remotingServer.localListenPort()); + } + + public void start() throws Exception { + this.defaultMQPullConsumer.start(); + this.remotingServer.start(); + this.filterServerOuterAPI.start(); + this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() + .setConnectBrokerByUser(true); + this.filterClassManager.start(); + this.filterServerStatsManager.start(); + } + + + public void shutdown() { + this.remotingServer.shutdown(); + this.remotingExecutor.shutdown(); + this.scheduledExecutorService.shutdown(); + this.defaultMQPullConsumer.shutdown(); + this.filterServerOuterAPI.shutdown(); + this.filterClassManager.shutdown(); + this.filterServerStatsManager.shutdown(); + } + + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } + + + public ExecutorService getRemotingExecutor() { + return remotingExecutor; + } + + + public void setRemotingExecutor(ExecutorService remotingExecutor) { + this.remotingExecutor = remotingExecutor; + } + + + public FiltersrvConfig getFiltersrvConfig() { + return filtersrvConfig; + } + + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + + public FilterServerOuterAPI getFilterServerOuterAPI() { + return filterServerOuterAPI; + } + + + public FilterClassManager getFilterClassManager() { + return filterClassManager; + } + + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public FilterServerStatsManager getFilterServerStatsManager() { + return filterServerStatsManager; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java new file mode 100644 index 0000000000..3fe6b2200a --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class FiltersrvStartup { + public static Logger log; + + public static void main(String[] args) { + start(createController(args)); + } + + public static FiltersrvController start(FiltersrvController controller) { + + try { + controller.start(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + String tip = "The Filter Server boot success, " + controller.localAddr(); + log.info(tip); + System.out.printf("%s%n", tip); + + return controller; + } + + public static FiltersrvController createController(String[] args) { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE)) { + NettySystemConfig.socketSndbufSize = 65535; + } + + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE)) { + NettySystemConfig.socketRcvbufSize = 1024; + } + + try { + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqfiltersrv", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + final FiltersrvConfig filtersrvConfig = new FiltersrvConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + InputStream in = new BufferedInputStream(new FileInputStream(file)); + Properties properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, filtersrvConfig); + System.out.printf("load config properties file OK, " + file + "%n"); + in.close(); + + String port = properties.getProperty("listenPort"); + if (port != null) { + filtersrvConfig.setConnectWhichBroker(String.format("127.0.0.1:%s", port)); + } + } + } + + nettyServerConfig.setListenPort(0); + nettyServerConfig.setServerAsyncSemaphoreValue(filtersrvConfig.getFsServerAsyncSemaphoreValue()); + nettyServerConfig.setServerCallbackExecutorThreads(filtersrvConfig + .getFsServerCallbackExecutorThreads()); + nettyServerConfig.setServerWorkerThreads(filtersrvConfig.getFsServerWorkerThreads()); + + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, filtersrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + System.exit(0); + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), filtersrvConfig); + if (null == filtersrvConfig.getRocketmqHome()) { + System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation%n"); + System.exit(-2); + } + + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(filtersrvConfig.getRocketmqHome() + "/conf/logback_filtersrv.xml"); + log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + + final FiltersrvController controller = + new FiltersrvController(filtersrvConfig, nettyServerConfig); + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + return controller; + } catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + return null; + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Filter server config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java new file mode 100644 index 0000000000..e17e5d2a30 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLDecoder; +import java.util.*; + + +public class DynaCode { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + + private static final String FILE_SP = System.getProperty("file.separator"); + + private static final String LINE_SP = System.getProperty("line.separator"); + + private String sourcePath = System.getProperty("user.home") + FILE_SP + "rocketmq_filter_class" + FILE_SP + + UtilAll.getPid(); + + private String outPutClassPath = sourcePath; + + + private ClassLoader parentClassLoader; + + + private List codeStrs; + + + private Map/* class */> loadClass; + + + private String classpath; + + + private String bootclasspath; + + + private String extdirs; + + + private String encoding = "UTF-8"; + + + private String target; + + + @SuppressWarnings("unchecked") + public DynaCode(String code) { + this(Thread.currentThread().getContextClassLoader(), Arrays.asList(code)); + } + + + public DynaCode(ClassLoader parentClassLoader, List codeStrs) { + this(extractClasspath(parentClassLoader), parentClassLoader, codeStrs); + } + + + public DynaCode(String classpath, ClassLoader parentClassLoader, List codeStrs) { + this.classpath = classpath; + this.parentClassLoader = parentClassLoader; + this.codeStrs = codeStrs; + this.loadClass = new HashMap>(codeStrs.size()); + } + + + private static String extractClasspath(ClassLoader cl) { + StringBuffer buf = new StringBuffer(); + while (cl != null) { + if (cl instanceof URLClassLoader) { + URL urls[] = ((URLClassLoader) cl).getURLs(); + for (int i = 0; i < urls.length; i++) { + if (buf.length() > 0) { + buf.append(File.pathSeparatorChar); + } + String s = urls[i].getFile(); + try { + s = URLDecoder.decode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + continue; + } + File f = new File(s); + buf.append(f.getAbsolutePath()); + } + } + cl = cl.getParent(); + } + return buf.toString(); + } + + + public DynaCode(List codeStrs) { + this(Thread.currentThread().getContextClassLoader(), codeStrs); + } + + public static Class compileAndLoadClass(final String className, final String javaSource) + throws Exception { + String classSimpleName = FilterAPI.simpleClassName(className); + String javaCode = javaSource; + + final String newClassSimpleName = classSimpleName + System.currentTimeMillis(); + String newJavaCode = javaCode.replaceAll(classSimpleName, newClassSimpleName); + + List codes = new ArrayList(); + codes.add(newJavaCode); + DynaCode dc = new DynaCode(codes); + dc.compileAndLoadClass(); + Map> map = dc.getLoadClass(); + + Class clazz = map.get(getQualifiedName(newJavaCode)); + return clazz; + } + + public void compileAndLoadClass() throws Exception { + String[] sourceFiles = this.uploadSrcFile(); + this.compile(sourceFiles); + this.loadClass(this.loadClass.keySet()); + } + + public Map> getLoadClass() { + return loadClass; + } + + public static String getQualifiedName(String code) { + StringBuilder sb = new StringBuilder(); + String className = getClassName(code); + if (StringUtils.isNotBlank(className)) { + + String packageName = getPackageName(code); + if (StringUtils.isNotBlank(packageName)) { + sb.append(packageName).append("."); + } + sb.append(className); + } + return sb.toString(); + } + + private String[] uploadSrcFile() throws Exception { + List srcFileAbsolutePaths = new ArrayList(codeStrs.size()); + for (String code : codeStrs) { + if (StringUtils.isNotBlank(code)) { + String packageName = getPackageName(code); + String className = getClassName(code); + if (StringUtils.isNotBlank(className)) { + File srcFile = null; + BufferedWriter bufferWriter = null; + try { + if (StringUtils.isBlank(packageName)) { + File pathFile = new File(sourcePath); + + if (!pathFile.exists()) { + if (!pathFile.mkdirs()) { + throw new RuntimeException("create PathFile Error!"); + } + } + srcFile = new File(sourcePath + FILE_SP + className + ".java"); + } else { + String srcPath = StringUtils.replace(packageName, ".", FILE_SP); + File pathFile = new File(sourcePath + FILE_SP + srcPath); + + if (!pathFile.exists()) { + if (!pathFile.mkdirs()) { + throw new RuntimeException("create PathFile Error!"); + } + } + srcFile = new File(pathFile.getAbsolutePath() + FILE_SP + className + ".java"); + } + synchronized (loadClass) { + loadClass.put(getFullClassName(code), null); + } + if (null != srcFile) { + LOGGER.warn("Dyna Create Java Source File:---->" + srcFile.getAbsolutePath()); + srcFileAbsolutePaths.add(srcFile.getAbsolutePath()); + srcFile.deleteOnExit(); + } + OutputStreamWriter outputStreamWriter = + new OutputStreamWriter(new FileOutputStream(srcFile), encoding); + bufferWriter = new BufferedWriter(outputStreamWriter); + for (String lineCode : code.split(LINE_SP)) { + bufferWriter.write(lineCode); + bufferWriter.newLine(); + } + bufferWriter.flush(); + } finally { + if (null != bufferWriter) { + bufferWriter.close(); + } + } + } + } + } + return srcFileAbsolutePaths.toArray(new String[srcFileAbsolutePaths.size()]); + } + + private void compile(String[] srcFiles) throws Exception { + String args[] = this.buildCompileJavacArgs(srcFiles); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new NullPointerException( + "ToolProvider.getSystemJavaCompiler() return null,please use JDK replace JRE!"); + } + int resultCode = compiler.run(null, null, err, args); + if (resultCode != 0) { + throw new Exception(err.toString(RemotingHelper.DEFAULT_CHARSET)); + } + } + + private void loadClass(Set classFullNames) throws ClassNotFoundException, MalformedURLException { + synchronized (loadClass) { + ClassLoader classLoader = + new URLClassLoader(new URL[]{new File(outPutClassPath).toURI().toURL()}, + parentClassLoader); + for (String key : classFullNames) { + Class classz = classLoader.loadClass(key); + if (null != classz) { + loadClass.put(key, classz); + LOGGER.info("Dyna Load Java Class File OK:----> className: " + key); + } else { + LOGGER.error("Dyna Load Java Class File Fail:----> className: " + key); + } + } + } + } + + public static String getClassName(String code) { + String className = StringUtils.substringBefore(code, "{"); + if (StringUtils.isBlank(className)) { + return className; + } + if (StringUtils.contains(code, " class ")) { + className = StringUtils.substringAfter(className, " class "); + if (StringUtils.contains(className, " extends ")) { + className = StringUtils.substringBefore(className, " extends ").trim(); + } else if (StringUtils.contains(className, " implements ")) { + className = StringUtils.trim(StringUtils.substringBefore(className, " implements ")); + } else { + className = StringUtils.trim(className); + } + } else if (StringUtils.contains(code, " interface ")) { + className = StringUtils.substringAfter(className, " interface "); + if (StringUtils.contains(className, " extends ")) { + className = StringUtils.substringBefore(className, " extends ").trim(); + } else { + className = StringUtils.trim(className); + } + } else if (StringUtils.contains(code, " enum ")) { + className = StringUtils.trim(StringUtils.substringAfter(className, " enum ")); + } else { + return StringUtils.EMPTY; + } + return className; + } + + public static String getPackageName(String code) { + String packageName = + StringUtils.substringBefore(StringUtils.substringAfter(code, "package "), ";").trim(); + return packageName; + } + + public static String getFullClassName(String code) { + String packageName = getPackageName(code); + String className = getClassName(code); + return StringUtils.isBlank(packageName) ? className : packageName + "." + className; + } + + private String[] buildCompileJavacArgs(String srcFiles[]) { + ArrayList args = new ArrayList(); + if (StringUtils.isNotBlank(classpath)) { + args.add("-classpath"); + args.add(classpath); + } + if (StringUtils.isNotBlank(outPutClassPath)) { + args.add("-d"); + args.add(outPutClassPath); + } + if (StringUtils.isNotBlank(sourcePath)) { + args.add("-sourcepath"); + args.add(sourcePath); + } + if (StringUtils.isNotBlank(bootclasspath)) { + args.add("-bootclasspath"); + args.add(bootclasspath); + } + if (StringUtils.isNotBlank(extdirs)) { + args.add("-extdirs"); + args.add(extdirs); + } + if (StringUtils.isNotBlank(encoding)) { + args.add("-encoding"); + args.add(encoding); + } + if (StringUtils.isNotBlank(target)) { + args.add("-target"); + args.add(target); + } + for (int i = 0; i < srcFiles.length; i++) { + args.add(srcFiles[i]); + } + return args.toArray(new String[args.size()]); + } + + public String getOutPutClassPath() { + return outPutClassPath; + } + + public void setOutPutClassPath(String outPutClassPath) { + this.outPutClassPath = outPutClassPath; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + + public ClassLoader getParentClassLoader() { + return parentClassLoader; + } + + public void setParentClassLoader(ClassLoader parentClassLoader) { + this.parentClassLoader = parentClassLoader; + } + + public String getClasspath() { + return classpath; + } + + public void setClasspath(String classpath) { + this.classpath = classpath; + } + + public String getBootclasspath() { + return bootclasspath; + } + + public void setBootclasspath(String bootclasspath) { + this.bootclasspath = bootclasspath; + } + + public String getExtdirs() { + return extdirs; + } + + public void setExtdirs(String extdirs) { + this.extdirs = extdirs; + } + + public String getEncoding() { + return encoding; + } + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + + public String getTarget() { + return target; + } + + public void setTarget(String target) { + this.target = target; + } +} \ No newline at end of file diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java new file mode 100644 index 0000000000..27b19ce38a --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +public interface FilterClassFetchMethod { + public String fetch(final String topic, final String consumerGroup, final String className); +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java new file mode 100644 index 0000000000..f3e747e139 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +import com.alibaba.rocketmq.common.filter.MessageFilter; + + +public class FilterClassInfo { + private String className; + private int classCRC; + private MessageFilter messageFilter; + + + public int getClassCRC() { + return classCRC; + } + + + public void setClassCRC(int classCRC) { + this.classCRC = classCRC; + } + + + public MessageFilter getMessageFilter() { + return messageFilter; + } + + + public void setMessageFilter(MessageFilter messageFilter) { + this.messageFilter = messageFilter; + } + + + public String getClassName() { + return className; + } + + + public void setClassName(String className) { + this.className = className; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java new file mode 100644 index 0000000000..8966ca26af --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +public class FilterClassLoader extends ClassLoader { + public final Class createNewClass(String name, byte[] b, int off, int len) throws ClassFormatError { + return this.defineClass(name, b, off, len); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java new file mode 100644 index 0000000000..618db8e21b --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.filter.MessageFilter; +import com.alibaba.rocketmq.filtersrv.FiltersrvController; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class FilterClassManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + + private final Object compileLock = new Object(); + private final FiltersrvController filtersrvController; + + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSGetClassScheduledThread")); + private ConcurrentHashMap filterClassTable = + new ConcurrentHashMap(128); + private FilterClassFetchMethod filterClassFetchMethod; + + + public FilterClassManager(FiltersrvController filtersrvController) { + this.filtersrvController = filtersrvController; + this.filterClassFetchMethod = + new HttpFilterClassFetchMethod(this.filtersrvController.getFiltersrvConfig() + .getFilterClassRepertoryUrl()); + } + + + public void start() { + if (!this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + fetchClassFromRemoteHost(); + } + }, 1, 1, TimeUnit.MINUTES); + } + } + + private void fetchClassFromRemoteHost() { + Iterator> it = this.filterClassTable.entrySet().iterator(); + while (it.hasNext()) { + try { + Entry next = it.next(); + FilterClassInfo filterClassInfo = next.getValue(); + String[] topicAndGroup = next.getKey().split("@"); + String responseStr = + this.filterClassFetchMethod.fetch(topicAndGroup[0], topicAndGroup[1], + filterClassInfo.getClassName()); + byte[] filterSourceBinary = responseStr.getBytes("UTF-8"); + int classCRC = UtilAll.crc32(responseStr.getBytes("UTF-8")); + if (classCRC != filterClassInfo.getClassCRC()) { + String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); + Class newClass = + DynaCode.compileAndLoadClass(filterClassInfo.getClassName(), javaSource); + Object newInstance = newClass.newInstance(); + filterClassInfo.setMessageFilter((MessageFilter) newInstance); + filterClassInfo.setClassCRC(classCRC); + + log.info("fetch Remote class File OK, {} {}", next.getKey(), + filterClassInfo.getClassName()); + } + } catch (Exception e) { + log.error("fetchClassFromRemoteHost Exception", e); + } + } + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + public boolean registerFilterClass(final String consumerGroup, final String topic, + final String className, final int classCRC, final byte[] filterSourceBinary) { + final String key = buildKey(consumerGroup, topic); + + + boolean registerNew = false; + FilterClassInfo filterClassInfoPrev = this.filterClassTable.get(key); + if (null == filterClassInfoPrev) { + registerNew = true; + } else { + if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + if (filterClassInfoPrev.getClassCRC() != classCRC && classCRC != 0) { + registerNew = true; + } + } + } + + if (registerNew) { + synchronized (this.compileLock) { + filterClassInfoPrev = this.filterClassTable.get(key); + if (null != filterClassInfoPrev && filterClassInfoPrev.getClassCRC() == classCRC) { + return true; + } + + try { + + FilterClassInfo filterClassInfoNew = new FilterClassInfo(); + filterClassInfoNew.setClassName(className); + filterClassInfoNew.setClassCRC(0); + filterClassInfoNew.setMessageFilter(null); + + if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); + Class newClass = DynaCode.compileAndLoadClass(className, javaSource); + Object newInstance = newClass.newInstance(); + filterClassInfoNew.setMessageFilter((MessageFilter) newInstance); + filterClassInfoNew.setClassCRC(classCRC); + } + + this.filterClassTable.put(key, filterClassInfoNew); + } catch (Throwable e) { + String info = + String + .format( + "FilterServer, registerFilterClass Exception, consumerGroup: %s topic: %s className: %s", + consumerGroup, topic, className); + log.error(info, e); + return false; + } + } + } + + return true; + } + + private static String buildKey(final String consumerGroup, final String topic) { + return topic + "@" + consumerGroup; + } + + public FilterClassInfo findFilterClass(final String consumerGroup, final String topic) { + return this.filterClassTable.get(buildKey(consumerGroup, topic)); + } + + + public FilterClassFetchMethod getFilterClassFetchMethod() { + return filterClassFetchMethod; + } + + + public void setFilterClassFetchMethod(FilterClassFetchMethod filterClassFetchMethod) { + this.filterClassFetchMethod = filterClassFetchMethod; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java new file mode 100644 index 0000000000..88cb572c04 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.filter; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.utils.HttpTinyClient; +import com.alibaba.rocketmq.common.utils.HttpTinyClient.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class HttpFilterClassFetchMethod implements FilterClassFetchMethod { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private final String url; + + + public HttpFilterClassFetchMethod(String url) { + this.url = url; + } + + + @Override + public String fetch(String topic, String consumerGroup, String className) { + String thisUrl = String.format("%s/%s.java", this.url, className); + + try { + HttpResult result = HttpTinyClient.httpGet(thisUrl, null, null, "UTF-8", 5000); + if (200 == result.code) { + return result.content; + } + } catch (Exception e) { + log.error( + String.format("call <%s> exception, Topic: %s Group: %s", thisUrl, topic, consumerGroup), e); + } + + return null; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java new file mode 100644 index 0000000000..105cfffe51 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv.processor; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.filter.FilterContext; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.filtersrv.FiltersrvController; +import com.alibaba.rocketmq.filtersrv.filter.FilterClassInfo; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.CommitLog; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class DefaultRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + + private final FiltersrvController filtersrvController; + + + public DefaultRequestProcessor(FiltersrvController filtersrvController) { + this.filtersrvController = filtersrvController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { + if (log.isDebugEnabled()) { + log.debug("receive request, {} {} {}", + request.getCode(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + request); + } + + switch (request.getCode()) { + case RequestCode.REGISTER_MESSAGE_FILTER_CLASS: + return registerMessageFilterClass(ctx, request); + case RequestCode.PULL_MESSAGE: + return pullMessageForward(ctx, request); + } + + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + private RemotingCommand registerMessageFilterClass(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final RegisterMessageFilterClassRequestHeader requestHeader = + (RegisterMessageFilterClassRequestHeader) request.decodeCommandCustomHeader(RegisterMessageFilterClassRequestHeader.class); + + try { + boolean ok = this.filtersrvController.getFilterClassManager().registerFilterClass(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), + requestHeader.getClassName(), + requestHeader.getClassCRC(), + request.getBody()); + if (!ok) { + throw new Exception("registerFilterClass error"); + } + } catch (Exception e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand pullMessageForward(final ChannelHandlerContext ctx, final RemotingCommand request) throws Exception { + final RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); + final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader(); + final PullMessageRequestHeader requestHeader = + (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class); + + final FilterContext filterContext = new FilterContext(); + filterContext.setConsumerGroup(requestHeader.getConsumerGroup()); + + + response.setOpaque(request.getOpaque()); + + DefaultMQPullConsumer pullConsumer = this.filtersrvController.getDefaultMQPullConsumer(); + final FilterClassInfo findFilterClass = + this.filtersrvController.getFilterClassManager() + .findFilterClass(requestHeader.getConsumerGroup(), requestHeader.getTopic()); + if (null == findFilterClass) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Find Filter class failed, not registered"); + return response; + } + + if (null == findFilterClass.getMessageFilter()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Find Filter class failed, registered but no class"); + return response; + } + + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + + + MessageQueue mq = new MessageQueue(); + mq.setTopic(requestHeader.getTopic()); + mq.setQueueId(requestHeader.getQueueId()); + mq.setBrokerName(this.filtersrvController.getBrokerName()); + long offset = requestHeader.getQueueOffset(); + int maxNums = requestHeader.getMaxMsgNums(); + + final PullCallback pullCallback = new PullCallback() { + + @Override + public void onSuccess(PullResult pullResult) { + responseHeader.setMaxOffset(pullResult.getMaxOffset()); + responseHeader.setMinOffset(pullResult.getMinOffset()); + responseHeader.setNextBeginOffset(pullResult.getNextBeginOffset()); + response.setRemark(null); + + switch (pullResult.getPullStatus()) { + case FOUND: + response.setCode(ResponseCode.SUCCESS); + + List msgListOK = new ArrayList(); + try { + for (MessageExt msg : pullResult.getMsgFoundList()) { + boolean match = findFilterClass.getMessageFilter().match(msg, filterContext); + if (match) { + msgListOK.add(msg); + } + } + + + if (!msgListOK.isEmpty()) { + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, msgListOK); + return; + } else { + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + } + } catch (Throwable e) { + final String error = + String.format("do Message Filter Exception, ConsumerGroup: %s Topic: %s ", + requestHeader.getConsumerGroup(), requestHeader.getTopic()); + log.error(error, e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(error + RemotingHelper.exceptionSimpleDesc(e)); + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); + return; + } + + break; + case NO_MATCHED_MSG: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + case NO_NEW_MSG: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_ILLEGAL: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + break; + default: + break; + } + + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); + } + + + @Override + public void onException(Throwable e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Pull Callback Exception, " + RemotingHelper.exceptionSimpleDesc(e)); + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, null); + return; + } + }; + + pullConsumer.pullBlockIfNotFound(mq, null, offset, maxNums, pullCallback); + + return null; + } + + private void returnResponse(final String group, final String topic, ChannelHandlerContext ctx, final RemotingCommand response, + final List msgList) { + if (null != msgList) { + ByteBuffer[] msgBufferList = new ByteBuffer[msgList.size()]; + int bodyTotalSize = 0; + for (int i = 0; i < msgList.size(); i++) { + try { + msgBufferList[i] = messageToByteBuffer(msgList.get(i)); + bodyTotalSize += msgBufferList[i].capacity(); + } catch (Exception e) { + log.error("messageToByteBuffer UnsupportedEncodingException", e); + } + } + + ByteBuffer body = ByteBuffer.allocate(bodyTotalSize); + for (ByteBuffer bb : msgBufferList) { + bb.flip(); + body.put(bb); + } + + response.setBody(body.array()); + + + this.filtersrvController.getFilterServerStatsManager().incGroupGetNums(group, topic, msgList.size()); + + this.filtersrvController.getFilterServerStatsManager().incGroupGetSize(group, topic, bodyTotalSize); + } + + try { + ctx.writeAndFlush(response).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (!future.isSuccess()) { + log.error("FilterServer response to " + future.channel().remoteAddress() + " failed", future.cause()); + log.error(response.toString()); + } + } + }); + } catch (Throwable e) { + log.error("FilterServer process request over, but response failed", e); + log.error(response.toString()); + } + } + + private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException { + int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag()); + if (msg.getBody() != null) { + if (msg.getBody().length >= this.filtersrvController.getFiltersrvConfig().getCompressMsgBodyOverHowmuch()) { + byte[] data = UtilAll.compress(msg.getBody(), this.filtersrvController.getFiltersrvConfig().getZipCompressLevel()); + if (data != null) { + msg.setBody(data); + sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + } + } + } + + final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0; + byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET); + final int topicLength = topicData.length; + String properties = MessageDecoder.messageProperties2String(msg.getProperties()); + byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET); + final int propertiesLength = propertiesData.length; + final int msgLen = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + 8 // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + 8 // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8 // 14 Prepared Transaction Offset + + 4 + bodyLength // 14 BODY + + 1 + topicLength // 15 TOPIC + + 2 + propertiesLength // 16 propertiesLength + + 0; + + ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); + + final MessageExt msgInner = msg; + + // 1 TOTALSIZE + msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + msgStoreItemMemory.putInt(CommitLog.MESSAGE_MAGIC_CODE); + // 3 BODYCRC + msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody())); + // 4 QUEUEID + msgStoreItemMemory.putInt(msgInner.getQueueId()); + // 5 FLAG + msgStoreItemMemory.putInt(msgInner.getFlag()); + // 6 QUEUEOFFSET + msgStoreItemMemory.putLong(msgInner.getQueueOffset()); + // 7 PHYSICALOFFSET + msgStoreItemMemory.putLong(msgInner.getCommitLogOffset()); + // 8 SYSFLAG + msgStoreItemMemory.putInt(sysFlag); + // 9 BORNTIMESTAMP + msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + // 10 BORNHOST + msgStoreItemMemory.put(msgInner.getBornHostBytes()); + // 11 STORETIMESTAMP + msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + msgStoreItemMemory.put(msgInner.getStoreHostBytes()); + // 13 RECONSUMETIMES + msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + // 14 Prepared Transaction Offset + msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + // 15 BODY + msgStoreItemMemory.putInt(bodyLength); + if (bodyLength > 0) + msgStoreItemMemory.put(msgInner.getBody()); + // 16 TOPIC + msgStoreItemMemory.put((byte) topicLength); + msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + msgStoreItemMemory.putShort((short) propertiesLength); + if (propertiesLength > 0) + msgStoreItemMemory.put(propertiesData); + + return msgStoreItemMemory; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java new file mode 100644 index 0000000000..3921c92914 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.filtersrv.stats; + +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.stats.StatsItemSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + + +public class FilterServerStatsManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FILTERSRV_LOGGER_NAME); + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSStatsThread")); + + // ConsumerGroup Get Nums + private final StatsItemSet groupGetNums = new StatsItemSet("GROUP_GET_NUMS", + this.scheduledExecutorService, log); + + // ConsumerGroup Get Size + private final StatsItemSet groupGetSize = new StatsItemSet("GROUP_GET_SIZE", + this.scheduledExecutorService, log); + + + public FilterServerStatsManager() { + } + + + public void start() { + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + public void incGroupGetNums(final String group, final String topic, final int incValue) { + this.groupGetNums.addValue(topic + "@" + group, incValue, 1); + } + + + public void incGroupGetSize(final String group, final String topic, final int incValue) { + this.groupGetSize.addValue(topic + "@" + group, incValue, 1); + } +} diff --git a/rocketmq-namesrv/pom.xml b/rocketmq-namesrv/pom.xml new file mode 100644 index 0000000000..3494f8fba9 --- /dev/null +++ b/rocketmq-namesrv/pom.xml @@ -0,0 +1,58 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-namesrv + rocketmq-namesrv ${project.version} + + + + junit + junit + test + + + com.alibaba.rocketmq + rocketmq-client + + + com.alibaba.rocketmq + rocketmq-tools + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java new file mode 100644 index 0000000000..82f2622f80 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv; + +import com.alibaba.rocketmq.common.Configuration; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.NamesrvConfig; +import com.alibaba.rocketmq.namesrv.kvconfig.KVConfigManager; +import com.alibaba.rocketmq.namesrv.processor.ClusterTestRequestProcessor; +import com.alibaba.rocketmq.namesrv.processor.DefaultRequestProcessor; +import com.alibaba.rocketmq.namesrv.routeinfo.BrokerHousekeepingService; +import com.alibaba.rocketmq.namesrv.routeinfo.RouteInfoManager; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public class NamesrvController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + + private final NamesrvConfig namesrvConfig; + + private final NettyServerConfig nettyServerConfig; + + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "NSScheduledThread")); + private final KVConfigManager kvConfigManager; + private final RouteInfoManager routeInfoManager; + + private RemotingServer remotingServer; + + private BrokerHousekeepingService brokerHousekeepingService; + + private ExecutorService remotingExecutor; + + private Configuration configuration; + + + public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig) { + this.namesrvConfig = namesrvConfig; + this.nettyServerConfig = nettyServerConfig; + this.kvConfigManager = new KVConfigManager(this); + this.routeInfoManager = new RouteInfoManager(); + this.brokerHousekeepingService = new BrokerHousekeepingService(this); + this.configuration = new Configuration( + log, + this.namesrvConfig, this.nettyServerConfig + ); + this.configuration.setStorePathFromConfig(this.namesrvConfig, "configStorePath"); + } + + + public boolean initialize() { + + this.kvConfigManager.load(); + + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.brokerHousekeepingService); + + + this.remotingExecutor = + Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), new ThreadFactoryImpl("RemotingExecutorThread_")); + + this.registerProcessor(); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + NamesrvController.this.routeInfoManager.scanNotActiveBroker(); + } + }, 5, 10, TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + NamesrvController.this.kvConfigManager.printAllPeriodically(); + } + }, 1, 10, TimeUnit.MINUTES); + + return true; + } + + + private void registerProcessor() { + if (namesrvConfig.isClusterTest()) { + + this.remotingServer.registerDefaultProcessor(new ClusterTestRequestProcessor(this, namesrvConfig.getProductEnvName()), + this.remotingExecutor); + } else { + + this.remotingServer.registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); + } + } + + + public void start() throws Exception { + this.remotingServer.start(); + } + + + public void shutdown() { + this.remotingServer.shutdown(); + this.remotingExecutor.shutdown(); + this.scheduledExecutorService.shutdown(); + } + + + public NamesrvConfig getNamesrvConfig() { + return namesrvConfig; + } + + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + + public KVConfigManager getKvConfigManager() { + return kvConfigManager; + } + + + public RouteInfoManager getRouteInfoManager() { + return routeInfoManager; + } + + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } + + public Configuration getConfiguration() { + return configuration; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java new file mode 100644 index 0000000000..286de3a6e3 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.NamesrvConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class NamesrvStartup { + public static Properties properties = null; + public static CommandLine commandLine = null; + + public static void main(String[] args) { + main0(args); + } + + public static NamesrvController main0(String[] args) { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE)) { + NettySystemConfig.socketSndbufSize = 4096; + } + + + if (null == System.getProperty(NettySystemConfig.COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE)) { + NettySystemConfig.socketRcvbufSize = 4096; + } + + try { + //PackageConflictDetect.detectFastjson(); + + Options options = ServerUtil.buildCommandlineOptions(new Options()); + commandLine = + ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + + final NamesrvConfig namesrvConfig = new NamesrvConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + nettyServerConfig.setListenPort(9876); + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + InputStream in = new BufferedInputStream(new FileInputStream(file)); + properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, namesrvConfig); + MixAll.properties2Object(properties, nettyServerConfig); + + namesrvConfig.setConfigStorePath(file); + + System.out.printf("load config properties file OK, " + file + "%n"); + in.close(); + } + } + + + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, namesrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + System.exit(0); + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig); + + if (null == namesrvConfig.getRocketmqHome()) { + System.out.printf("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation%n"); + System.exit(-2); + } + + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(namesrvConfig.getRocketmqHome() + "/conf/logback_namesrv.xml"); + final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + + + MixAll.printObjectProperties(log, namesrvConfig); + MixAll.printObjectProperties(log, nettyServerConfig); + + + final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig); + + // remember all configs to prevent discard + controller.getConfiguration().registerConfig(properties); + + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + + controller.start(); + + String tip = "The Name Server boot success. serializeType=" + RemotingCommand.getSerializeTypeConfigInThisServer(); + log.info(tip); + System.out.printf(tip + "%n"); + + return controller; + } catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Name server config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java new file mode 100644 index 0000000000..a83586c46b --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.kvconfig; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * @author shijia.wxr + */ +public class KVConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + + private final NamesrvController namesrvController; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final HashMap> configTable = + new HashMap>(); + + + public KVConfigManager(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + public void load() { + String content = MixAll.file2String(this.namesrvController.getNamesrvConfig().getKvConfigPath()); + if (content != null) { + KVConfigSerializeWrapper kvConfigSerializeWrapper = + KVConfigSerializeWrapper.fromJson(content, KVConfigSerializeWrapper.class); + if (null != kvConfigSerializeWrapper) { + this.configTable.putAll(kvConfigSerializeWrapper.getConfigTable()); + log.info("load KV config table OK"); + } + } + } + + + public void putKVConfig(final String namespace, final String key, final String value) { + try { + this.lock.writeLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null == kvTable) { + kvTable = new HashMap(); + this.configTable.put(namespace, kvTable); + log.info("putKVConfig create new Namespace {}", namespace); + } + + final String prev = kvTable.put(key, value); + if (null != prev) { + log.info("putKVConfig update config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } else { + log.info("putKVConfig create new config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("putKVConfig InterruptedException", e); + } + + this.persist(); + } + + public void persist() { + try { + this.lock.readLock().lockInterruptibly(); + try { + KVConfigSerializeWrapper kvConfigSerializeWrapper = new KVConfigSerializeWrapper(); + kvConfigSerializeWrapper.setConfigTable(this.configTable); + + String content = kvConfigSerializeWrapper.toJson(); + + if (null != content) { + MixAll.string2File(content, this.namesrvController.getNamesrvConfig().getKvConfigPath()); + } + } catch (IOException e) { + log.error("persist kvconfig Exception, " + + this.namesrvController.getNamesrvConfig().getKvConfigPath(), e); + } finally { + this.lock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("persist InterruptedException", e); + } + + } + + public void deleteKVConfig(final String namespace, final String key) { + try { + this.lock.writeLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + String value = kvTable.remove(key); + log.info("deleteKVConfig delete a config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (InterruptedException e) { + log.error("deleteKVConfig InterruptedException", e); + } + + this.persist(); + } + + public byte[] getKVListByNamespace(final String namespace) { + try { + this.lock.readLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + KVTable table = new KVTable(); + table.setTable(kvTable); + return table.encode(); + } + } finally { + this.lock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getKVListByNamespace InterruptedException", e); + } + + return null; + } + + public String getKVConfig(final String namespace, final String key) { + try { + this.lock.readLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + return kvTable.get(key); + } + } finally { + this.lock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getKVConfig InterruptedException", e); + } + + return null; + } + + public void printAllPeriodically() { + try { + this.lock.readLock().lockInterruptibly(); + try { + log.info("--------------------------------------------------------"); + + { + log.info("configTable SIZE: {}", this.configTable.size()); + Iterator>> it = + this.configTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + Iterator> itSub = next.getValue().entrySet().iterator(); + while (itSub.hasNext()) { + Entry nextSub = itSub.next(); + log.info("configTable NS: {} Key: {} Value: {}", next.getKey(), nextSub.getKey(), + nextSub.getValue()); + } + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("printAllPeriodically InterruptedException", e); + } + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java new file mode 100644 index 0000000000..3a91028e7b --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.kvconfig; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; + + +/** + * @author shijia.wxr + */ +public class KVConfigSerializeWrapper extends RemotingSerializable { + private HashMap> configTable; + + + public HashMap> getConfigTable() { + return configTable; + } + + + public void setConfigTable(HashMap> configTable) { + this.configTable = configTable; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java new file mode 100644 index 0000000000..b0b158d723 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.processor; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author manhong.yqd + */ +public class ClusterTestRequestProcessor extends DefaultRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private final DefaultMQAdminExt adminExt; + private final String productEnvName; + + + public ClusterTestRequestProcessor(NamesrvController namesrvController, String productEnvName) { + super(namesrvController); + this.productEnvName = productEnvName; + adminExt = new DefaultMQAdminExt(); + adminExt.setInstanceName("CLUSTER_TEST_NS_INS_" + productEnvName); + adminExt.setUnitName(productEnvName); + try { + adminExt.start(); + } catch (MQClientException e) { + e.printStackTrace(); + } + } + + + @Override + public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetRouteInfoRequestHeader requestHeader = + (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); + + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + if (topicRouteData != null) { + String orderTopicConf = + this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, + requestHeader.getTopic()); + topicRouteData.setOrderTopicConf(orderTopicConf); + } else { + try { + topicRouteData = adminExt.examineTopicRouteInfo(requestHeader.getTopic()); + } catch (Exception e) { + log.info("get route info by topic from product environment failed. envName={},", productEnvName); + } + } + + if (topicRouteData != null) { + byte[] content = topicRouteData.encode(); + response.setBody(content); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("No topic route info in name server for the topic: " + requestHeader.getTopic() + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java new file mode 100644 index 0000000000..118198efc8 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -0,0 +1,491 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.processor; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MQVersion.Version; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.RegisterBrokerBody; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.header.GetTopicsByClusterRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.*; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public class DefaultRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + + protected final NamesrvController namesrvController; + + + public DefaultRequestProcessor(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + if (log.isDebugEnabled()) { + log.debug("receive request, {} {} {}", + request.getCode(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + request); + } + + switch (request.getCode()) { + case RequestCode.PUT_KV_CONFIG: + return this.putKVConfig(ctx, request); + case RequestCode.GET_KV_CONFIG: + return this.getKVConfig(ctx, request); + case RequestCode.DELETE_KV_CONFIG: + return this.deleteKVConfig(ctx, request); + case RequestCode.REGISTER_BROKER: + Version brokerVersion = MQVersion.value2Version(request.getVersion()); + if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) { + return this.registerBrokerWithFilterServer(ctx, request); + } + else { + return this.registerBroker(ctx, request); + } + case RequestCode.UNREGISTER_BROKER: + return this.unregisterBroker(ctx, request); + case RequestCode.GET_ROUTEINTO_BY_TOPIC: + return this.getRouteInfoByTopic(ctx, request); + case RequestCode.GET_BROKER_CLUSTER_INFO: + return this.getBrokerClusterInfo(ctx, request); + case RequestCode.WIPE_WRITE_PERM_OF_BROKER: + return this.wipeWritePermOfBroker(ctx, request); + case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER: + return getAllTopicListFromNameserver(ctx, request); + case RequestCode.DELETE_TOPIC_IN_NAMESRV: + return deleteTopicInNamesrv(ctx, request); + case RequestCode.GET_KVLIST_BY_NAMESPACE: + return this.getKVListByNamespace(ctx, request); + case RequestCode.GET_TOPICS_BY_CLUSTER: + return this.getTopicsByCluster(ctx, request); + case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS: + return this.getSystemTopicListFromNs(ctx, request); + case RequestCode.GET_UNIT_TOPIC_LIST: + return this.getUnitTopicList(ctx, request); + case RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST: + return this.getHasUnitSubTopicList(ctx, request); + case RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST: + return this.getHasUnitSubUnUnitTopicList(ctx, request); + case RequestCode.UPDATE_NAMESRV_CONFIG: + return this.updateConfig(ctx, request); + case RequestCode.GET_NAMESRV_CONFIG: + return this.getConfig(ctx, request); + default: + break; + } + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + + public RemotingCommand putKVConfig(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final PutKVConfigRequestHeader requestHeader = + (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class); + + this.namesrvController.getKvConfigManager().putKVConfig( + requestHeader.getNamespace(), + requestHeader.getKey(), + requestHeader.getValue() + ); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand getKVConfig(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(GetKVConfigResponseHeader.class); + final GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response.readCustomHeader(); + final GetKVConfigRequestHeader requestHeader = + (GetKVConfigRequestHeader) request.decodeCommandCustomHeader(GetKVConfigRequestHeader.class); + + String value = this.namesrvController.getKvConfigManager().getKVConfig( + requestHeader.getNamespace(), + requestHeader.getKey() + ); + + if (value != null) { + responseHeader.setValue(value); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("No config item, Namespace: " + requestHeader.getNamespace() + " Key: " + requestHeader.getKey()); + return response; + } + + public RemotingCommand deleteKVConfig(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final DeleteKVConfigRequestHeader requestHeader = + (DeleteKVConfigRequestHeader) request.decodeCommandCustomHeader(DeleteKVConfigRequestHeader.class); + + this.namesrvController.getKvConfigManager().deleteKVConfig( + requestHeader.getNamespace(), + requestHeader.getKey() + ); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader(); + final RegisterBrokerRequestHeader requestHeader = + (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + + RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); + + if (request.getBody() != null) { + registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), RegisterBrokerBody.class); + } else { + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setCounter(new AtomicLong(0)); + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestatmp(0); + } + + RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker( + requestHeader.getClusterName(), + requestHeader.getBrokerAddr(), + requestHeader.getBrokerName(), + requestHeader.getBrokerId(), + requestHeader.getHaServerAddr(), + registerBrokerBody.getTopicConfigSerializeWrapper(), + registerBrokerBody.getFilterServerList(), + ctx.channel()); + + responseHeader.setHaServerAddr(result.getHaServerAddr()); + responseHeader.setMasterAddr(result.getMasterAddr()); + + + byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); + response.setBody(jsonValue); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand registerBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader(); + final RegisterBrokerRequestHeader requestHeader = + (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + + TopicConfigSerializeWrapper topicConfigWrapper; + if (request.getBody() != null) { + topicConfigWrapper = TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class); + } else { + topicConfigWrapper = new TopicConfigSerializeWrapper(); + topicConfigWrapper.getDataVersion().setCounter(new AtomicLong(0)); + topicConfigWrapper.getDataVersion().setTimestatmp(0); + } + + RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker( + requestHeader.getClusterName(), + requestHeader.getBrokerAddr(), + requestHeader.getBrokerName(), + requestHeader.getBrokerId(), + requestHeader.getHaServerAddr(), + topicConfigWrapper, + null, + ctx.channel() + ); + + responseHeader.setHaServerAddr(result.getHaServerAddr()); + responseHeader.setMasterAddr(result.getMasterAddr()); + + + byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); + response.setBody(jsonValue); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand unregisterBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final UnRegisterBrokerRequestHeader requestHeader = + (UnRegisterBrokerRequestHeader) request.decodeCommandCustomHeader(UnRegisterBrokerRequestHeader.class); + + this.namesrvController.getRouteInfoManager().unregisterBroker( + requestHeader.getClusterName(), + requestHeader.getBrokerAddr(), + requestHeader.getBrokerName(), + requestHeader.getBrokerId()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetRouteInfoRequestHeader requestHeader = + (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); + + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + + if (topicRouteData != null) { + if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { + String orderTopicConf = + this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, + requestHeader.getTopic()); + topicRouteData.setOrderTopicConf(orderTopicConf); + } + + byte[] content = topicRouteData.encode(); + response.setBody(content); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("No topic route info in name server for the topic: " + requestHeader.getTopic() + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + + private RemotingCommand getBrokerClusterInfo(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] content = this.namesrvController.getRouteInfoManager().getAllClusterInfo(); + response.setBody(content); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand wipeWritePermOfBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(WipeWritePermOfBrokerResponseHeader.class); + final WipeWritePermOfBrokerResponseHeader responseHeader = (WipeWritePermOfBrokerResponseHeader) response.readCustomHeader(); + final WipeWritePermOfBrokerRequestHeader requestHeader = + (WipeWritePermOfBrokerRequestHeader) request.decodeCommandCustomHeader(WipeWritePermOfBrokerRequestHeader.class); + + int wipeTopicCnt = this.namesrvController.getRouteInfoManager().wipeWritePermOfBrokerByLock(requestHeader.getBrokerName()); + + log.info("wipe write perm of broker[{}], client: {}, {}", + requestHeader.getBrokerName(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + wipeTopicCnt); + + responseHeader.setWipeTopicCount(wipeTopicCnt); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getAllTopicListFromNameserver(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getAllTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand deleteTopicInNamesrv(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final DeleteTopicInNamesrvRequestHeader requestHeader = + (DeleteTopicInNamesrvRequestHeader) request.decodeCommandCustomHeader(DeleteTopicInNamesrvRequestHeader.class); + + this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getKVListByNamespace(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetKVListByNamespaceRequestHeader requestHeader = + (GetKVListByNamespaceRequestHeader) request.decodeCommandCustomHeader(GetKVListByNamespaceRequestHeader.class); + + byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace( + requestHeader.getNamespace()); + if (null != jsonValue) { + response.setBody(jsonValue); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("No config item, Namespace: " + requestHeader.getNamespace()); + return response; + } + + private RemotingCommand getTopicsByCluster(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetTopicsByClusterRequestHeader requestHeader = + (GetTopicsByClusterRequestHeader) request.decodeCommandCustomHeader(GetTopicsByClusterRequestHeader.class); + + byte[] body = this.namesrvController.getRouteInfoManager().getTopicsByCluster(requestHeader.getCluster()); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getSystemTopicListFromNs(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getSystemTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getUnitTopics(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getHasUnitSubTopicList(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand updateConfig(ChannelHandlerContext ctx, RemotingCommand request) { + log.info("updateConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = request.getBody(); + if (body != null) { + String bodyStr; + try { + bodyStr = new String(body, MixAll.DEFAULT_CHARSET); + } catch (UnsupportedEncodingException e) { + log.error("updateConfig byte array to string error: ", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + + if (bodyStr == null) { + log.error("updateConfig get null body!"); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("string2Properties error"); + return response; + } + + Properties properties = MixAll.string2Properties(bodyStr); + if (properties == null) { + log.error("updateConfig MixAll.string2Properties error {}", bodyStr); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("string2Properties error"); + return response; + } + + this.namesrvController.getConfiguration().update(properties); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private RemotingCommand getConfig(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + String content = this.namesrvController.getConfiguration().getAllConfigsFormatString(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("getConfig error, ", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java new file mode 100644 index 0000000000..2f123fb33c --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.routeinfo; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class BrokerHousekeepingService implements ChannelEventListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private final NamesrvController namesrvController; + + + public BrokerHousekeepingService(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + } + + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } + + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } + + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java new file mode 100644 index 0000000000..18450c6f58 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -0,0 +1,815 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.routeinfo; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import io.netty.channel.Channel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * @author shijia.wxr + */ +public class RouteInfoManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private final static long BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final HashMap> topicQueueTable; + private final HashMap brokerAddrTable; + private final HashMap> clusterAddrTable; + private final HashMap brokerLiveTable; + private final HashMap/* Filter Server */> filterServerTable; + + + public RouteInfoManager() { + this.topicQueueTable = new HashMap>(1024); + this.brokerAddrTable = new HashMap(128); + this.clusterAddrTable = new HashMap>(32); + this.brokerLiveTable = new HashMap(256); + this.filterServerTable = new HashMap>(256); + } + + public byte[] getAllClusterInfo() { + ClusterInfo clusterInfoSerializeWrapper = new ClusterInfo(); + clusterInfoSerializeWrapper.setBrokerAddrTable(this.brokerAddrTable); + clusterInfoSerializeWrapper.setClusterAddrTable(this.clusterAddrTable); + return clusterInfoSerializeWrapper.encode(); + } + + public void deleteTopic(final String topic) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + this.topicQueueTable.remove(topic); + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("deleteTopic Exception", e); + } + } + + public byte[] getAllTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + topicList.getTopicList().addAll(this.topicQueueTable.keySet()); + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + public RegisterBrokerResult registerBroker( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final String haServerAddr, + final TopicConfigSerializeWrapper topicConfigWrapper, + final List filterServerList, + final Channel channel) { + RegisterBrokerResult result = new RegisterBrokerResult(); + try { + try { + this.lock.writeLock().lockInterruptibly(); + + + Set brokerNames = this.clusterAddrTable.get(clusterName); + if (null == brokerNames) { + brokerNames = new HashSet(); + this.clusterAddrTable.put(clusterName, brokerNames); + } + brokerNames.add(brokerName); + + boolean registerFirst = false; + + + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null == brokerData) { + registerFirst = true; + brokerData = new BrokerData(); + brokerData.setBrokerName(brokerName); + HashMap brokerAddrs = new HashMap(); + brokerData.setBrokerAddrs(brokerAddrs); + + this.brokerAddrTable.put(brokerName, brokerData); + } + String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr); + registerFirst = registerFirst || (null == oldAddr); + + + if (null != topicConfigWrapper // + && MixAll.MASTER_ID == brokerId) { + if (this.isBrokerTopicConfigChanged(brokerAddr, topicConfigWrapper.getDataVersion())// + || registerFirst) { + ConcurrentHashMap tcTable = + topicConfigWrapper.getTopicConfigTable(); + if (tcTable != null) { + for (Map.Entry entry : tcTable.entrySet()) { + this.createAndUpdateQueueData(brokerName, entry.getValue()); + } + } + } + } + + + BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddr, + new BrokerLiveInfo( + System.currentTimeMillis(), + topicConfigWrapper.getDataVersion(), + channel, + haServerAddr)); + if (null == prevBrokerLiveInfo) { + log.info("new broker registerd, {} HAServer: {}", brokerAddr, haServerAddr); + } + + + if (filterServerList != null) { + if (filterServerList.isEmpty()) { + this.filterServerTable.remove(brokerAddr); + } else { + this.filterServerTable.put(brokerAddr, filterServerList); + } + } + + + if (MixAll.MASTER_ID != brokerId) { + String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (masterAddr != null) { + BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.get(masterAddr); + if (brokerLiveInfo != null) { + result.setHaServerAddr(brokerLiveInfo.getHaServerAddr()); + result.setMasterAddr(masterAddr); + } + } + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("registerBroker Exception", e); + } + + return result; + } + + private boolean isBrokerTopicConfigChanged(final String brokerAddr, final DataVersion dataVersion) { + BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); + if (null == prev || !prev.getDataVersion().equals(dataVersion)) { + return true; + } + + return false; + } + + private void createAndUpdateQueueData(final String brokerName, final TopicConfig topicConfig) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(brokerName); + queueData.setWriteQueueNums(topicConfig.getWriteQueueNums()); + queueData.setReadQueueNums(topicConfig.getReadQueueNums()); + queueData.setPerm(topicConfig.getPerm()); + queueData.setTopicSynFlag(topicConfig.getTopicSysFlag()); + + List queueDataList = this.topicQueueTable.get(topicConfig.getTopicName()); + if (null == queueDataList) { + queueDataList = new LinkedList(); + queueDataList.add(queueData); + this.topicQueueTable.put(topicConfig.getTopicName(), queueDataList); + log.info("new topic registerd, {} {}", topicConfig.getTopicName(), queueData); + } else { + boolean addNewOne = true; + + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + if (qd.equals(queueData)) { + addNewOne = false; + } else { + log.info("topic changed, {} OLD: {} NEW: {}", topicConfig.getTopicName(), qd, + queueData); + it.remove(); + } + } + } + + if (addNewOne) { + queueDataList.add(queueData); + } + } + } + + public int wipeWritePermOfBrokerByLock(final String brokerName) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + return wipeWritePermOfBroker(brokerName); + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("wipeWritePermOfBrokerByLock Exception", e); + } + + return 0; + } + + private int wipeWritePermOfBroker(final String brokerName) { + int wipeTopicCnt = 0; + Iterator>> itTopic = this.topicQueueTable.entrySet().iterator(); + while (itTopic.hasNext()) { + Entry> entry = itTopic.next(); + List qdList = entry.getValue(); + + Iterator it = qdList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + int perm = qd.getPerm(); + perm &= ~PermName.PERM_WRITE; + qd.setPerm(perm); + wipeTopicCnt++; + } + } + } + + return wipeTopicCnt; + } + + public void unregisterBroker( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddr); + if (brokerLiveInfo != null) { + log.info("unregisterBroker, remove from brokerLiveTable {}, {}", + brokerLiveInfo != null ? "OK" : "Failed", + brokerAddr + ); + } + + this.filterServerTable.remove(brokerAddr); + + boolean removeBrokerName = false; + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + String addr = brokerData.getBrokerAddrs().remove(brokerId); + log.info("unregisterBroker, remove addr from brokerAddrTable {}, {}", + addr != null ? "OK" : "Failed", + brokerAddr + ); + + if (brokerData.getBrokerAddrs().isEmpty()) { + this.brokerAddrTable.remove(brokerName); + log.info("unregisterBroker, remove name from brokerAddrTable OK, {}", + brokerName + ); + + removeBrokerName = true; + } + } + + if (removeBrokerName) { + Set nameSet = this.clusterAddrTable.get(clusterName); + if (nameSet != null) { + boolean removed = nameSet.remove(brokerName); + log.info("unregisterBroker, remove name from clusterAddrTable {}, {}", + removed ? "OK" : "Failed", + brokerName); + + if (nameSet.isEmpty()) { + this.clusterAddrTable.remove(clusterName); + log.info("unregisterBroker, remove cluster from clusterAddrTable {}", + clusterName + ); + } + } + this.removeTopicByBrokerName(brokerName); + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("unregisterBroker Exception", e); + } + } + + private void removeTopicByBrokerName(final String brokerName) { + Iterator>> itMap = this.topicQueueTable.entrySet().iterator(); + while (itMap.hasNext()) { + Entry> entry = itMap.next(); + + String topic = entry.getKey(); + List queueDataList = entry.getValue(); + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + log.info("removeTopicByBrokerName, remove one broker's topic {} {}", topic, qd); + it.remove(); + } + } + + if (queueDataList.isEmpty()) { + log.info("removeTopicByBrokerName, remove the topic all queue {}", topic); + itMap.remove(); + } + } + } + + public TopicRouteData pickupTopicRouteData(final String topic) { + TopicRouteData topicRouteData = new TopicRouteData(); + boolean foundQueueData = false; + boolean foundBrokerData = false; + Set brokerNameSet = new HashSet(); + List brokerDataList = new LinkedList(); + topicRouteData.setBrokerDatas(brokerDataList); + + HashMap> filterServerMap = new HashMap>(); + topicRouteData.setFilterServerTable(filterServerMap); + + try { + try { + this.lock.readLock().lockInterruptibly(); + List queueDataList = this.topicQueueTable.get(topic); + if (queueDataList != null) { + topicRouteData.setQueueDatas(queueDataList); + foundQueueData = true; + + + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + brokerNameSet.add(qd.getBrokerName()); + } + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + BrokerData brokerDataClone = new BrokerData(); + brokerDataClone.setBrokerName(brokerData.getBrokerName()); + brokerDataClone.setBrokerAddrs((HashMap) brokerData + .getBrokerAddrs().clone()); + brokerDataList.add(brokerDataClone); + foundBrokerData = true; + for (final String brokerAddr : brokerDataClone.getBrokerAddrs().values()) { + List filterServerList = this.filterServerTable.get(brokerAddr); + filterServerMap.put(brokerAddr, filterServerList); + } + } + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("pickupTopicRouteData Exception", e); + } + + if (log.isDebugEnabled()) { + log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); + } + + if (foundBrokerData && foundQueueData) { + return topicRouteData; + } + + return null; + } + + public void scanNotActiveBroker() { + Iterator> it = this.brokerLiveTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long last = next.getValue().getLastUpdateTimestamp(); + if ((last + BROKER_CHANNEL_EXPIRED_TIME) < System.currentTimeMillis()) { + RemotingUtil.closeChannel(next.getValue().getChannel()); + it.remove(); + log.warn("The broker channel expired, {} {}ms", next.getKey(), BROKER_CHANNEL_EXPIRED_TIME); + this.onChannelDestroy(next.getKey(), next.getValue().getChannel()); + } + } + } + + public void onChannelDestroy(String remoteAddr, Channel channel) { + String brokerAddrFound = null; + if (channel != null) { + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator> itBrokerLiveTable = + this.brokerLiveTable.entrySet().iterator(); + while (itBrokerLiveTable.hasNext()) { + Entry entry = itBrokerLiveTable.next(); + if (entry.getValue().getChannel() == channel) { + brokerAddrFound = entry.getKey(); + break; + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("onChannelDestroy Exception", e); + } + } + + if (null == brokerAddrFound) { + brokerAddrFound = remoteAddr; + } else { + log.info("the broker's channel destroyed, {}, clean it's data structure at once", brokerAddrFound); + } + + + if (brokerAddrFound != null && brokerAddrFound.length() > 0) { + + try { + try { + this.lock.writeLock().lockInterruptibly(); + this.brokerLiveTable.remove(brokerAddrFound); + this.filterServerTable.remove(brokerAddrFound); + String brokerNameFound = null; + boolean removeBrokerName = false; + Iterator> itBrokerAddrTable = + this.brokerAddrTable.entrySet().iterator(); + while (itBrokerAddrTable.hasNext() && (null == brokerNameFound)) { + BrokerData brokerData = itBrokerAddrTable.next().getValue(); + + Iterator> it = brokerData.getBrokerAddrs().entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + Long brokerId = entry.getKey(); + String brokerAddr = entry.getValue(); + if (brokerAddr.equals(brokerAddrFound)) { + brokerNameFound = brokerData.getBrokerName(); + it.remove(); + log.info("remove brokerAddr[{}, {}] from brokerAddrTable, because channel destroyed", + brokerId, brokerAddr); + break; + } + } + + if (brokerData.getBrokerAddrs().isEmpty()) { + removeBrokerName = true; + itBrokerAddrTable.remove(); + log.info("remove brokerName[{}] from brokerAddrTable, because channel destroyed", + brokerData.getBrokerName()); + } + } + + if (brokerNameFound != null && removeBrokerName) { + Iterator>> it = this.clusterAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String clusterName = entry.getKey(); + Set brokerNames = entry.getValue(); + boolean removed = brokerNames.remove(brokerNameFound); + if (removed) { + log.info("remove brokerName[{}], clusterName[{}] from clusterAddrTable, because channel destroyed", + brokerNameFound, clusterName); + + + if (brokerNames.isEmpty()) { + log.info("remove the clusterName[{}] from clusterAddrTable, because channel destroyed and no broker in this cluster", + clusterName); + it.remove(); + } + + break; + } + } + } + + if (removeBrokerName) { + Iterator>> itTopicQueueTable = + this.topicQueueTable.entrySet().iterator(); + while (itTopicQueueTable.hasNext()) { + Entry> entry = itTopicQueueTable.next(); + String topic = entry.getKey(); + List queueDataList = entry.getValue(); + + Iterator itQueueData = queueDataList.iterator(); + while (itQueueData.hasNext()) { + QueueData queueData = itQueueData.next(); + if (queueData.getBrokerName().equals(brokerNameFound)) { + itQueueData.remove(); + log.info("remove topic[{} {}], from topicQueueTable, because channel destroyed", + topic, queueData); + } + } + + if (queueDataList.isEmpty()) { + itTopicQueueTable.remove(); + log.info("remove topic[{}] all queue, from topicQueueTable, because channel destroyed", + topic); + } + } + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("onChannelDestroy Exception", e); + } + } + } + + public void printAllPeriodically() { + try { + try { + this.lock.readLock().lockInterruptibly(); + log.info("--------------------------------------------------------"); + { + log.info("topicQueueTable SIZE: {}", this.topicQueueTable.size()); + Iterator>> it = this.topicQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + log.info("topicQueueTable Topic: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("brokerAddrTable SIZE: {}", this.brokerAddrTable.size()); + Iterator> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("brokerAddrTable brokerName: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("brokerLiveTable SIZE: {}", this.brokerLiveTable.size()); + Iterator> it = this.brokerLiveTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("brokerLiveTable brokerAddr: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("clusterAddrTable SIZE: {}", this.clusterAddrTable.size()); + Iterator>> it = this.clusterAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + log.info("clusterAddrTable clusterName: {} {}", next.getKey(), next.getValue()); + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("printAllPeriodically Exception", e); + } + } + + + public byte[] getSystemTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + for (Map.Entry> entry : clusterAddrTable.entrySet()) { + topicList.getTopicList().add(entry.getKey()); + topicList.getTopicList().addAll(entry.getValue()); + } + + if (brokerAddrTable != null && !brokerAddrTable.isEmpty()) { + Iterator it = brokerAddrTable.keySet().iterator(); + while (it.hasNext()) { + BrokerData bd = brokerAddrTable.get(it.next()); + HashMap brokerAddrs = bd.getBrokerAddrs(); + if (bd.getBrokerAddrs() != null && !bd.getBrokerAddrs().isEmpty()) { + Iterator it2 = brokerAddrs.keySet().iterator(); + topicList.setBrokerAddr(brokerAddrs.get(it2.next())); + break; + } + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + public byte[] getTopicsByCluster(String cluster) { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Set brokerNameSet = this.clusterAddrTable.get(cluster); + for (String brokerName : brokerNameSet) { + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + for (QueueData queueData : queueDatas) { + if (brokerName.equals(queueData.getBrokerName())) { + topicList.getTopicList().add(topic); + break; + } + } + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + public byte[] getUnitTopics() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + public byte[] getHasUnitSubTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + public byte[] getHasUnitSubUnUnitTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && !TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag()) + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } +} + + +class BrokerLiveInfo { + private long lastUpdateTimestamp; + private DataVersion dataVersion; + private Channel channel; + private String haServerAddr; + + + public BrokerLiveInfo(long lastUpdateTimestamp, DataVersion dataVersion, Channel channel, + String haServerAddr) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + this.dataVersion = dataVersion; + this.channel = channel; + this.haServerAddr = haServerAddr; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } + + + public Channel getChannel() { + return channel; + } + + + public void setChannel(Channel channel) { + this.channel = channel; + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + @Override + public String toString() { + return "BrokerLiveInfo [lastUpdateTimestamp=" + lastUpdateTimestamp + ", dataVersion=" + dataVersion + + ", channel=" + channel + ", haServerAddr=" + haServerAddr + "]"; + } +} diff --git a/rocketmq-remoting/pom.xml b/rocketmq-remoting/pom.xml new file mode 100644 index 0000000000..b2295971fb --- /dev/null +++ b/rocketmq-remoting/pom.xml @@ -0,0 +1,51 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-remoting + rocketmq-remoting ${project.version} + + + + + junit + junit + test + + + com.alibaba + fastjson + + + io.netty + netty-all + + + org.slf4j + slf4j-api + + + diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java new file mode 100644 index 0000000000..eff9551cc7 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import io.netty.channel.Channel; + + +/** + * @author shijia.wxr + * + */ +public interface ChannelEventListener { + void onChannelConnect(final String remoteAddr, final Channel channel); + + + void onChannelClose(final String remoteAddr, final Channel channel); + + + void onChannelException(final String remoteAddr, final Channel channel); + + + void onChannelIdle(final String remoteAddr, final Channel channel); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java new file mode 100644 index 0000000000..7e0bd8c3a9 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public interface CommandCustomHeader { + void checkFields() throws RemotingCommandException; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java new file mode 100644 index 0000000000..6ba27e1d51 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; + + +/** + * @author shijia.wxr + * + */ +public interface InvokeCallback { + public void operationComplete(final ResponseFuture responseFuture); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java new file mode 100644 index 0000000000..cc2d594664 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +public interface RPCHook { + void doBeforeRequest(final String remoteAddr, final RemotingCommand request); + + + void doAfterResponse(final String remoteAddr, final RemotingCommand request, + final RemotingCommand response); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java new file mode 100644 index 0000000000..ad8c0be2f4 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + +import java.util.List; +import java.util.concurrent.ExecutorService; + + +/** + * @author shijia.wxr + * + */ +public interface RemotingClient extends RemotingService { + + public void updateNameServerAddressList(final List addrs); + + + public List getNameServerAddressList(); + + + public RemotingCommand invokeSync(final String addr, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException; + + + public void invokeAsync(final String addr, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) throws InterruptedException, RemotingConnectException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException; + + + public void invokeOneway(final String addr, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, + RemotingTimeoutException, RemotingSendRequestException; + + + public void registerProcessor(final int requestCode, final NettyRequestProcessor processor, + final ExecutorService executor); + + + public boolean isChannelWriteable(final String addr); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java new file mode 100644 index 0000000000..ae84c1b56a --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.Channel; + +import java.util.concurrent.ExecutorService; + + +/** + * @author shijia.wxr + * + */ +public interface RemotingServer extends RemotingService { + + void registerProcessor(final int requestCode, final NettyRequestProcessor processor, + final ExecutorService executor); + + + void registerDefaultProcessor(final NettyRequestProcessor processor, final ExecutorService executor); + + + int localListenPort(); + + + Pair getProcessorPair(final int requestCode); + + + RemotingCommand invokeSync(final Channel channel, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, + RemotingTimeoutException; + + + void invokeAsync(final Channel channel, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) throws InterruptedException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException; + + + void invokeOneway(final Channel channel, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, + RemotingSendRequestException; + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java new file mode 100644 index 0000000000..cddac3ef5c --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting; + +public interface RemotingService { + void start(); + + + void shutdown(); + + + void registerRPCHook(RPCHook rpcHook); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java new file mode 100644 index 0000000000..4ca077dcba --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author shijia.wxr + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) +public @interface CFNotNull { +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java new file mode 100644 index 0000000000..1318854c86 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author shijia.wxr + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) +public @interface CFNullable { +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java new file mode 100644 index 0000000000..091224ecb7 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +/** + * @author shijia.wxr + */ +public class Pair { + private T1 object1; + private T2 object2; + + + public Pair(T1 object1, T2 object2) { + this.object1 = object1; + this.object2 = object2; + } + + + public T1 getObject1() { + return object1; + } + + + public void setObject1(T1 object1) { + this.object1 = object1; + } + + + public T2 getObject2() { + return object2; + } + + + public void setObject2(T2 object2) { + this.object2 = object2; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java new file mode 100644 index 0000000000..9dcdd83864 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.Channel; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; + + +/** + * @author shijia.wxr + */ +public class RemotingHelper { + public static final String ROCKETMQ_REMOTING = "RocketmqRemoting"; + public static final String DEFAULT_CHARSET = "UTF-8"; + + public static String exceptionSimpleDesc(final Throwable e) { + StringBuffer sb = new StringBuffer(); + if (e != null) { + sb.append(e.toString()); + + StackTraceElement[] stackTrace = e.getStackTrace(); + if (stackTrace != null && stackTrace.length > 0) { + StackTraceElement elment = stackTrace[0]; + sb.append(", "); + sb.append(elment.toString()); + } + } + + return sb.toString(); + } + + public static SocketAddress string2SocketAddress(final String addr) { + String[] s = addr.split(":"); + InetSocketAddress isa = new InetSocketAddress(s[0], Integer.parseInt(s[1])); + return isa; + } + + public static RemotingCommand invokeSync(final String addr, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + long beginTime = System.currentTimeMillis(); + SocketAddress socketAddress = RemotingUtil.string2SocketAddress(addr); + SocketChannel socketChannel = RemotingUtil.connect(socketAddress); + if (socketChannel != null) { + boolean sendRequestOK = false; + + try { + + socketChannel.configureBlocking(true); + + //bugfix http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4614802 + socketChannel.socket().setSoTimeout((int) timeoutMillis); + + + ByteBuffer byteBufferRequest = request.encode(); + while (byteBufferRequest.hasRemaining()) { + int length = socketChannel.write(byteBufferRequest); + if (length > 0) { + if (byteBufferRequest.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + + throw new RemotingSendRequestException(addr); + } + } + } else { + throw new RemotingSendRequestException(addr); + } + + + Thread.sleep(1); + } + + sendRequestOK = true; + + ByteBuffer byteBufferSize = ByteBuffer.allocate(4); + while (byteBufferSize.hasRemaining()) { + int length = socketChannel.read(byteBufferSize); + if (length > 0) { + if (byteBufferSize.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + + throw new RemotingTimeoutException(addr, timeoutMillis); + } + } + } else { + throw new RemotingTimeoutException(addr, timeoutMillis); + } + + + Thread.sleep(1); + } + + int size = byteBufferSize.getInt(0); + ByteBuffer byteBufferBody = ByteBuffer.allocate(size); + while (byteBufferBody.hasRemaining()) { + int length = socketChannel.read(byteBufferBody); + if (length > 0) { + if (byteBufferBody.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + + throw new RemotingTimeoutException(addr, timeoutMillis); + } + } + } else { + throw new RemotingTimeoutException(addr, timeoutMillis); + } + + + Thread.sleep(1); + } + + + byteBufferBody.flip(); + return RemotingCommand.decode(byteBufferBody); + } catch (IOException e) { + e.printStackTrace(); + + if (sendRequestOK) { + throw new RemotingTimeoutException(addr, timeoutMillis); + } else { + throw new RemotingSendRequestException(addr); + } + } finally { + try { + socketChannel.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } else { + throw new RemotingConnectException(addr); + } + } + + + public static String parseChannelRemoteAddr(final Channel channel) { + if (null == channel) { + return ""; + } + SocketAddress remote = channel.remoteAddress(); + final String addr = remote != null ? remote.toString() : ""; + + if (addr.length() > 0) { + int index = addr.lastIndexOf("/"); + if (index >= 0) { + return addr.substring(index + 1); + } + + return addr; + } + + return ""; + } + + + public static String parseChannelRemoteName(final Channel channel) { + if (null == channel) { + return ""; + } + final InetSocketAddress remote = (InetSocketAddress) channel.remoteAddress(); + if (remote != null) { + return remote.getAddress().getHostName(); + } + return ""; + } + + + public static String parseSocketAddressAddr(SocketAddress socketAddress) { + if (socketAddress != null) { + final String addr = socketAddress.toString(); + + if (addr.length() > 0) { + return addr.substring(1); + } + } + return ""; + } + + + public static String parseSocketAddressName(SocketAddress socketAddress) { + + final InetSocketAddress addrs = (InetSocketAddress) socketAddress; + if (addrs != null) { + return addrs.getAddress().getHostName(); + } + return ""; + } + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java new file mode 100644 index 0000000000..af2348f9d1 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketAddress; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.nio.channels.spi.SelectorProvider; +import java.util.ArrayList; +import java.util.Enumeration; + + +/** + * @author shijia.wxr + */ +public class RemotingUtil { + public static final String OS_NAME = System.getProperty("os.name"); + + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static boolean isLinuxPlatform = false; + private static boolean isWindowsPlatform = false; + + static { + if (OS_NAME != null && OS_NAME.toLowerCase().indexOf("linux") >= 0) { + isLinuxPlatform = true; + } + + if (OS_NAME != null && OS_NAME.toLowerCase().indexOf("windows") >= 0) { + isWindowsPlatform = true; + } + } + + public static boolean isWindowsPlatform() { + return isWindowsPlatform; + } + + public static Selector openSelector() throws IOException { + Selector result = null; + + if (isLinuxPlatform()) { + try { + final Class providerClazz = Class.forName("sun.nio.ch.EPollSelectorProvider"); + if (providerClazz != null) { + try { + final Method method = providerClazz.getMethod("provider"); + if (method != null) { + final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null); + if (selectorProvider != null) { + result = selectorProvider.openSelector(); + } + } + } catch (final Exception e) { + log.warn("Open ePoll Selector for linux platform exception", e); + } + } + } catch (final Exception e) { + // ignore + } + } + + if (result == null) { + result = Selector.open(); + } + + return result; + } + + public static boolean isLinuxPlatform() { + return isLinuxPlatform; + } + + public static String getLocalAddress() { + try { + // Traversal Network interface to get the first non-loopback and non-private address + Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); + ArrayList ipv4Result = new ArrayList(); + ArrayList ipv6Result = new ArrayList(); + while (enumeration.hasMoreElements()) { + final NetworkInterface networkInterface = enumeration.nextElement(); + final Enumeration en = networkInterface.getInetAddresses(); + while (en.hasMoreElements()) { + final InetAddress address = en.nextElement(); + if (!address.isLoopbackAddress()) { + if (address instanceof Inet6Address) { + ipv6Result.add(normalizeHostAddress(address)); + } else { + ipv4Result.add(normalizeHostAddress(address)); + } + } + } + } + + // prefer ipv4 + if (!ipv4Result.isEmpty()) { + for (String ip : ipv4Result) { + if (ip.startsWith("127.0") || ip.startsWith("192.168")) { + continue; + } + + return ip; + } + + return ipv4Result.get(ipv4Result.size() - 1); + } else if (!ipv6Result.isEmpty()) { + return ipv6Result.get(0); + } + //If failed to find,fall back to localhost + final InetAddress localHost = InetAddress.getLocalHost(); + return normalizeHostAddress(localHost); + } catch (SocketException e) { + e.printStackTrace(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + return null; + } + + + public static String normalizeHostAddress(final InetAddress localHost) { + if (localHost instanceof Inet6Address) { + return "[" + localHost.getHostAddress() + "]"; + } else { + return localHost.getHostAddress(); + } + } + + public static SocketAddress string2SocketAddress(final String addr) { + String[] s = addr.split(":"); + InetSocketAddress isa = new InetSocketAddress(s[0], Integer.parseInt(s[1])); + return isa; + } + + + public static String socketAddress2String(final SocketAddress addr) { + StringBuilder sb = new StringBuilder(); + InetSocketAddress inetSocketAddress = (InetSocketAddress) addr; + sb.append(inetSocketAddress.getAddress().getHostAddress()); + sb.append(":"); + sb.append(inetSocketAddress.getPort()); + return sb.toString(); + } + + + public static SocketChannel connect(SocketAddress remote) { + return connect(remote, 1000 * 5); + } + + + public static SocketChannel connect(SocketAddress remote, final int timeoutMillis) { + SocketChannel sc = null; + try { + sc = SocketChannel.open(); + sc.configureBlocking(true); + sc.socket().setSoLinger(false, -1); + sc.socket().setTcpNoDelay(true); + sc.socket().setReceiveBufferSize(1024 * 64); + sc.socket().setSendBufferSize(1024 * 64); + sc.socket().connect(remote, timeoutMillis); + sc.configureBlocking(false); + return sc; + } catch (Exception e) { + if (sc != null) { + try { + sc.close(); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + } + + return null; + } + + + public static void closeChannel(Channel channel) { + final String addrRemote = RemotingHelper.parseChannelRemoteAddr(channel); + channel.close().addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + log.info("closeChannel: close the connection to remote address[{}] result: {}", addrRemote, + future.isSuccess()); + } + }); + } + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java new file mode 100644 index 0000000000..c24e8b3166 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * @author shijia.wxr + */ +public class SemaphoreReleaseOnlyOnce { + private final AtomicBoolean released = new AtomicBoolean(false); + private final Semaphore semaphore; + + + public SemaphoreReleaseOnlyOnce(Semaphore semaphore) { + this.semaphore = semaphore; + } + + + public void release() { + if (this.semaphore != null) { + if (this.released.compareAndSet(false, true)) { + this.semaphore.release(); + } + } + } + + + public Semaphore getSemaphore() { + return semaphore; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java new file mode 100644 index 0000000000..365c670e3b --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Base class for background thread + * + * @author shijia.wxr + * + */ +public abstract class ServiceThread implements Runnable { + private static final Logger STLOG = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final long JOIN_TIME = 90 * 1000; + protected final Thread thread; + protected volatile boolean hasNotified = false; + protected volatile boolean stopped = false; + + + public ServiceThread() { + this.thread = new Thread(this, this.getServiceName()); + } + + + public abstract String getServiceName(); + + + public void start() { + this.thread.start(); + } + + + public void shutdown() { + this.shutdown(false); + } + + public void shutdown(final boolean interrupt) { + this.stopped = true; + STLOG.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + try { + if (interrupt) { + this.thread.interrupt(); + } + + long beginTime = System.currentTimeMillis(); + this.thread.join(this.getJointime()); + long eclipseTime = System.currentTimeMillis() - beginTime; + STLOG.info("join thread " + this.getServiceName() + " eclipse time(ms) " + eclipseTime + " " + + this.getJointime()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public long getJointime() { + return JOIN_TIME; + } + + public void stop() { + this.stop(false); + } + + public void stop(final boolean interrupt) { + this.stopped = true; + STLOG.info("stop thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + if (interrupt) { + this.thread.interrupt(); + } + } + + public void makeStop() { + this.stopped = true; + STLOG.info("makestop thread " + this.getServiceName()); + } + + public void wakeup() { + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + protected void waitForRunning(long interval) { + synchronized (this) { + if (this.hasNotified) { + this.hasNotified = false; + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + this.hasNotified = false; + this.onWaitEnd(); + } + } + } + + protected void onWaitEnd() { + } + + public boolean isStopped() { + return stopped; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java new file mode 100644 index 0000000000..fe5cab9bb5 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingCommandException extends RemotingException { + private static final long serialVersionUID = -6061365915274953096L; + + + public RemotingCommandException(String message) { + super(message, null); + } + + + public RemotingCommandException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java new file mode 100644 index 0000000000..5c546bdc35 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingConnectException extends RemotingException { + private static final long serialVersionUID = -5565366231695911316L; + + + public RemotingConnectException(String addr) { + this(addr, null); + } + + + public RemotingConnectException(String addr, Throwable cause) { + super("connect to <" + addr + "> failed", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java new file mode 100644 index 0000000000..2c4b672bfc --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingException extends Exception { + private static final long serialVersionUID = -5690687334570505110L; + + + public RemotingException(String message) { + super(message); + } + + + public RemotingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java new file mode 100644 index 0000000000..e29e1a29dd --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingSendRequestException extends RemotingException { + private static final long serialVersionUID = 5391285827332471674L; + + + public RemotingSendRequestException(String addr) { + this(addr, null); + } + + + public RemotingSendRequestException(String addr, Throwable cause) { + super("send request to <" + addr + "> failed", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java new file mode 100644 index 0000000000..b8c1bb0803 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingTimeoutException extends RemotingException { + + private static final long serialVersionUID = 4106899185095245979L; + + + public RemotingTimeoutException(String message) { + super(message); + } + + + public RemotingTimeoutException(String addr, long timeoutMillis) { + this(addr, timeoutMillis, null); + } + + + public RemotingTimeoutException(String addr, long timeoutMillis, Throwable cause) { + super("wait response on the channel <" + addr + "> timeout, " + timeoutMillis + "(ms)", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java new file mode 100644 index 0000000000..41be8b38d4 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * @author shijia.wxr + */ +public class RemotingTooMuchRequestException extends RemotingException { + private static final long serialVersionUID = 4326919581254519654L; + + + public RemotingTooMuchRequestException(String message) { + super(message); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java new file mode 100644 index 0000000000..4665b28e5e --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + * @author shijia.wxr + * + */ +public class NettyClientConfig { + /** + * Worker thread number + */ + private int clientWorkerThreads = 4; + private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors(); + private int clientOnewaySemaphoreValue = NettySystemConfig.CLIENT_ONEWAY_SEMAPHORE_VALUE; + private int clientAsyncSemaphoreValue = NettySystemConfig.CLIENT_ASYNC_SEMAPHORE_VALUE; + private int connectTimeoutMillis = 3000; + private long channelNotActiveInterval = 1000 * 60; + + /** + * IdleStateEvent will be triggered when neither read nor write was performed for + * the specified period of this time. Specify {@code 0} to disable + */ + private int clientChannelMaxIdleTimeSeconds = 120; + + private int clientSocketSndBufSize = NettySystemConfig.socketSndbufSize; + private int clientSocketRcvBufSize = NettySystemConfig.socketRcvbufSize; + private boolean clientPooledByteBufAllocatorEnable = false; + private boolean clientCloseSocketIfTimeout = false; + + public boolean isClientCloseSocketIfTimeout() { + return clientCloseSocketIfTimeout; + } + + public void setClientCloseSocketIfTimeout(final boolean clientCloseSocketIfTimeout) { + this.clientCloseSocketIfTimeout = clientCloseSocketIfTimeout; + } + + public int getClientWorkerThreads() { + return clientWorkerThreads; + } + + + public void setClientWorkerThreads(int clientWorkerThreads) { + this.clientWorkerThreads = clientWorkerThreads; + } + + + public int getClientOnewaySemaphoreValue() { + return clientOnewaySemaphoreValue; + } + + + public void setClientOnewaySemaphoreValue(int clientOnewaySemaphoreValue) { + this.clientOnewaySemaphoreValue = clientOnewaySemaphoreValue; + } + + + public int getConnectTimeoutMillis() { + return connectTimeoutMillis; + } + + + public void setConnectTimeoutMillis(int connectTimeoutMillis) { + this.connectTimeoutMillis = connectTimeoutMillis; + } + + + public int getClientCallbackExecutorThreads() { + return clientCallbackExecutorThreads; + } + + + public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) { + this.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + } + + + public long getChannelNotActiveInterval() { + return channelNotActiveInterval; + } + + + public void setChannelNotActiveInterval(long channelNotActiveInterval) { + this.channelNotActiveInterval = channelNotActiveInterval; + } + + + public int getClientAsyncSemaphoreValue() { + return clientAsyncSemaphoreValue; + } + + + public void setClientAsyncSemaphoreValue(int clientAsyncSemaphoreValue) { + this.clientAsyncSemaphoreValue = clientAsyncSemaphoreValue; + } + + + public int getClientChannelMaxIdleTimeSeconds() { + return clientChannelMaxIdleTimeSeconds; + } + + + public void setClientChannelMaxIdleTimeSeconds(int clientChannelMaxIdleTimeSeconds) { + this.clientChannelMaxIdleTimeSeconds = clientChannelMaxIdleTimeSeconds; + } + + + public int getClientSocketSndBufSize() { + return clientSocketSndBufSize; + } + + + public void setClientSocketSndBufSize(int clientSocketSndBufSize) { + this.clientSocketSndBufSize = clientSocketSndBufSize; + } + + + public int getClientSocketRcvBufSize() { + return clientSocketRcvBufSize; + } + + + public void setClientSocketRcvBufSize(int clientSocketRcvBufSize) { + this.clientSocketRcvBufSize = clientSocketRcvBufSize; + } + + + public boolean isClientPooledByteBufAllocatorEnable() { + return clientPooledByteBufAllocatorEnable; + } + + + public void setClientPooledByteBufAllocatorEnable(boolean clientPooledByteBufAllocatorEnable) { + this.clientPooledByteBufAllocatorEnable = clientPooledByteBufAllocatorEnable; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java new file mode 100644 index 0000000000..9e685336a7 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; + + +/** + * @author shijia.wxr + * + */ +public class NettyDecoder extends LengthFieldBasedFrameDecoder { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final int FRAME_MAX_LENGTH = // + Integer.parseInt(System.getProperty("com.rocketmq.remoting.frameMaxLength", "16777216")); + + + public NettyDecoder() { + super(FRAME_MAX_LENGTH, 0, 4, 0, 4); + } + + + @Override + public Object decode(ChannelHandlerContext ctx, ByteBuf in) throws Exception { + ByteBuf frame = null; + try { + frame = (ByteBuf) super.decode(ctx, in); + if (null == frame) { + return null; + } + + ByteBuffer byteBuffer = frame.nioBuffer(); + + return RemotingCommand.decode(byteBuffer); + } catch (Exception e) { + log.error("decode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e); + RemotingUtil.closeChannel(ctx.channel()); + } finally { + if (null != frame) { + frame.release(); + } + } + + return null; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java new file mode 100644 index 0000000000..c6c901c4fd --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; + + +/** + * @author shijia.wxr + * + */ +public class NettyEncoder extends MessageToByteEncoder { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + + @Override + public void encode(ChannelHandlerContext ctx, RemotingCommand remotingCommand, ByteBuf out) + throws Exception { + try { + ByteBuffer header = remotingCommand.encodeHeader(); + out.writeBytes(header); + byte[] body = remotingCommand.getBody(); + if (body != null) { + out.writeBytes(body); + } + } catch (Exception e) { + log.error("encode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e); + if (remotingCommand != null) { + log.error(remotingCommand.toString()); + } + RemotingUtil.closeChannel(ctx.channel()); + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java new file mode 100644 index 0000000000..14a2071457 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.channel.Channel; + + +/** + * @author shijia.wxr + */ +public class NettyEvent { + private final NettyEventType type; + private final String remoteAddr; + private final Channel channel; + + + public NettyEvent(NettyEventType type, String remoteAddr, Channel channel) { + this.type = type; + this.remoteAddr = remoteAddr; + this.channel = channel; + } + + + public NettyEventType getType() { + return type; + } + + + public String getRemoteAddr() { + return remoteAddr; + } + + + public Channel getChannel() { + return channel; + } + + + @Override + public String toString() { + return "NettyEvent [type=" + type + ", remoteAddr=" + remoteAddr + ", channel=" + channel + "]"; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java new file mode 100644 index 0000000000..3113147c01 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + * @author shijia.wxr + * + */ +public enum NettyEventType { + CONNECT, + CLOSE, + IDLE, + EXCEPTION +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java new file mode 100644 index 0000000000..70ae5b5159 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -0,0 +1,452 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import com.alibaba.rocketmq.remoting.common.ServiceThread; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketAddress; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + + +/** + * @author shijia.wxr + */ +public abstract class NettyRemotingAbstract { + private static final Logger PLOG = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + + + protected final Semaphore semaphoreOneway; + + + protected final Semaphore semaphoreAsync; + + + protected final ConcurrentHashMap responseTable = + new ConcurrentHashMap(256); + + protected final HashMap> processorTable = + new HashMap>(64); + protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter(); + + protected Pair defaultRequestProcessor; + + + public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { + this.semaphoreOneway = new Semaphore(permitsOneway, true); + this.semaphoreAsync = new Semaphore(permitsAsync, true); + } + + public abstract ChannelEventListener getChannelEventListener(); + + public void putNettyEvent(final NettyEvent event) { + this.nettyEventExecuter.putNettyEvent(event); + } + + public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + final RemotingCommand cmd = msg; + if (cmd != null) { + switch (cmd.getType()) { + case REQUEST_COMMAND: + processRequestCommand(ctx, cmd); + break; + case RESPONSE_COMMAND: + processResponseCommand(ctx, cmd); + break; + default: + break; + } + } + } + + public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) { + final Pair matched = this.processorTable.get(cmd.getCode()); + final Pair pair = null == matched ? this.defaultRequestProcessor : matched; + final int opaque = cmd.getOpaque(); + + if (pair != null) { + Runnable run = new Runnable() { + @Override + public void run() { + try { + RPCHook rpcHook = NettyRemotingAbstract.this.getRPCHook(); + if (rpcHook != null) { + rpcHook.doBeforeRequest(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); + } + + final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd); + if (rpcHook != null) { + rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd, response); + } + + if (!cmd.isOnewayRPC()) { + if (response != null) { + response.setOpaque(opaque); + response.markResponseType(); + try { + ctx.writeAndFlush(response); + } catch (Throwable e) { + PLOG.error("process request over, but response failed", e); + PLOG.error(cmd.toString()); + PLOG.error(response.toString()); + } + } else { + + } + } + } catch (Throwable e) { + if (!"com.aliyun.openservices.ons.api.impl.authority.exception.AuthenticationException" + .equals(e.getClass().getCanonicalName())) { + PLOG.error("process request exception", e); + PLOG.error(cmd.toString()); + } + + if (!cmd.isOnewayRPC()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, // + RemotingHelper.exceptionSimpleDesc(e)); + response.setOpaque(opaque); + ctx.writeAndFlush(response); + } + } + } + }; + + if (pair.getObject1().rejectRequest()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, + "[REJECTREQUEST]system busy, start flow control for a while"); + response.setOpaque(opaque); + ctx.writeAndFlush(response); + return; + } + + try { + final RequestTask requestTask = new RequestTask(run, ctx.channel(), cmd); + pair.getObject2().submit(requestTask); + } catch (RejectedExecutionException e) { + if ((System.currentTimeMillis() % 10000) == 0) { + PLOG.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) // + + ", too many requests and system thread pool busy, RejectedExecutionException " // + + pair.getObject2().toString() // + + " request code: " + cmd.getCode()); + } + + if (!cmd.isOnewayRPC()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, + "[OVERLOAD]system busy, start flow control for a while"); + response.setOpaque(opaque); + ctx.writeAndFlush(response); + } + } + } else { + String error = " request type " + cmd.getCode() + " not supported"; + final RemotingCommand response = + RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); + response.setOpaque(opaque); + ctx.writeAndFlush(response); + PLOG.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error); + } + } + + public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) { + final int opaque = cmd.getOpaque(); + final ResponseFuture responseFuture = responseTable.get(opaque); + if (responseFuture != null) { + responseFuture.setResponseCommand(cmd); + + responseFuture.release(); + + responseTable.remove(opaque); + + if (responseFuture.getInvokeCallback() != null) { + boolean runInThisThread = false; + ExecutorService executor = this.getCallbackExecutor(); + if (executor != null) { + try { + executor.submit(new Runnable() { + @Override + public void run() { + try { + responseFuture.executeInvokeCallback(); + } catch (Throwable e) { + PLOG.warn("execute callback in executor exception, and callback throw", e); + } + } + }); + } catch (Exception e) { + runInThisThread = true; + PLOG.warn("execute callback in executor exception, maybe executor busy", e); + } + } else { + runInThisThread = true; + } + + if (runInThisThread) { + try { + responseFuture.executeInvokeCallback(); + } catch (Throwable e) { + PLOG.warn("executeInvokeCallback Exception", e); + } + } + } else { + responseFuture.putResponse(cmd); + } + } else { + PLOG.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + PLOG.warn(cmd.toString()); + } + } + + public abstract RPCHook getRPCHook(); + + abstract public ExecutorService getCallbackExecutor(); + + public void scanResponseTable() { + final List rfList = new LinkedList(); + Iterator> it = this.responseTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ResponseFuture rep = next.getValue(); + + if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) { + rep.release(); + it.remove(); + rfList.add(rep); + PLOG.warn("remove timeout request, " + rep); + } + } + + for (ResponseFuture rf : rfList) { + try { + rf.executeInvokeCallback(); + } catch (Throwable e) { + PLOG.warn("scanResponseTable, operationComplete Exception", e); + } + } + } + + public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { + final int opaque = request.getOpaque(); + + try { + final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null); + this.responseTable.put(opaque, responseFuture); + final SocketAddress addr = channel.remoteAddress(); + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + if (f.isSuccess()) { + responseFuture.setSendRequestOK(true); + return; + } else { + responseFuture.setSendRequestOK(false); + } + + responseTable.remove(opaque); + responseFuture.setCause(f.cause()); + responseFuture.putResponse(null); + PLOG.warn("send a request command to channel <" + addr + "> failed."); + } + }); + + RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); + if (null == responseCommand) { + if (responseFuture.isSendRequestOK()) { + throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, + responseFuture.getCause()); + } else { + throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); + } + } + + return responseCommand; + } finally { + this.responseTable.remove(opaque); + } + } + + public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + final int opaque = request.getOpaque(); + boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + if (acquired) { + final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); + + final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, invokeCallback, once); + this.responseTable.put(opaque, responseFuture); + try { + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + if (f.isSuccess()) { + responseFuture.setSendRequestOK(true); + return; + } else { + responseFuture.setSendRequestOK(false); + } + + responseFuture.putResponse(null); + responseTable.remove(opaque); + try { + responseFuture.executeInvokeCallback(); + } catch (Throwable e) { + PLOG.warn("excute callback in writeAndFlush addListener, and callback throw", e); + } finally { + responseFuture.release(); + } + + PLOG.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); + } + }); + } catch (Exception e) { + responseFuture.release(); + PLOG.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + } + } else { + String info = + String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // + timeoutMillis, // + this.semaphoreAsync.getQueueLength(), // + this.semaphoreAsync.availablePermits()// + ); + PLOG.warn(info); + throw new RemotingTooMuchRequestException(info); + } + } + + public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + request.markOnewayRPC(); + boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + if (acquired) { + final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway); + try { + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + once.release(); + if (!f.isSuccess()) { + PLOG.warn("send a request command to channel <" + channel.remoteAddress() + "> failed."); + } + } + }); + } catch (Exception e) { + once.release(); + PLOG.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed."); + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + } + } else { + if (timeoutMillis <= 0) { + throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast"); + } else { + String info = String.format( + "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // + timeoutMillis, // + this.semaphoreOneway.getQueueLength(), // + this.semaphoreOneway.availablePermits()// + ); + PLOG.warn(info); + throw new RemotingTimeoutException(info); + } + } + } + + class NettyEventExecuter extends ServiceThread { + private final LinkedBlockingQueue eventQueue = new LinkedBlockingQueue(); + private final int maxSize = 10000; + + + public void putNettyEvent(final NettyEvent event) { + if (this.eventQueue.size() <= maxSize) { + this.eventQueue.add(event); + } else { + PLOG.warn("event queue size[{}] enough, so drop this event {}", this.eventQueue.size(), event.toString()); + } + } + + + @Override + public void run() { + PLOG.info(this.getServiceName() + " service started"); + + final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener(); + + while (!this.isStopped()) { + try { + NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS); + if (event != null && listener != null) { + switch (event.getType()) { + case IDLE: + listener.onChannelIdle(event.getRemoteAddr(), event.getChannel()); + break; + case CLOSE: + listener.onChannelClose(event.getRemoteAddr(), event.getChannel()); + break; + case CONNECT: + listener.onChannelConnect(event.getRemoteAddr(), event.getChannel()); + break; + case EXCEPTION: + listener.onChannelException(event.getRemoteAddr(), event.getChannel()); + break; + default: + break; + + } + } + } catch (Exception e) { + PLOG.warn(this.getServiceName() + " service has exception. ", e); + } + } + + PLOG.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return NettyEventExecuter.class.getSimpleName(); + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java new file mode 100644 index 0000000000..68555c57cb --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java @@ -0,0 +1,682 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPromise; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultEventExecutorGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.SocketAddress; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + + private static final long LOCK_TIMEOUT_MILLIS = 3000; + + private final NettyClientConfig nettyClientConfig; + private final Bootstrap bootstrap = new Bootstrap(); + private final EventLoopGroup eventLoopGroupWorker; + private final Lock lockChannelTables = new ReentrantLock(); + private final ConcurrentHashMap channelTables = new ConcurrentHashMap(); + + private final Timer timer = new Timer("ClientHouseKeepingService", true); + + private final AtomicReference> namesrvAddrList = new AtomicReference>(); + private final AtomicReference namesrvAddrChoosed = new AtomicReference(); + private final AtomicInteger namesrvIndex = new AtomicInteger(initValueIndex()); + private final Lock lockNamesrvChannel = new ReentrantLock(); + + private final ExecutorService publicExecutor; + private final ChannelEventListener channelEventListener; + private DefaultEventExecutorGroup defaultEventExecutorGroup; + private RPCHook rpcHook; + + public NettyRemotingClient(final NettyClientConfig nettyClientConfig) { + this(nettyClientConfig, null); + } + + public NettyRemotingClient(final NettyClientConfig nettyClientConfig, // + final ChannelEventListener channelEventListener) { + super(nettyClientConfig.getClientOnewaySemaphoreValue(), nettyClientConfig.getClientAsyncSemaphoreValue()); + this.nettyClientConfig = nettyClientConfig; + this.channelEventListener = channelEventListener; + + int publicThreadNums = nettyClientConfig.getClientCallbackExecutorThreads(); + if (publicThreadNums <= 0) { + publicThreadNums = 4; + } + + this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyClientPublicExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + + this.eventLoopGroupWorker = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClientSelector_%d", this.threadIndex.incrementAndGet())); + } + }); + } + + private static int initValueIndex() { + Random r = new Random(); + + return Math.abs(r.nextInt() % 999) % 999; + } + + @Override + public void start() { + this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(// + nettyClientConfig.getClientWorkerThreads(), // + new ThreadFactory() { + + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyClientWorkerThread_" + this.threadIndex.incrementAndGet()); + } + }); + + Bootstrap handler = this.bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class)// + .option(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_KEEPALIVE, false) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, nettyClientConfig.getConnectTimeoutMillis()) + .option(ChannelOption.SO_SNDBUF, nettyClientConfig.getClientSocketSndBufSize()) + .option(ChannelOption.SO_RCVBUF, nettyClientConfig.getClientSocketRcvBufSize()) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast( + defaultEventExecutorGroup, + new NettyEncoder(), + new NettyDecoder(), + new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()), + new NettyConnetManageHandler(), + new NettyClientHandler()); + } + }); + + this.timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + try { + NettyRemotingClient.this.scanResponseTable(); + } catch (Exception e) { + log.error("scanResponseTable exception", e); + } + } + }, 1000 * 3, 1000); + + if (this.channelEventListener != null) { + this.nettyEventExecuter.start(); + } + } + + @Override + public void shutdown() { + try { + this.timer.cancel(); + + for (ChannelWrapper cw : this.channelTables.values()) { + this.closeChannel(null, cw.getChannel()); + } + + this.channelTables.clear(); + + this.eventLoopGroupWorker.shutdownGracefully(); + + if (this.nettyEventExecuter != null) { + this.nettyEventExecuter.shutdown(); + } + + if (this.defaultEventExecutorGroup != null) { + this.defaultEventExecutorGroup.shutdownGracefully(); + } + } catch (Exception e) { + log.error("NettyRemotingClient shutdown exception, ", e); + } + + if (this.publicExecutor != null) { + try { + this.publicExecutor.shutdown(); + } catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + } + } + + public void closeChannel(final String addr, final Channel channel) { + if (null == channel) + return; + + final String addrRemote = null == addr ? RemotingHelper.parseChannelRemoteAddr(channel) : addr; + + try { + if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + boolean removeItemFromTable = true; + final ChannelWrapper prevCW = this.channelTables.get(addrRemote); + + log.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, prevCW != null); + + if (null == prevCW) { + log.info("closeChannel: the channel[{}] has been removed from the channel table before", addrRemote); + removeItemFromTable = false; + } else if (prevCW.getChannel() != channel) { + log.info("closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.", + addrRemote); + removeItemFromTable = false; + } + + if (removeItemFromTable) { + this.channelTables.remove(addrRemote); + log.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + } + + RemotingUtil.closeChannel(channel); + } catch (Exception e) { + log.error("closeChannel: close the channel exception", e); + } finally { + this.lockChannelTables.unlock(); + } + } else { + log.warn("closeChannel: try to lock channel table, but timeout, {}ms", LOCK_TIMEOUT_MILLIS); + } + } catch (InterruptedException e) { + log.error("closeChannel exception", e); + } + } + + @Override + public void registerRPCHook(RPCHook rpcHook) { + this.rpcHook = rpcHook; + } + + public void closeChannel(final Channel channel) { + if (null == channel) + return; + + try { + if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + boolean removeItemFromTable = true; + ChannelWrapper prevCW = null; + String addrRemote = null; + for (Map.Entry entry : channelTables.entrySet()) { + String key = entry.getKey(); + ChannelWrapper prev = entry.getValue(); + if (prev.getChannel() != null) { + if (prev.getChannel() == channel) { + prevCW = prev; + addrRemote = key; + break; + } + } + } + + if (null == prevCW) { + log.info("eventCloseChannel: the channel[{}] has been removed from the channel table before", addrRemote); + removeItemFromTable = false; + } + + if (removeItemFromTable) { + this.channelTables.remove(addrRemote); + log.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + RemotingUtil.closeChannel(channel); + } + } catch (Exception e) { + log.error("closeChannel: close the channel exception", e); + } finally { + this.lockChannelTables.unlock(); + } + } else { + log.warn("closeChannel: try to lock channel table, but timeout, {}ms", LOCK_TIMEOUT_MILLIS); + } + } catch (InterruptedException e) { + log.error("closeChannel exception", e); + } + } + + @Override + public void updateNameServerAddressList(List addrs) { + List old = this.namesrvAddrList.get(); + boolean update = false; + + if (!addrs.isEmpty()) { + if (null == old) { + update = true; + } else if (addrs.size() != old.size()) { + update = true; + } else { + for (int i = 0; i < addrs.size() && !update; i++) { + if (!old.contains(addrs.get(i))) { + update = true; + } + } + } + + if (update) { + Collections.shuffle(addrs); + this.namesrvAddrList.set(addrs); + } + } + } + + @Override + public List getNameServerAddressList() { + return this.namesrvAddrList.get(); + } + + @Override + public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis) + throws InterruptedException, RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + RemotingCommand response = this.invokeSyncImpl(channel, request, timeoutMillis); + if (this.rpcHook != null) { + this.rpcHook.doAfterResponse(RemotingHelper.parseChannelRemoteAddr(channel), request, response); + } + return response; + } catch (RemotingSendRequestException e) { + log.warn("invokeSync: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } catch (RemotingTimeoutException e) { + if (nettyClientConfig.isClientCloseSocketIfTimeout()) { + this.closeChannel(addr, channel); + log.warn("invokeSync: close socket because of timeout, {}ms, {}", timeoutMillis, addr); + } + log.warn("invokeSync: wait response timeout exception, the channel[{}]", addr); + throw e; + } + } else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + private Channel getAndCreateChannel(final String addr) throws InterruptedException { + if (null == addr) + return getAndCreateNameserverChannel(); + + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + + return this.createChannel(addr); + } + + private Channel getAndCreateNameserverChannel() throws InterruptedException { + String addr = this.namesrvAddrChoosed.get(); + if (addr != null) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + } + + final List addrList = this.namesrvAddrList.get(); + if (this.lockNamesrvChannel.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + addr = this.namesrvAddrChoosed.get(); + if (addr != null) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + } + + if (addrList != null && !addrList.isEmpty()) { + for (int i = 0; i < addrList.size(); i++) { + int index = this.namesrvIndex.incrementAndGet(); + index = Math.abs(index); + index = index % addrList.size(); + String newAddr = addrList.get(index); + + this.namesrvAddrChoosed.set(newAddr); + Channel channelNew = this.createChannel(newAddr); + if (channelNew != null) + return channelNew; + } + } + } catch (Exception e) { + log.error("getAndCreateNameserverChannel: create name server channel exception", e); + } finally { + this.lockNamesrvChannel.unlock(); + } + } else { + log.warn("getAndCreateNameserverChannel: try to lock name server, but timeout, {}ms", LOCK_TIMEOUT_MILLIS); + } + + return null; + } + + private Channel createChannel(final String addr) throws InterruptedException { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + + + if (this.lockChannelTables.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + try { + boolean createNewConnection = false; + cw = this.channelTables.get(addr); + if (cw != null) { + + if (cw.isOK()) { + return cw.getChannel(); + } else if (!cw.getChannelFuture().isDone()) { + createNewConnection = false; + } else { + this.channelTables.remove(addr); + createNewConnection = true; + } + } else { + createNewConnection = true; + } + + if (createNewConnection) { + ChannelFuture channelFuture = this.bootstrap.connect(RemotingHelper.string2SocketAddress(addr)); + log.info("createChannel: begin to connect remote host[{}] asynchronously", addr); + cw = new ChannelWrapper(channelFuture); + this.channelTables.put(addr, cw); + } + } catch (Exception e) { + log.error("createChannel: create channel exception", e); + } finally { + this.lockChannelTables.unlock(); + } + } else { + log.warn("createChannel: try to lock channel table, but timeout, {}ms", LOCK_TIMEOUT_MILLIS); + } + + if (cw != null) { + ChannelFuture channelFuture = cw.getChannelFuture(); + if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { + if (cw.isOK()) { + log.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString()); + return cw.getChannel(); + } else { + log.warn("createChannel: connect remote host[" + addr + "] failed, " + channelFuture.toString(), channelFuture.cause()); + } + } else { + log.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(), + channelFuture.toString()); + } + } + + return null; + } + + @Override + public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) + throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, + RemotingSendRequestException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback); + } catch (RemotingSendRequestException e) { + log.warn("invokeAsync: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } + } else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + @Override + public void invokeOneway(String addr, RemotingCommand request, long timeoutMillis) throws InterruptedException, + RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + this.invokeOnewayImpl(channel, request, timeoutMillis); + } catch (RemotingSendRequestException e) { + log.warn("invokeOneway: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } + } else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + @Override + public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { + ExecutorService executorThis = executor; + if (null == executor) { + executorThis = this.publicExecutor; + } + + Pair pair = new Pair(processor, executorThis); + this.processorTable.put(requestCode, pair); + } + + @Override + public boolean isChannelWriteable(String addr) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.isWriteable(); + } + return true; + } + + @Override + public ChannelEventListener getChannelEventListener() { + return channelEventListener; + } + + @Override + public RPCHook getRPCHook() { + return this.rpcHook; + } + + @Override + public ExecutorService getCallbackExecutor() { + return this.publicExecutor; + } + + public List getNamesrvAddrList() { + return namesrvAddrList.get(); + } + + public RPCHook getRpcHook() { + return rpcHook; + } + + static class ChannelWrapper { + private final ChannelFuture channelFuture; + + + public ChannelWrapper(ChannelFuture channelFuture) { + this.channelFuture = channelFuture; + } + + + public boolean isOK() { + return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); + } + + + public boolean isWriteable() { + return this.channelFuture.channel().isWritable(); + } + + + private Channel getChannel() { + return this.channelFuture.channel(); + } + + + public ChannelFuture getChannelFuture() { + return channelFuture; + } + } + + class NettyClientHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + processMessageReceived(ctx, msg); + + } + } + + class NettyConnetManageHandler extends ChannelDuplexHandler { + @Override + public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) + throws Exception { + final String local = localAddress == null ? "UNKNOW" : localAddress.toString(); + final String remote = remoteAddress == null ? "UNKNOW" : remoteAddress.toString(); + log.info("NETTY CLIENT PIPELINE: CONNECT {} => {}", local, remote); + super.connect(ctx, remoteAddress, localAddress, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress.toString(), ctx.channel())); + } + } + + + @Override + public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY CLIENT PIPELINE: DISCONNECT {}", remoteAddress); + closeChannel(ctx.channel()); + super.disconnect(ctx, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel())); + } + } + + + @Override + public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); + closeChannel(ctx.channel()); + super.close(ctx, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel())); + } + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent evnet = (IdleStateEvent) evt; + if (evnet.state().equals(IdleState.ALL_IDLE)) { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress); + closeChannel(ctx.channel()); + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this + .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress.toString(), ctx.channel())); + } + } + } + + ctx.fireUserEventTriggered(evt); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress); + log.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause); + closeChannel(ctx.channel()); + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress.toString(), ctx.channel())); + } + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java new file mode 100644 index 0000000000..a14947e3c4 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java @@ -0,0 +1,384 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultEventExecutorGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetSocketAddress; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + + +public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private final ServerBootstrap serverBootstrap; + private final EventLoopGroup eventLoopGroupSelector; + private final EventLoopGroup eventLoopGroupBoss; + private final NettyServerConfig nettyServerConfig; + + private final ExecutorService publicExecutor; + private final ChannelEventListener channelEventListener; + + private final Timer timer = new Timer("ServerHouseKeepingService", true); + private DefaultEventExecutorGroup defaultEventExecutorGroup; + + private RPCHook rpcHook; + + + private int port = 0; + + + public NettyRemotingServer(final NettyServerConfig nettyServerConfig) { + this(nettyServerConfig, null); + } + + + public NettyRemotingServer(final NettyServerConfig nettyServerConfig, final ChannelEventListener channelEventListener) { + super(nettyServerConfig.getServerOnewaySemaphoreValue(), nettyServerConfig.getServerAsyncSemaphoreValue()); + this.serverBootstrap = new ServerBootstrap(); + this.nettyServerConfig = nettyServerConfig; + this.channelEventListener = channelEventListener; + + int publicThreadNums = nettyServerConfig.getServerCallbackExecutorThreads(); + if (publicThreadNums <= 0) { + publicThreadNums = 4; + } + + this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyServerPublicExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + + this.eventLoopGroupBoss = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyBoss_%d", this.threadIndex.incrementAndGet())); + } + }); + + if (RemotingUtil.isLinuxPlatform() // + && nettyServerConfig.isUseEpollNativeSelector()) { + this.eventLoopGroupSelector = new EpollEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + private int threadTotal = nettyServerConfig.getServerSelectorThreads(); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerEPOLLSelector_%d_%d", threadTotal, this.threadIndex.incrementAndGet())); + } + }); + } else { + this.eventLoopGroupSelector = new NioEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + private int threadTotal = nettyServerConfig.getServerSelectorThreads(); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerNIOSelector_%d_%d", threadTotal, this.threadIndex.incrementAndGet())); + } + }); + } + } + + + @Override + public void start() { + this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(// + nettyServerConfig.getServerWorkerThreads(), // + new ThreadFactory() { + + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyServerCodecThread_" + this.threadIndex.incrementAndGet()); + } + }); + + ServerBootstrap childHandler = + this.serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector).channel(NioServerSocketChannel.class) + .option(ChannelOption.SO_BACKLOG, 1024) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_KEEPALIVE, false) + .childOption(ChannelOption.TCP_NODELAY, true) + .option(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSndBufSize()) + .option(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketRcvBufSize()) + .localAddress(new InetSocketAddress(this.nettyServerConfig.getListenPort())) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast( + defaultEventExecutorGroup, + new NettyEncoder(), + new NettyDecoder(), + new IdleStateHandler(0, 0, nettyServerConfig.getServerChannelMaxIdleTimeSeconds()), + new NettyConnetManageHandler(), + new NettyServerHandler()); + } + }); + + if (nettyServerConfig.isServerPooledByteBufAllocatorEnable()) { + childHandler.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + } + + try { + ChannelFuture sync = this.serverBootstrap.bind().sync(); + InetSocketAddress addr = (InetSocketAddress) sync.channel().localAddress(); + this.port = addr.getPort(); + } catch (InterruptedException e1) { + throw new RuntimeException("this.serverBootstrap.bind().sync() InterruptedException", e1); + } + + if (this.channelEventListener != null) { + this.nettyEventExecuter.start(); + } + + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + NettyRemotingServer.this.scanResponseTable(); + } catch (Exception e) { + log.error("scanResponseTable exception", e); + } + } + }, 1000 * 3, 1000); + } + + @Override + public void shutdown() { + try { + if (this.timer != null) { + this.timer.cancel(); + } + + this.eventLoopGroupBoss.shutdownGracefully(); + + this.eventLoopGroupSelector.shutdownGracefully(); + + if (this.nettyEventExecuter != null) { + this.nettyEventExecuter.shutdown(); + } + + if (this.defaultEventExecutorGroup != null) { + this.defaultEventExecutorGroup.shutdownGracefully(); + } + } catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + + if (this.publicExecutor != null) { + try { + this.publicExecutor.shutdown(); + } catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + } + } + + @Override + public void registerRPCHook(RPCHook rpcHook) { + this.rpcHook = rpcHook; + } + + @Override + public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { + ExecutorService executorThis = executor; + if (null == executor) { + executorThis = this.publicExecutor; + } + + Pair pair = new Pair(processor, executorThis); + this.processorTable.put(requestCode, pair); + } + + @Override + public void registerDefaultProcessor(NettyRequestProcessor processor, ExecutorService executor) { + this.defaultRequestProcessor = new Pair(processor, executor); + } + + @Override + public int localListenPort() { + return this.port; + } + + @Override + public Pair getProcessorPair(int requestCode) { + return processorTable.get(requestCode); + } + + @Override + public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { + return this.invokeSyncImpl(channel, request, timeoutMillis); + } + + @Override + public void invokeAsync(Channel channel, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback); + } + + @Override + public void invokeOneway(Channel channel, RemotingCommand request, long timeoutMillis) throws InterruptedException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + this.invokeOnewayImpl(channel, request, timeoutMillis); + } + + @Override + public ChannelEventListener getChannelEventListener() { + return channelEventListener; + } + + @Override + public RPCHook getRPCHook() { + return this.rpcHook; + } + + @Override + public ExecutorService getCallbackExecutor() { + return this.publicExecutor; + } + + class NettyServerHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + processMessageReceived(ctx, msg); + } + } + + class NettyConnetManageHandler extends ChannelDuplexHandler { + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelRegistered {}", remoteAddress); + super.channelRegistered(ctx); + } + + + @Override + public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelUnregistered, the channel[{}]", remoteAddress); + super.channelUnregistered(ctx); + } + + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelActive, the channel[{}]", remoteAddress); + super.channelActive(ctx); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress.toString(), ctx.channel())); + } + } + + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelInactive, the channel[{}]", remoteAddress); + super.channelInactive(ctx); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress.toString(), ctx.channel())); + } + } + + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent evnet = (IdleStateEvent) evt; + if (evnet.state().equals(IdleState.ALL_IDLE)) { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY SERVER PIPELINE: IDLE exception [{}]", remoteAddress); + RemotingUtil.closeChannel(ctx.channel()); + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this + .putNettyEvent(new NettyEvent(NettyEventType.IDLE, remoteAddress.toString(), ctx.channel())); + } + } + } + + ctx.fireUserEventTriggered(evt); + } + + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY SERVER PIPELINE: exceptionCaught {}", remoteAddress); + log.warn("NETTY SERVER PIPELINE: exceptionCaught exception.", cause); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress.toString(), ctx.channel())); + } + + RemotingUtil.closeChannel(ctx.channel()); + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java new file mode 100644 index 0000000000..dae7f9e76b --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; + + +/** + * Common remoting command processor + * + * @author shijia.wxr + * + */ +public interface NettyRequestProcessor { + RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws Exception; + boolean rejectRequest(); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java new file mode 100644 index 0000000000..7922206153 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + + * + * @author shijia.wxr + * + */ +public class NettyServerConfig implements Cloneable { + private int listenPort = 8888; + private int serverWorkerThreads = 8; + private int serverCallbackExecutorThreads = 0; + private int serverSelectorThreads = 3; + private int serverOnewaySemaphoreValue = 256; + private int serverAsyncSemaphoreValue = 64; + private int serverChannelMaxIdleTimeSeconds = 120; + + private int serverSocketSndBufSize = NettySystemConfig.socketSndbufSize; + private int serverSocketRcvBufSize = NettySystemConfig.socketRcvbufSize; + private boolean serverPooledByteBufAllocatorEnable = true; + + /** + * make make install + * + * + * ../glibc-2.10.1/configure \ --prefix=/usr \ --with-headers=/usr/include \ + * --host=x86_64-linux-gnu \ --build=x86_64-pc-linux-gnu \ --without-gd + */ + private boolean useEpollNativeSelector = false; + + + public int getListenPort() { + return listenPort; + } + + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + + public int getServerWorkerThreads() { + return serverWorkerThreads; + } + + + public void setServerWorkerThreads(int serverWorkerThreads) { + this.serverWorkerThreads = serverWorkerThreads; + } + + + public int getServerSelectorThreads() { + return serverSelectorThreads; + } + + + public void setServerSelectorThreads(int serverSelectorThreads) { + this.serverSelectorThreads = serverSelectorThreads; + } + + + public int getServerOnewaySemaphoreValue() { + return serverOnewaySemaphoreValue; + } + + + public void setServerOnewaySemaphoreValue(int serverOnewaySemaphoreValue) { + this.serverOnewaySemaphoreValue = serverOnewaySemaphoreValue; + } + + + public int getServerCallbackExecutorThreads() { + return serverCallbackExecutorThreads; + } + + + public void setServerCallbackExecutorThreads(int serverCallbackExecutorThreads) { + this.serverCallbackExecutorThreads = serverCallbackExecutorThreads; + } + + + public int getServerAsyncSemaphoreValue() { + return serverAsyncSemaphoreValue; + } + + + public void setServerAsyncSemaphoreValue(int serverAsyncSemaphoreValue) { + this.serverAsyncSemaphoreValue = serverAsyncSemaphoreValue; + } + + + public int getServerChannelMaxIdleTimeSeconds() { + return serverChannelMaxIdleTimeSeconds; + } + + + public void setServerChannelMaxIdleTimeSeconds(int serverChannelMaxIdleTimeSeconds) { + this.serverChannelMaxIdleTimeSeconds = serverChannelMaxIdleTimeSeconds; + } + + + public int getServerSocketSndBufSize() { + return serverSocketSndBufSize; + } + + + public void setServerSocketSndBufSize(int serverSocketSndBufSize) { + this.serverSocketSndBufSize = serverSocketSndBufSize; + } + + + public int getServerSocketRcvBufSize() { + return serverSocketRcvBufSize; + } + + + public void setServerSocketRcvBufSize(int serverSocketRcvBufSize) { + this.serverSocketRcvBufSize = serverSocketRcvBufSize; + } + + + public boolean isServerPooledByteBufAllocatorEnable() { + return serverPooledByteBufAllocatorEnable; + } + + + public void setServerPooledByteBufAllocatorEnable(boolean serverPooledByteBufAllocatorEnable) { + this.serverPooledByteBufAllocatorEnable = serverPooledByteBufAllocatorEnable; + } + + + public boolean isUseEpollNativeSelector() { + return useEpollNativeSelector; + } + + + public void setUseEpollNativeSelector(boolean useEpollNativeSelector) { + this.useEpollNativeSelector = useEpollNativeSelector; + } + + @Override + public Object clone() throws CloneNotSupportedException { + return (NettyServerConfig) super.clone(); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java new file mode 100644 index 0000000000..41589ce7bd --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting.netty; + +public class NettySystemConfig { + public static final String COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = + "com.rocketmq.remoting.nettyPooledByteBufAllocatorEnable"; + public static final String COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE = // + "com.rocketmq.remoting.socket.sndbuf.size"; + public static final String COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE = // + "com.rocketmq.remoting.socket.rcvbuf.size"; + public static final String COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE = // + "com.rocketmq.remoting.clientAsyncSemaphoreValue"; + public static final String COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE = // + "com.rocketmq.remoting.clientOnewaySemaphoreValue"; + public static final boolean NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE = // + Boolean + .parseBoolean(System.getProperty(COM_ROCKETMQ_REMOTING_NETTY_POOLED_BYTE_BUF_ALLOCATOR_ENABLE, "false")); + public static int socketSndbufSize = // + Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_SNDBUF_SIZE, "65535")); + public static int socketRcvbufSize = // + Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_SOCKET_RCVBUF_SIZE, "65535")); + public static final int CLIENT_ASYNC_SEMAPHORE_VALUE = // + Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_ASYNC_SEMAPHORE_VALUE, "65535")); + public static final int CLIENT_ONEWAY_SEMAPHORE_VALUE = // + Integer.parseInt(System.getProperty(COM_ROCKETMQ_REMOTING_CLIENT_ONEWAY_SEMAPHORE_VALUE, "65535")); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/RequestTask.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/RequestTask.java new file mode 100644 index 0000000000..e02ae48242 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/RequestTask.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting.netty; + + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.Channel; + +public class RequestTask implements Runnable { + private final Runnable runnable; + private final long createTimestamp = System.currentTimeMillis(); + private final Channel channel; + private final RemotingCommand request; + private boolean stopRun = false; + + public RequestTask(final Runnable runnable, final Channel channel, final RemotingCommand request) { + this.runnable = runnable; + this.channel = channel; + this.request = request; + } + + @Override + public int hashCode() { + int result = runnable != null ? runnable.hashCode() : 0; + result = 31 * result + (int) (getCreateTimestamp() ^ (getCreateTimestamp() >>> 32)); + result = 31 * result + (channel != null ? channel.hashCode() : 0); + result = 31 * result + (request != null ? request.hashCode() : 0); + result = 31 * result + (isStopRun() ? 1 : 0); + return result; + } + + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (!(o instanceof RequestTask)) return false; + + final RequestTask that = (RequestTask) o; + + if (getCreateTimestamp() != that.getCreateTimestamp()) return false; + if (isStopRun() != that.isStopRun()) return false; + if (channel != null ? !channel.equals(that.channel) : that.channel != null) return false; + return request != null ? request.getOpaque() == that.request.getOpaque() : that.request == null; + + } + + public long getCreateTimestamp() { + return createTimestamp; + } + + public boolean isStopRun() { + return stopRun; + } + + public void setStopRun(final boolean stopRun) { + this.stopRun = stopRun; + } + + @Override + public void run() { + if (!this.stopRun) this.runnable.run(); + } + + public void returnResponse(int code, String remark) { + final RemotingCommand response = RemotingCommand.createResponseCommand(code, remark); + response.setOpaque(request.getOpaque()); + this.channel.writeAndFlush(response); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java new file mode 100644 index 0000000000..b6185f18cc --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * @author shijia.wxr + */ +public class ResponseFuture { + private final int opaque; + private final long timeoutMillis; + private final InvokeCallback invokeCallback; + private final long beginTimestamp = System.currentTimeMillis(); + private final CountDownLatch countDownLatch = new CountDownLatch(1); + + private final SemaphoreReleaseOnlyOnce once; + + private final AtomicBoolean executeCallbackOnlyOnce = new AtomicBoolean(false); + private volatile RemotingCommand responseCommand; + private volatile boolean sendRequestOK = true; + private volatile Throwable cause; + + + public ResponseFuture(int opaque, long timeoutMillis, InvokeCallback invokeCallback, + SemaphoreReleaseOnlyOnce once) { + this.opaque = opaque; + this.timeoutMillis = timeoutMillis; + this.invokeCallback = invokeCallback; + this.once = once; + } + + + public void executeInvokeCallback() { + if (invokeCallback != null) { + if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { + invokeCallback.operationComplete(this); + } + } + } + + + public void release() { + if (this.once != null) { + this.once.release(); + } + } + + + public boolean isTimeout() { + long diff = System.currentTimeMillis() - this.beginTimestamp; + return diff > this.timeoutMillis; + } + + + public RemotingCommand waitResponse(final long timeoutMillis) throws InterruptedException { + this.countDownLatch.await(timeoutMillis, TimeUnit.MILLISECONDS); + return this.responseCommand; + } + + + public void putResponse(final RemotingCommand responseCommand) { + this.responseCommand = responseCommand; + this.countDownLatch.countDown(); + } + + + public long getBeginTimestamp() { + return beginTimestamp; + } + + + public boolean isSendRequestOK() { + return sendRequestOK; + } + + + public void setSendRequestOK(boolean sendRequestOK) { + this.sendRequestOK = sendRequestOK; + } + + + public long getTimeoutMillis() { + return timeoutMillis; + } + + + public InvokeCallback getInvokeCallback() { + return invokeCallback; + } + + + public Throwable getCause() { + return cause; + } + + + public void setCause(Throwable cause) { + this.cause = cause; + } + + + public RemotingCommand getResponseCommand() { + return responseCommand; + } + + + public void setResponseCommand(RemotingCommand responseCommand) { + this.responseCommand = responseCommand; + } + + + public int getOpaque() { + return opaque; + } + + + @Override + public String toString() { + return "ResponseFuture [responseCommand=" + responseCommand + ", sendRequestOK=" + sendRequestOK + + ", cause=" + cause + ", opaque=" + opaque + ", timeoutMillis=" + timeoutMillis + + ", invokeCallback=" + invokeCallback + ", beginTimestamp=" + beginTimestamp + + ", countDownLatch=" + countDownLatch + "]"; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java new file mode 100644 index 0000000000..9f4adbe146 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting.protocol; + +public enum LanguageCode { + JAVA((byte) 0), + CPP((byte) 1), + DOTNET((byte) 2), + PYTHON((byte) 3), + DELPHI((byte) 4), + ERLANG((byte) 5), + RUBY((byte) 6), + OTHER((byte) 7), + HTTP((byte) 8); + + private byte code; + + LanguageCode(byte code) { + this.code = code; + } + + public static LanguageCode valueOf(byte code) { + for (LanguageCode languageCode : LanguageCode.values()) { + if (languageCode.getCode() == code) { + return languageCode; + } + } + return null; + } + + public byte getCode() { + return code; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java new file mode 100644 index 0000000000..a09dd3b15c --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java @@ -0,0 +1,569 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * @author shijia.wxr + */ +public class RemotingCommand { + public static final String SERIALIZE_TYPE_PROPERTY = "rocketmq.serialize.type"; + public static final String SERIALIZE_TYPE_ENV = "ROCKETMQ_SERIALIZE_TYPE"; + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.ROCKETMQ_REMOTING); + private static final int RPC_TYPE = 0; // 0, REQUEST_COMMAND + private static final int RPC_ONEWAY = 1; // 0, RPC + + private static final Map, Field[]> CLASS_HASH_MAP = + new HashMap, Field[]>(); + private static final Map CANONICAL_NAME_CACHE = new HashMap(); + // 1, RESPONSE_COMMAND + private static final Map NOT_NULL_ANNOTATION_CACHE = new HashMap(); + // 1, Oneway + + private static final String STRING_CANONICAL_NAME = String.class.getCanonicalName(); + private static final String DOUBLE_CANONICAL_NAME_1 = Double.class.getCanonicalName(); + private static final String DOUBLE_CANONICAL_NAME_2 = double.class.getCanonicalName(); + private static final String INTEGER_CANONICAL_NAME_1 = Integer.class.getCanonicalName(); + private static final String INTEGER_CANONICAL_NAME_2 = int.class.getCanonicalName(); + private static final String LONG_CANONICAL_NAME_1 = Long.class.getCanonicalName(); + private static final String LONG_CANONICAL_NAME_2 = long.class.getCanonicalName(); + private static final String BOOLEAN_CANONICAL_NAME_1 = Boolean.class.getCanonicalName(); + private static final String BOOLEAN_CANONICAL_NAME_2 = boolean.class.getCanonicalName(); + public static final String REMOTING_VERSION_KEY = "rocketmq.remoting.version"; + private static volatile int configVersion = -1; + private static AtomicInteger requestId = new AtomicInteger(0); + + private static SerializeType serializeTypeConfigInThisServer = SerializeType.JSON; + + static { + final String protocol = System.getProperty(SERIALIZE_TYPE_PROPERTY, System.getenv(SERIALIZE_TYPE_ENV)); + if (!isBlank(protocol)) { + try { + serializeTypeConfigInThisServer = SerializeType.valueOf(protocol); + } catch (IllegalArgumentException e) { + throw new RuntimeException("parser specified protocol error. protocol=" + protocol, e); + } + } + } + + /** + + */ + private int code; + private LanguageCode language = LanguageCode.JAVA; + private int version = 0; + private int opaque = requestId.getAndIncrement(); + private int flag = 0; + private String remark; + private HashMap extFields; + private transient CommandCustomHeader customHeader; + /** + + */ + private SerializeType serializeTypeCurrentRPC = serializeTypeConfigInThisServer; + /** + + */ + private transient byte[] body; + + + protected RemotingCommand() { + } + + public static RemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader) { + RemotingCommand cmd = new RemotingCommand(); + cmd.setCode(code); + cmd.customHeader = customHeader; + setCmdVersion(cmd); + return cmd; + } + + private static void setCmdVersion(RemotingCommand cmd) { + if (configVersion >= 0) { + cmd.setVersion(configVersion); + } else { + String v = System.getProperty(REMOTING_VERSION_KEY); + if (v != null) { + int value = Integer.parseInt(v); + cmd.setVersion(value); + configVersion = value; + } + } + } + + public static RemotingCommand createResponseCommand(Class classHeader) { + RemotingCommand cmd = createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, "not set any response code", classHeader); + + return cmd; + } + + /** + + */ + public static RemotingCommand createResponseCommand(int code, String remark, Class classHeader) { + RemotingCommand cmd = new RemotingCommand(); + cmd.markResponseType(); + cmd.setCode(code); + cmd.setRemark(remark); + setCmdVersion(cmd); + + if (classHeader != null) { + try { + CommandCustomHeader objectHeader = classHeader.newInstance(); + cmd.customHeader = objectHeader; + } catch (InstantiationException e) { + return null; + } catch (IllegalAccessException e) { + return null; + } + } + + return cmd; + } + + public void markResponseType() { + int bits = 1 << RPC_TYPE; + this.flag |= bits; + } + + public static RemotingCommand createResponseCommand(int code, String remark) { + return createResponseCommand(code, remark, null); + } + + public static RemotingCommand decode(final byte[] array) { + ByteBuffer byteBuffer = ByteBuffer.wrap(array); + return decode(byteBuffer); + } + + public static RemotingCommand decode(final ByteBuffer byteBuffer) { + int length = byteBuffer.limit(); + int oriHeaderLen = byteBuffer.getInt(); + int headerLength = getHeaderLength(oriHeaderLen); + + byte[] headerData = new byte[headerLength]; + byteBuffer.get(headerData); + + RemotingCommand cmd = headerDecode(headerData, getProtocolType(oriHeaderLen)); + + int bodyLength = length - 4 - headerLength; + byte[] bodyData = null; + if (bodyLength > 0) { + bodyData = new byte[bodyLength]; + byteBuffer.get(bodyData); + } + cmd.body = bodyData; + + return cmd; + } + + public static int getHeaderLength(int length) { + return length & 0xFFFFFF; + } + + private static RemotingCommand headerDecode(byte[] headerData, SerializeType type) { + switch (type) { + case JSON: + RemotingCommand resultJson = RemotingSerializable.decode(headerData, RemotingCommand.class); + resultJson.setSerializeTypeCurrentRPC(type); + return resultJson; + case ROCKETMQ: + RemotingCommand resultRMQ = RocketMQSerializable.rocketMQProtocolDecode(headerData); + resultRMQ.setSerializeTypeCurrentRPC(type); + return resultRMQ; + default: + break; + } + + return null; + } + + public static SerializeType getProtocolType(int source) { + return SerializeType.valueOf((byte) ((source >> 24) & 0xFF)); + } + + public static int createNewRequestId() { + return requestId.incrementAndGet(); + } + + public static SerializeType getSerializeTypeConfigInThisServer() { + return serializeTypeConfigInThisServer; + } + + private static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + return true; + } + + public CommandCustomHeader readCustomHeader() { + return customHeader; + } + + public void writeCustomHeader(CommandCustomHeader customHeader) { + this.customHeader = customHeader; + } + + public CommandCustomHeader decodeCommandCustomHeader(Class classHeader) throws RemotingCommandException { + CommandCustomHeader objectHeader; + try { + objectHeader = classHeader.newInstance(); + } catch (InstantiationException e) { + return null; + } catch (IllegalAccessException e) { + return null; + } + + if (this.extFields != null) { + + Field[] fields = getClazzFields(classHeader); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String fieldName = field.getName(); + if (!fieldName.startsWith("this")) { + try { + String value = this.extFields.get(fieldName); + if (null == value) { + Annotation annotation = getNotNullAnnotation(field); + if (annotation != null) { + throw new RemotingCommandException("the custom field <" + fieldName + "> is null"); + } + + continue; + } + + field.setAccessible(true); + String type = getCanonicalName(field.getType()); + Object valueParsed; + + if (type.equals(STRING_CANONICAL_NAME)) { + valueParsed = value; + } else if (type.equals(INTEGER_CANONICAL_NAME_1) || type.equals(INTEGER_CANONICAL_NAME_2)) { + valueParsed = Integer.parseInt(value); + } else if (type.equals(LONG_CANONICAL_NAME_1) || type.equals(LONG_CANONICAL_NAME_2)) { + valueParsed = Long.parseLong(value); + } else if (type.equals(BOOLEAN_CANONICAL_NAME_1) || type.equals(BOOLEAN_CANONICAL_NAME_2)) { + valueParsed = Boolean.parseBoolean(value); + } else if (type.equals(DOUBLE_CANONICAL_NAME_1) || type.equals(DOUBLE_CANONICAL_NAME_2)) { + valueParsed = Double.parseDouble(value); + } else { + throw new RemotingCommandException("the custom field <" + fieldName + "> type is not supported"); + } + + field.set(objectHeader, valueParsed); + + } catch (Throwable e) { + } + } + } + } + + objectHeader.checkFields(); + } + + return objectHeader; + } + + private Field[] getClazzFields(Class classHeader) { + Field[] field = CLASS_HASH_MAP.get(classHeader); + + if (field == null) { + field = classHeader.getDeclaredFields(); + synchronized (CLASS_HASH_MAP) { + CLASS_HASH_MAP.put(classHeader, field); + } + } + return field; + } + + private Annotation getNotNullAnnotation(Field field) { + Annotation annotation = NOT_NULL_ANNOTATION_CACHE.get(field); + + if (annotation == null) { + annotation = field.getAnnotation(CFNotNull.class); + synchronized (NOT_NULL_ANNOTATION_CACHE) { + NOT_NULL_ANNOTATION_CACHE.put(field, annotation); + } + } + return annotation; + } + + private String getCanonicalName(Class clazz) { + String name = CANONICAL_NAME_CACHE.get(clazz); + + if (name == null) { + name = clazz.getCanonicalName(); + synchronized (CANONICAL_NAME_CACHE) { + CANONICAL_NAME_CACHE.put(clazz, name); + } + } + return name; + } + + public ByteBuffer encode() { + // 1> header length size + int length = 4; + + // 2> header data length + byte[] headerData = this.headerEncode(); + length += headerData.length; + + // 3> body data length + if (this.body != null) { + length += body.length; + } + + ByteBuffer result = ByteBuffer.allocate(4 + length); + + // length + result.putInt(length); + + // header length + result.put(markProtocolType(headerData.length, serializeTypeCurrentRPC)); + + // header data + result.put(headerData); + + // body data; + if (this.body != null) { + result.put(this.body); + } + + result.flip(); + + return result; + } + + private byte[] headerEncode() { + this.makeCustomHeaderToNet(); + if (SerializeType.ROCKETMQ == serializeTypeCurrentRPC) { + return RocketMQSerializable.rocketMQProtocolEncode(this); + } else { + return RemotingSerializable.encode(this); + } + } + + public static byte[] markProtocolType(int source, SerializeType type) { + byte[] result = new byte[4]; + + result[0] = type.getCode(); + result[1] = (byte) ((source >> 16) & 0xFF); + result[2] = (byte) ((source >> 8) & 0xFF); + result[3] = (byte) (source & 0xFF); + return result; + } + + public void makeCustomHeaderToNet() { + if (this.customHeader != null) { + Field[] fields = getClazzFields(customHeader.getClass()); + if (null == this.extFields) { + this.extFields = new HashMap(); + } + + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(this.customHeader); + } catch (IllegalArgumentException e) { + } catch (IllegalAccessException e) { + } + + if (value != null) { + this.extFields.put(name, value.toString()); + } + } + } + } + } + } + + public ByteBuffer encodeHeader() { + return encodeHeader(this.body != null ? this.body.length : 0); + } + + /** + + */ + public ByteBuffer encodeHeader(final int bodyLength) { + // 1> header length size + int length = 4; + + // 2> header data length + byte[] headerData; + headerData = this.headerEncode(); + + length += headerData.length; + + // 3> body data length + length += bodyLength; + + ByteBuffer result = ByteBuffer.allocate(4 + length - bodyLength); + + // length + result.putInt(length); + + // header length + result.put(markProtocolType(headerData.length, serializeTypeCurrentRPC)); + + // header data + result.put(headerData); + + result.flip(); + + return result; + } + + public void markOnewayRPC() { + int bits = 1 << RPC_ONEWAY; + this.flag |= bits; + } + + @JSONField(serialize = false) + public boolean isOnewayRPC() { + int bits = 1 << RPC_ONEWAY; + return (this.flag & bits) == bits; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + @JSONField(serialize = false) + public RemotingCommandType getType() { + if (this.isResponseType()) { + return RemotingCommandType.RESPONSE_COMMAND; + } + + return RemotingCommandType.REQUEST_COMMAND; + } + + @JSONField(serialize = false) + public boolean isResponseType() { + int bits = 1 << RPC_TYPE; + return (this.flag & bits) == bits; + } + + public LanguageCode getLanguage() { + return language; + } + + public void setLanguage(LanguageCode language) { + this.language = language; + } + + public int getVersion() { + return version; + } + + public void setVersion(int version) { + this.version = version; + } + + public int getOpaque() { + return opaque; + } + + public void setOpaque(int opaque) { + this.opaque = opaque; + } + + public int getFlag() { + return flag; + } + + public void setFlag(int flag) { + this.flag = flag; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public byte[] getBody() { + return body; + } + + public void setBody(byte[] body) { + this.body = body; + } + + public HashMap getExtFields() { + return extFields; + } + + public void setExtFields(HashMap extFields) { + this.extFields = extFields; + } + + public void addExtField(String key, String value) { + if (null == extFields) { + extFields = new HashMap(); + } + extFields.put(key, value); + } + + @Override + public String toString() { + return "RemotingCommand [code=" + code + ", language=" + language + ", version=" + version + ", opaque=" + opaque + ", flag(B)=" + + Integer.toBinaryString(flag) + ", remark=" + remark + ", extFields=" + extFields + ", serializeTypeCurrentRPC=" + + serializeTypeCurrentRPC + "]"; + } + + + public SerializeType getSerializeTypeCurrentRPC() { + return serializeTypeCurrentRPC; + } + + + public void setSerializeTypeCurrentRPC(SerializeType serializeTypeCurrentRPC) { + this.serializeTypeCurrentRPC = serializeTypeCurrentRPC; + } +} \ No newline at end of file diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java new file mode 100644 index 0000000000..c0ac288e95 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +/** + * @author shijia.wxr + * + */ +public enum RemotingCommandType { + REQUEST_COMMAND, + RESPONSE_COMMAND; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java new file mode 100644 index 0000000000..c144ea617d --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +import com.alibaba.fastjson.JSON; + +import java.nio.charset.Charset; + + +/** + * @author shijia.wxr + * + */ +public abstract class RemotingSerializable { + public final static Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + + public static byte[] encode(final Object obj) { + final String json = toJson(obj, false); + if (json != null) { + return json.getBytes(CHARSET_UTF8); + } + return null; + } + + public static String toJson(final Object obj, boolean prettyFormat) { + return JSON.toJSONString(obj, prettyFormat); + } + + public static T decode(final byte[] data, Class classOfT) { + final String json = new String(data, CHARSET_UTF8); + return fromJson(json, classOfT); + } + + public static T fromJson(String json, Class classOfT) { + return JSON.parseObject(json, classOfT); + } + + public byte[] encode() { + final String json = this.toJson(); + if (json != null) { + return json.getBytes(CHARSET_UTF8); + } + return null; + } + + public String toJson() { + return toJson(false); + } + + public String toJson(final boolean prettyFormat) { + return toJson(this, prettyFormat); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java new file mode 100644 index 0000000000..085c750cc7 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting.protocol; + +public class RemotingSysResponseCode { + + public static final int SUCCESS = 0; + + public static final int SYSTEM_ERROR = 1; + + public static final int SYSTEM_BUSY = 2; + + public static final int REQUEST_CODE_NOT_SUPPORTED = 3; + + public static final int TRANSACTION_FAILED = 4; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RocketMQSerializable.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RocketMQSerializable.java new file mode 100644 index 0000000000..6956aab778 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RocketMQSerializable.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + + +/** + * @author manhong.yqd + * + */ +public class RocketMQSerializable { + public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + + public static byte[] rocketMQProtocolEncode(RemotingCommand cmd) { + // String remark + byte[] remarkBytes = null; + int remarkLen = 0; + if (cmd.getRemark() != null && cmd.getRemark().length() > 0) { + remarkBytes = cmd.getRemark().getBytes(RemotingSerializable.CHARSET_UTF8); + remarkLen = remarkBytes.length; + } + + // HashMap extFields + byte[] extFieldsBytes = null; + int extLen = 0; + if (cmd.getExtFields() != null && !cmd.getExtFields().isEmpty()) { + extFieldsBytes = mapSerialize(cmd.getExtFields()); + extLen = extFieldsBytes.length; + } + + // ################### cal total length + int totalLen = calTotalLen(remarkLen, extLen); + + // ################### content + ByteBuffer headerBuffer = ByteBuffer.allocate(totalLen); + // int code(~32767) + headerBuffer.putShort((short) cmd.getCode()); + // LanguageCode language + headerBuffer.put(cmd.getLanguage().getCode()); + // int version(~32767) + headerBuffer.putShort((short) cmd.getVersion()); + // int opaque + headerBuffer.putInt(cmd.getOpaque()); + // int flag + headerBuffer.putInt(cmd.getFlag()); + // String remark + if (remarkBytes != null) { + headerBuffer.putInt(remarkBytes.length); + headerBuffer.put(remarkBytes); + } else { + headerBuffer.putInt(0); + } + // HashMap extFields; + if (extFieldsBytes != null) { + headerBuffer.putInt(extFieldsBytes.length); + headerBuffer.put(extFieldsBytes); + } else { + headerBuffer.putInt(0); + } + + return headerBuffer.array(); + } + + public static byte[] mapSerialize(HashMap map) { + // keySize+key+valSize+val + // keySize+key+valSize+val + if (null == map || map.isEmpty()) + return null; + + int totalLength = 0; + int kvLength; + Iterator> it = map.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (entry.getKey() != null && entry.getValue() != null) { + kvLength = + // keySize + Key + 2 + entry.getKey().getBytes(RemotingSerializable.CHARSET_UTF8).length + // valSize + val + + 4 + entry.getValue().getBytes(RemotingSerializable.CHARSET_UTF8).length; + totalLength += kvLength; + } + } + + ByteBuffer content = ByteBuffer.allocate(totalLength); + byte[] key; + byte[] val; + it = map.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (entry.getKey() != null && entry.getValue() != null) { + key = entry.getKey().getBytes(RemotingSerializable.CHARSET_UTF8); + val = entry.getValue().getBytes(RemotingSerializable.CHARSET_UTF8); + + content.putShort((short) key.length); + content.put(key); + + content.putInt(val.length); + content.put(val); + } + } + + return content.array(); + } + + private static int calTotalLen(int remark, int ext) { + // int code(~32767) + int length = 2 + // LanguageCode language + + 1 + // int version(~32767) + + 2 + // int opaque + + 4 + // int flag + + 4 + // String remark + + 4 + remark + // HashMap extFields + + 4 + ext; + + return length; + } + + public static RemotingCommand rocketMQProtocolDecode(final byte[] headerArray) { + RemotingCommand cmd = new RemotingCommand(); + ByteBuffer headerBuffer = ByteBuffer.wrap(headerArray); + // int code(~32767) + cmd.setCode(headerBuffer.getShort()); + // LanguageCode language + cmd.setLanguage(LanguageCode.valueOf(headerBuffer.get())); + // int version(~32767) + cmd.setVersion(headerBuffer.getShort()); + // int opaque + cmd.setOpaque(headerBuffer.getInt()); + // int flag + cmd.setFlag(headerBuffer.getInt()); + // String remark + int remarkLength = headerBuffer.getInt(); + if (remarkLength > 0) { + byte[] remarkContent = new byte[remarkLength]; + headerBuffer.get(remarkContent); + cmd.setRemark(new String(remarkContent, RemotingSerializable.CHARSET_UTF8)); + } + + // HashMap extFields + int extFieldsLength = headerBuffer.getInt(); + if (extFieldsLength > 0) { + byte[] extFieldsBytes = new byte[extFieldsLength]; + headerBuffer.get(extFieldsBytes); + cmd.setExtFields(mapDeserialize(extFieldsBytes)); + } + return cmd; + } + + public static HashMap mapDeserialize(byte[] bytes) { + if (bytes == null || bytes.length <= 0) + return null; + + HashMap map = new HashMap(); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + + short keySize = 0; + byte[] keyContent = null; + int valSize = 0; + byte[] valContent = null; + while (byteBuffer.hasRemaining()) { + keySize = byteBuffer.getShort(); + keyContent = new byte[keySize]; + byteBuffer.get(keyContent); + + valSize = byteBuffer.getInt(); + valContent = new byte[valSize]; + byteBuffer.get(valContent); + + map.put(new String(keyContent, RemotingSerializable.CHARSET_UTF8), new String(valContent, + RemotingSerializable.CHARSET_UTF8)); + } + return map; + } + + + public static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + return true; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/SerializeType.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/SerializeType.java new file mode 100644 index 0000000000..b46d8d8ff5 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/SerializeType.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting.protocol; + +public enum SerializeType { + JSON((byte) 0), + ROCKETMQ((byte) 1); + + private byte code; + + SerializeType(byte code) { + this.code = code; + } + + public static SerializeType valueOf(byte code) { + for (SerializeType serializeType : SerializeType.values()) { + if (serializeType.getCode() == code) { + return serializeType; + } + } + return null; + } + + public byte getCode() { + return code; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.sevialize.txt b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.sevialize.txt new file mode 100644 index 0000000000..976cdf2932 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.sevialize.txt @@ -0,0 +1,6 @@ +// +// Remoting protocol V0.1 draft +// +// protocol

+// 1 2 3 4 +// \ No newline at end of file diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt new file mode 100644 index 0000000000..7444cbbd82 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt @@ -0,0 +1,6 @@ +// +// Remoting protocol V0.1 draft +// +// protocol
+// 1 2 3 4 +// diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java new file mode 100644 index 0000000000..6baa013603 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ExceptionTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.*; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.junit.Test; + +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class ExceptionTest { + private static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + RemotingServer client = new NettyRemotingServer(config); + client.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + + @Override + public boolean rejectRequest() { + return false; + } + }, Executors.newCachedThreadPool()); + client.start(); + return client; + } + + @Test + public void test_CONNECT_EXCEPTION() { + RemotingClient client = createRemotingClient(); + + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = null; + try { + response = client.invokeSync("localhost:8888", request, 1000 * 3); + } catch (RemotingConnectException e) { + e.printStackTrace(); + } catch (RemotingSendRequestException e) { + e.printStackTrace(); + } catch (RemotingTimeoutException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println("invoke result = " + response); + assertTrue(null == response); + + client.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + private static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java new file mode 100644 index 0000000000..97d16633c1 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: MixTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import org.junit.Test; + + +/** + * @author shijia.wxr + */ +public class MixTest { + @Test + public void test_extFieldsValue() { + + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java new file mode 100644 index 0000000000..e4ff94890f --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Test; + + +/** + + * + * @author shijia.wxr + * + */ +public class NettyConnectionTest { + @Test + public void test_connect_timeout() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + try { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + } catch (Exception e) { + e.printStackTrace(); + } + } + + client.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + config.setClientChannelMaxIdleTimeSeconds(15); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java new file mode 100644 index 0000000000..23145f4426 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.*; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; + +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + * + */ +public class NettyIdleTest { + // @Test + public void test_idle_event() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 10; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println(i + " invoke result = " + response); + assertTrue(response != null); + + Thread.sleep(1000 * 10); + } + + Thread.sleep(1000 * 60); + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + public static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + config.setServerChannelMaxIdleTimeSeconds(30); + RemotingServer remotingServer = new NettyRemotingServer(config); + remotingServer.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + + @Override + public boolean rejectRequest() { + return false; + } + }, Executors.newCachedThreadPool()); + remotingServer.start(); + return remotingServer; + } + + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + config.setClientChannelMaxIdleTimeSeconds(15); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java new file mode 100644 index 0000000000..7d4f68e3bb --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: NettyRPCTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.remoting.netty.*; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import io.netty.channel.ChannelHandlerContext; +import org.junit.Test; + +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class NettyRPCTest { + @Test + public void test_RPC_Sync() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + TestRequestHeader requestHeader = new TestRequestHeader(); + requestHeader.setCount(i); + requestHeader.setMessageTitle("HelloMessageTitle"); + RemotingCommand request = RemotingCommand.createRequestCommand(0, requestHeader); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3000); + System.out.println("invoke result = " + response); + assertTrue(response != null); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + public static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + RemotingServer remotingServer = new NettyRemotingServer(config); + remotingServer.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + + @Override + public boolean rejectRequest() { + return false; + } + }, Executors.newCachedThreadPool()); + remotingServer.start(); + return remotingServer; + } + + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + + @Test + public void test_RPC_Oneway() throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + request.setRemark(String.valueOf(i)); + client.invokeOneway("localhost:8888", request, 1000 * 3); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + + @Test + public void test_RPC_Async() throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + request.setRemark(String.valueOf(i)); + client.invokeAsync("localhost:8888", request, 1000 * 3, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + System.out.println(responseFuture.getResponseCommand()); + } + }); + } + + Thread.sleep(1000 * 3); + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + + @Test + public void test_server_call_client() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + final RemotingServer server = createRemotingServer(); + final RemotingClient client = createRemotingClient(); + + server.registerProcessor(0, new NettyRequestProcessor() { + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + try { + return server.invokeSync(ctx.channel(), request, 1000 * 10); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (RemotingSendRequestException e) { + e.printStackTrace(); + } catch (RemotingTimeoutException e) { + e.printStackTrace(); + } + + return null; + } + + @Override + public boolean rejectRequest() { + return false; + } + }, Executors.newCachedThreadPool()); + + client.registerProcessor(0, new NettyRequestProcessor() { + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("client receive server request = " + request); + request.setRemark("client remark"); + return request; + } + + @Override + public boolean rejectRequest() { + return false; + } + }, Executors.newCachedThreadPool()); + + for (int i = 0; i < 3; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println("invoke result = " + response); + assertTrue(response != null); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + +} + + +class TestRequestHeader implements CommandCustomHeader { + @CFNullable + private Integer count; + + @CFNullable + private String messageTitle; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Integer getCount() { + return count; + } + + + public void setCount(Integer count) { + this.count = count; + } + + + public String getMessageTitle() { + return messageTitle; + } + + + public void setMessageTitle(String messageTitle) { + this.messageTitle = messageTitle; + } +} + + +class TestResponseHeader implements CommandCustomHeader { + @CFNullable + private Integer count; + + @CFNullable + private String messageTitle; + + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getMessageTitle() { + return messageTitle; + } + + public void setMessageTitle(String messageTitle) { + this.messageTitle = messageTitle; + } + + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java new file mode 100644 index 0000000000..fc3e708cb5 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: SyncInvokeTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class SyncInvokeTest { + @Test + public void test_RPC_Sync() throws Exception { + RemotingServer server = NettyRPCTest.createRemotingServer(); + RemotingClient client = NettyRPCTest.createRemotingClient(); + + for (int i = 0; i < 100; i++) { + try { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println(i + "\t" + "invoke result = " + response); + assertTrue(response != null); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java new file mode 100644 index 0000000000..3d18d2338a --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + */ +package com.alibaba.rocketmq.subclass; + +import org.junit.Test; + + +/** + * @author shijia.wxr + */ +public class TestSubClassAuto { + @Test + public void test_sub() { + + } +} diff --git a/rocketmq-srvutil/pom.xml b/rocketmq-srvutil/pom.xml new file mode 100644 index 0000000000..98b4e295aa --- /dev/null +++ b/rocketmq-srvutil/pom.xml @@ -0,0 +1,51 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-srvutil + rocketmq-srvutil ${project.version} + + + + + junit + junit + test + + + ${project.groupId} + rocketmq-remoting + + + ${project.groupId} + rocketmq-common + + + commons-cli + commons-cli + + + diff --git a/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java b/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java new file mode 100644 index 0000000000..72fa9a42c8 --- /dev/null +++ b/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.srvutil; + +import org.apache.commons.cli.*; + +import java.util.Properties; + +public class ServerUtil { + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("n", "namesrvAddr", true, + "Name server address list, eg: 192.168.0.1:9876;192.168.0.2:9876"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static CommandLine parseCmdLine(final String appName, String[] args, Options options, + CommandLineParser parser) { + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp(appName, options, true); + return null; + } + } catch (ParseException e) { + hf.printHelp(appName, options, true); + } + + return commandLine; + } + + + public static void printCommandLineHelp(final String appName, final Options options) { + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + hf.printHelp(appName, options, true); + } + + + public static Properties commandLine2Properties(final CommandLine commandLine) { + Properties properties = new Properties(); + Option[] opts = commandLine.getOptions(); + + if (opts != null) { + for (Option opt : opts) { + String name = opt.getLongOpt(); + String value = commandLine.getOptionValue(name); + if (value != null) { + properties.setProperty(name, value); + } + } + } + + return properties; + } + +} diff --git a/rocketmq-store/pom.xml b/rocketmq-store/pom.xml new file mode 100644 index 0000000000..440e521a37 --- /dev/null +++ b/rocketmq-store/pom.xml @@ -0,0 +1,46 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-store + rocketmq-store ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-common + + + net.java.dev.jna + jna + + + diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMappedFileService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMappedFileService.java new file mode 100644 index 0000000000..40eee7a660 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMappedFileService.java @@ -0,0 +1,342 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.config.BrokerRole; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; + + +/** + * Create MappedFile in advance + * + * @author shijia.wxr + */ +public class AllocateMappedFileService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static int waitTimeOut = 1000 * 5; + private ConcurrentHashMap requestTable = + new ConcurrentHashMap(); + private PriorityBlockingQueue requestQueue = + new PriorityBlockingQueue(); + private volatile boolean hasException = false; + private DefaultMessageStore messageStore; + + + public AllocateMappedFileService(DefaultMessageStore messageStore) { + this.messageStore = messageStore; + } + + + public MappedFile putRequestAndReturnMappedFile(String nextFilePath, String nextNextFilePath, int fileSize) { + int canSubmitRequests = 2; + if (this.messageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool() + && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool + canSubmitRequests = this.messageStore.getTransientStorePool().remainBufferNumbs() - this.requestQueue.size(); + } + } + + AllocateRequest nextReq = new AllocateRequest(nextFilePath, fileSize); + boolean nextPutOK = this.requestTable.putIfAbsent(nextFilePath, nextReq) == null; + + if (nextPutOK) { + if (canSubmitRequests <= 0) { + log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " + + "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().remainBufferNumbs()); + this.requestTable.remove(nextFilePath); + return null; + } + boolean offerOK = this.requestQueue.offer(nextReq); + if (!offerOK) { + log.warn("never expected here, add a request to preallocate queue failed"); + } + canSubmitRequests--; + } + + AllocateRequest nextNextReq = new AllocateRequest(nextNextFilePath, fileSize); + boolean nextNextPutOK = this.requestTable.putIfAbsent(nextNextFilePath, nextNextReq) == null; + if (nextNextPutOK) { + if (canSubmitRequests <= 0) { + log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " + + "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().remainBufferNumbs()); + this.requestTable.remove(nextNextFilePath); + } else { + boolean offerOK = this.requestQueue.offer(nextNextReq); + if (!offerOK) { + log.warn("never expected here, add a request to preallocate queue failed"); + } + } + } + + if (hasException) { + log.warn(this.getServiceName() + " service has exception. so return null"); + return null; + } + + AllocateRequest result = this.requestTable.get(nextFilePath); + try { + if (result != null) { + boolean waitOK = result.getCountDownLatch().await(waitTimeOut, TimeUnit.MILLISECONDS); + if (!waitOK) { + log.warn("create mmap timeout " + result.getFilePath() + " " + result.getFileSize()); + return null; + } else { + this.requestTable.remove(nextFilePath); + return result.getMappedFile(); + } + } else { + log.error("find preallocate mmap failed, this never happen"); + } + } catch (InterruptedException e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + + return null; + } + + + @Override + public String getServiceName() { + return AllocateMappedFileService.class.getSimpleName(); + } + + + public void shutdown() { + this.stopped = true; + this.thread.interrupt(); + + try { + this.thread.join(this.getJointime()); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + for (AllocateRequest req : this.requestTable.values()) { + if (req.mappedFile != null) { + log.info("delete pre allocated maped file, {}", req.mappedFile.getFileName()); + req.mappedFile.destroy(1000); + } + } + } + + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped() && this.mmapOperation()) { + + } + log.info(this.getServiceName() + " service end"); + } + + + /** + * Only interrupted by the external thread, will return false + */ + private boolean mmapOperation() { + boolean isSuccess = false; + AllocateRequest req = null; + try { + req = this.requestQueue.take(); + AllocateRequest expectedRequest = this.requestTable.get(req.getFilePath()); + if (null == expectedRequest) { + log.warn("this mmap request expired, maybe cause timeout " + req.getFilePath() + " " + + req.getFileSize()); + return true; + } + if (expectedRequest != req) { + log.warn("never expected here, maybe cause timeout " + req.getFilePath() + " " + + req.getFileSize() + ", req:" + req + ", expectedRequest:" + expectedRequest); + return true; + } + + if (req.getMappedFile() == null) { + long beginTime = System.currentTimeMillis(); + + MappedFile mappedFile; + if (messageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + try { + mappedFile = ServiceLoader.load(MappedFile.class).iterator().next(); + mappedFile.init(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool()); + } catch (RuntimeException e) { + log.warn("Use default implementation."); + mappedFile = new MappedFile(req.getFilePath(), req.getFileSize(), messageStore.getTransientStorePool()); + } + } else { + mappedFile = new MappedFile(req.getFilePath(), req.getFileSize()); + } + + long eclipseTime = UtilAll.computeEclipseTimeMilliseconds(beginTime); + if (eclipseTime > 10) { + int queueSize = this.requestQueue.size(); + log.warn("create mappedFile spent time(ms) " + eclipseTime + " queue size " + queueSize + + " " + req.getFilePath() + " " + req.getFileSize()); + } + + // pre write mappedFile + if (mappedFile.getFileSize() >= this.messageStore.getMessageStoreConfig() + .getMapedFileSizeCommitLog() + && + this.messageStore.getMessageStoreConfig().isWarmMapedFileEnable()) { + mappedFile.warmMappedFile(this.messageStore.getMessageStoreConfig().getFlushDiskType(), + this.messageStore.getMessageStoreConfig().getFlushLeastPagesWhenWarmMapedFile()); + } + + req.setMappedFile(mappedFile); + this.hasException = false; + isSuccess = true; + } + } catch (InterruptedException e) { + log.warn(this.getServiceName() + " service has exception, maybe by shutdown"); + this.hasException = true; + return false; + } catch (IOException e) { + log.warn(this.getServiceName() + " service has exception. ", e); + this.hasException = true; + if (null != req) { + requestQueue.offer(req); + try { + Thread.sleep(1); + } catch (InterruptedException e1) { + } + } + } finally { + if (req != null && isSuccess) + req.getCountDownLatch().countDown(); + } + return true; + } + + static class AllocateRequest implements Comparable { + // Full file path + private String filePath; + private int fileSize; + private CountDownLatch countDownLatch = new CountDownLatch(1); + private volatile MappedFile mappedFile = null; + + + public AllocateRequest(String filePath, int fileSize) { + this.filePath = filePath; + this.fileSize = fileSize; + } + + + public String getFilePath() { + return filePath; + } + + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + + public int getFileSize() { + return fileSize; + } + + + public void setFileSize(int fileSize) { + this.fileSize = fileSize; + } + + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + + public MappedFile getMappedFile() { + return mappedFile; + } + + + public void setMappedFile(MappedFile mappedFile) { + this.mappedFile = mappedFile; + } + + + public int compareTo(AllocateRequest other) { + if (this.fileSize < other.fileSize) + return 1; + else if (this.fileSize > other.fileSize) { + return -1; + } else { + int mIndex = this.filePath.lastIndexOf(File.separator); + long mName = Long.parseLong(this.filePath.substring(mIndex + 1)); + int oIndex = other.filePath.lastIndexOf(File.separator); + long oName = Long.parseLong(other.filePath.substring(oIndex + 1)); + if (mName < oName) { + return -1; + } else if (mName > oName) { + return 1; + } else { + return 0; + } + } + // return this.fileSize < other.fileSize ? 1 : this.fileSize > + // other.fileSize ? -1 : 0; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((filePath == null) ? 0 : filePath.hashCode()); + result = prime * result + fileSize; + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + AllocateRequest other = (AllocateRequest) obj; + if (filePath == null) { + if (other.filePath != null) + return false; + } else if (!filePath.equals(other.filePath)) + return false; + if (fileSize != other.fileSize) + return false; + return true; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java new file mode 100644 index 0000000000..778bd8805f --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; + + +/** + * Write messages callback interface + * + * @author shijia.wxr + * + */ +public interface AppendMessageCallback { + + /** + * After message serialization, write MapedByteBuffer + * + * @param byteBuffer + * @param maxBlank + * @param msg + * + * @return How many bytes to write + */ + AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, + final int maxBlank, final MessageExtBrokerInner msg); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java new file mode 100644 index 0000000000..a87a9179a5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * When write a message to the commit log, returns results + * + * @author shijia.wxr + */ +public class AppendMessageResult { + // Return code + private AppendMessageStatus status; + // Where to start writing + private long wroteOffset; + // Write Bytes + private int wroteBytes; + // Message ID + private String msgId; + // Message storage timestamp + private long storeTimestamp; + // Consume queue's offset(step by one) + private long logicsOffset; + private long pagecacheRT = 0; + + public AppendMessageResult(AppendMessageStatus status) { + this(status, 0, 0, "", 0, 0, 0); + } + + public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, String msgId, + long storeTimestamp, long logicsOffset, long pagecacheRT) { + this.status = status; + this.wroteOffset = wroteOffset; + this.wroteBytes = wroteBytes; + this.msgId = msgId; + this.storeTimestamp = storeTimestamp; + this.logicsOffset = logicsOffset; + this.pagecacheRT = pagecacheRT; + } + + public long getPagecacheRT() { + return pagecacheRT; + } + + public void setPagecacheRT(final long pagecacheRT) { + this.pagecacheRT = pagecacheRT; + } + + public boolean isOk() { + return this.status == AppendMessageStatus.PUT_OK; + } + + + public AppendMessageStatus getStatus() { + return status; + } + + + public void setStatus(AppendMessageStatus status) { + this.status = status; + } + + + public long getWroteOffset() { + return wroteOffset; + } + + + public void setWroteOffset(long wroteOffset) { + this.wroteOffset = wroteOffset; + } + + + public int getWroteBytes() { + return wroteBytes; + } + + + public void setWroteBytes(int wroteBytes) { + this.wroteBytes = wroteBytes; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } + + + public long getLogicsOffset() { + return logicsOffset; + } + + + public void setLogicsOffset(long logicsOffset) { + this.logicsOffset = logicsOffset; + } + + @Override + public String toString() { + return "AppendMessageResult{" + + "status=" + status + + ", wroteOffset=" + wroteOffset + + ", wroteBytes=" + wroteBytes + + ", msgId='" + msgId + '\'' + + ", storeTimestamp=" + storeTimestamp + + ", logicsOffset=" + logicsOffset + + ", pagecacheRT=" + pagecacheRT + + '}'; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java new file mode 100644 index 0000000000..97234c08c6 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * When write a message to the commit log, returns code + * + * @author shijia.wxr + * + */ +public enum AppendMessageStatus { + PUT_OK, + END_OF_FILE, + MESSAGE_SIZE_EXCEEDED, + PROPERTIES_SIZE_EXCEEDED, + UNKNOWN_ERROR, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java new file mode 100644 index 0000000000..23f305d22f --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java @@ -0,0 +1,1296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.FlushDiskType; +import com.alibaba.rocketmq.store.ha.HAService; +import com.alibaba.rocketmq.store.schedule.ScheduleMessageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * Store all metadata downtime for recovery, data protection reliability + * + * @author shijia.wxr + */ +public class CommitLog { + // Message's MAGIC CODE daa320a7 + public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + // End of file empty MAGIC CODE cbd43194 + private final static int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8; + private final MappedFileQueue mappedFileQueue; + private final DefaultMessageStore defaultMessageStore; + private final FlushCommitLogService flushCommitLogService; + + //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods + private final FlushCommitLogService commitLogService; + + private final AppendMessageCallback appendMessageCallback; + private HashMap topicQueueTable = new HashMap(1024); + private volatile long confirmOffset = -1L; + + private volatile long beginTimeInLock = 0; + + //true: Can lock, false : in lock. + private AtomicBoolean putMessageSpinLock = new AtomicBoolean(true); + + private ReentrantLock putMessageNormalLock = new ReentrantLock(); // NonfairSync + + public CommitLog(final DefaultMessageStore defaultMessageStore) { + this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(), + defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), defaultMessageStore.getAllocateMappedFileService()); + this.defaultMessageStore = defaultMessageStore; + + if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { + this.flushCommitLogService = new GroupCommitService(); + } else { + this.flushCommitLogService = new FlushRealTimeService(); + } + + this.commitLogService = new CommitRealTimeService(); + + this.appendMessageCallback = new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize()); + } + + public boolean load() { + boolean result = this.mappedFileQueue.load(); + log.info("load commit log " + (result ? "OK" : "Failed")); + return result; + } + + public void start() { + this.flushCommitLogService.start(); + + if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + this.commitLogService.start(); + } + } + + public void shutdown() { + if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + this.commitLogService.shutdown(); + } + + this.flushCommitLogService.shutdown(); + } + + public long flush() { + this.mappedFileQueue.commit(0); + this.mappedFileQueue.flush(0); + return this.mappedFileQueue.getFlushedWhere(); + } + + public long getMaxOffset() { + return this.mappedFileQueue.getMaxOffset(); + } + + public long remainHowManyDataToCommit() { + return this.mappedFileQueue.remainHowManyDataToCommit(); + } + + public long remainHowManyDataToFlush() { + return this.mappedFileQueue.remainHowManyDataToFlush(); + } + + + public int deleteExpiredFile(// + final long expiredTime, // + final int deleteFilesInterval, // + final long intervalForcibly, // + final boolean cleanImmediately// + ) { + return this.mappedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); + } + + + /** + * Read CommitLog data, use data replication + */ + public SelectMappedBufferResult getData(final long offset) { + return this.getData(offset, offset == 0); + } + + + public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { + int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, returnFirstOnNotFound); + if (mappedFile != null) { + int pos = (int) (offset % mappedFileSize); + SelectMappedBufferResult result = mappedFile.selectMappedBuffer(pos); + return result; + } + + return null; + } + + + /** + * When the normal exit, data recovery, all memory data have been flush + */ + public void recoverNormally() { + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + final List mappedFiles = this.mappedFileQueue.getMappedFiles(); + if (!mappedFiles.isEmpty()) { + // Began to recover from the last third file + int index = mappedFiles.size() - 3; + if (index < 0) + index = 0; + + MappedFile mappedFile = mappedFiles.get(index); + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + long processOffset = mappedFile.getFileFromOffset(); + long mappedFileOffset = 0; + while (true) { + DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); + int size = dispatchRequest.getMsgSize(); + // Normal data + if (dispatchRequest.isSuccess() && size > 0) { + mappedFileOffset += size; + } + // Come the end of the file, switch to the next file Since the + // return 0 representatives met last hole, + // this can not be included in truncate offset + else if (dispatchRequest.isSuccess() && size == 0) { + index++; + if (index >= mappedFiles.size()) { + // Current branch can not happen + log.info("recover last 3 physics file over, last maped file " + mappedFile.getFileName()); + break; + } else { + mappedFile = mappedFiles.get(index); + byteBuffer = mappedFile.sliceByteBuffer(); + processOffset = mappedFile.getFileFromOffset(); + mappedFileOffset = 0; + log.info("recover next physics file, " + mappedFile.getFileName()); + } + } + // Intermediate file read error + else if (!dispatchRequest.isSuccess()) { + log.info("recover physics file end, " + mappedFile.getFileName()); + break; + } + } + + processOffset += mappedFileOffset; + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); + } + } + + public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC) { + return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true); + } + + private void doNothingForDeadCode(final Object obj) { + if (obj != null) { + if (log.isDebugEnabled()) { + log.debug(String.valueOf(obj.hashCode())); + } + } + } + + /** + * check the message and returns the message size + * + * @return 0 Come the end of the file // >0 Normal messages // -1 Message + * checksum failure + */ + public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) { + try { + // 1 TOTAL SIZE + int totalSize = byteBuffer.getInt(); + + // 2 MAGIC CODE + int magicCode = byteBuffer.getInt(); + switch (magicCode) { + case MESSAGE_MAGIC_CODE: + break; + case BLANK_MAGIC_CODE: + return new DispatchRequest(0, true /* success */); + default: + log.warn("found a illegal magic code 0x" + Integer.toHexString(magicCode)); + return new DispatchRequest(-1, false /* success */); + } + + byte[] bytesContent = new byte[totalSize]; + + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + + // 5 FLAG + int flag = byteBuffer.getInt(); + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + + // 10 + ByteBuffer byteBuffer1 = byteBuffer.get(bytesContent, 0, 8); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + + // 12 + ByteBuffer byteBuffer2 = byteBuffer.get(bytesContent, 0, 8); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + if (readBody) { + byteBuffer.get(bytesContent, 0, bodyLen); + + if (checkCRC) { + int crc = UtilAll.crc32(bytesContent, 0, bodyLen); + if (crc != bodyCRC) { + log.warn("CRC check failed. bodyCRC={}, currentCRC={}", crc, bodyCRC); + return new DispatchRequest(-1, false/* success */); + } + } + } else { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byteBuffer.get(bytesContent, 0, topicLen); + String topic = new String(bytesContent, 0, topicLen, MessageDecoder.CHARSET_UTF8); + + long tagsCode = 0; + String keys = ""; + String uniqKey = null; + + // 17 properties + short propertiesLength = byteBuffer.getShort(); + if (propertiesLength > 0) { + byteBuffer.get(bytesContent, 0, propertiesLength); + String properties = new String(bytesContent, 0, propertiesLength, MessageDecoder.CHARSET_UTF8); + Map propertiesMap = MessageDecoder.string2messageProperties(properties); + + keys = propertiesMap.get(MessageConst.PROPERTY_KEYS); + + uniqKey = propertiesMap.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + + String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS); + if (tags != null && tags.length() > 0) { + tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags); + } + + // Timing message processing + { + String t = propertiesMap.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL); + if (ScheduleMessageService.SCHEDULE_TOPIC.equals(topic) && t != null) { + int delayLevel = Integer.parseInt(t); + + if (delayLevel > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { + delayLevel = this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel(); + } + + if (delayLevel > 0) { + tagsCode = this.defaultMessageStore.getScheduleMessageService().computeDeliverTimestamp(delayLevel, + storeTimestamp); + } + } + } + } + + int readLength = calMsgLength(bodyLen, topicLen, propertiesLength); + if (totalSize != readLength) { + doNothingForDeadCode(reconsumeTimes); + doNothingForDeadCode(flag); + doNothingForDeadCode(bornTimeStamp); + doNothingForDeadCode(byteBuffer1); + doNothingForDeadCode(byteBuffer2); + log.error( + "[BUG]read total count not equals msg total size. totalSize={}, readTotalCount={}, bodyLen={}, topicLen={}, propertiesLength={}", + totalSize, readLength, bodyLen, topicLen, propertiesLength); + return new DispatchRequest(totalSize, false/* success */); + } + + return new DispatchRequest(// + topic, // 1 + queueId, // 2 + physicOffset, // 3 + totalSize, // 4 + tagsCode, // 5 + storeTimestamp, // 6 + queueOffset, // 7 + keys, // 8 + uniqKey, //9 + sysFlag, // 9 + preparedTransactionOffset// 10 + ); + } catch (Exception e) { + } + + return new DispatchRequest(-1, false /* success */); + } + + private int calMsgLength(int bodyLength, int topicLength, int propertiesLength) { + final int msgLen = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + 8 // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + 8 // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8 // 14 Prepared Transaction Offset + + 4 + (bodyLength > 0 ? bodyLength : 0) // 14 BODY + + 1 + topicLength // 15 TOPIC + + 2 + (propertiesLength > 0 ? propertiesLength : 0) // 16 + // propertiesLength + + 0; + return msgLen; + } + + public long getConfirmOffset() { + return this.confirmOffset; + } + + public void setConfirmOffset(long phyOffset) { + this.confirmOffset = phyOffset; + } + + public void recoverAbnormally() { + // recover by the minimum time stamp + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + final List mappedFiles = this.mappedFileQueue.getMappedFiles(); + if (!mappedFiles.isEmpty()) { + // Looking beginning to recover from which file + int index = mappedFiles.size() - 1; + MappedFile mappedFile = null; + for (; index >= 0; index--) { + mappedFile = mappedFiles.get(index); + if (this.isMappedFileMatchedRecover(mappedFile)) { + log.info("recover from this maped file " + mappedFile.getFileName()); + break; + } + } + + if (index < 0) { + index = 0; + mappedFile = mappedFiles.get(index); + } + + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + long processOffset = mappedFile.getFileFromOffset(); + long mappedFileOffset = 0; + while (true) { + DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); + int size = dispatchRequest.getMsgSize(); + + // Normal data + if (size > 0) { + mappedFileOffset += size; + + + if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) { + if (dispatchRequest.getCommitLogOffset() < this.defaultMessageStore.getConfirmOffset()) { + this.defaultMessageStore.doDispatch(dispatchRequest); + } + } else { + this.defaultMessageStore.doDispatch(dispatchRequest); + } + } + // Intermediate file read error + else if (size == -1) { + log.info("recover physics file end, " + mappedFile.getFileName()); + break; + } + // Come the end of the file, switch to the next file + // Since the return 0 representatives met last hole, this can + // not be included in truncate offset + else if (size == 0) { + index++; + if (index >= mappedFiles.size()) { + // The current branch under normal circumstances should + // not happen + log.info("recover physics file over, last maped file " + mappedFile.getFileName()); + break; + } else { + mappedFile = mappedFiles.get(index); + byteBuffer = mappedFile.sliceByteBuffer(); + processOffset = mappedFile.getFileFromOffset(); + mappedFileOffset = 0; + log.info("recover next physics file, " + mappedFile.getFileName()); + } + } + } + + processOffset += mappedFileOffset; + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); + + // Clear ConsumeQueue redundant data + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } + // Commitlog case files are deleted + else { + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); + this.defaultMessageStore.destroyLogics(); + } + } + + private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + + int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSTION); + if (magicCode != MESSAGE_MAGIC_CODE) { + return false; + } + + long storeTimestamp = byteBuffer.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + if (0 == storeTimestamp) { + return false; + } + + if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()// + && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { + log.info("find check timestamp, {} {}", // + storeTimestamp, // + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } else { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { + log.info("find check timestamp, {} {}", // + storeTimestamp, // + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } + + return false; + } + + private void notifyMessageArriving() { + + } + + public boolean resetOffset(long offset) { + return this.mappedFileQueue.resetOffset(offset); + } + + public long getBeginTimeInLock() { + return beginTimeInLock; + } + + public PutMessageResult putMessage(final MessageExtBrokerInner msg) { + // Set the storage time + msg.setStoreTimestamp(System.currentTimeMillis()); + // Set the message body BODY CRC (consider the most appropriate setting + // on the client) + msg.setBodyCRC(UtilAll.crc32(msg.getBody())); + // Back to Results + AppendMessageResult result = null; + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + String topic = msg.getTopic(); + int queueId = msg.getQueueId(); + + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE// + || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { + // Delay Delivery + if (msg.getDelayTimeLevel() > 0) { + if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { + msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); + } + + topic = ScheduleMessageService.SCHEDULE_TOPIC; + queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); + + // Backup real topic, queueId + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + + msg.setTopic(topic); + msg.setQueueId(queueId); + } + } + + long eclipseTimeInLock = 0; + MappedFile unlockMappedFile = null; + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + + lockForPutMessage(); //spin... + try { + long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); + this.beginTimeInLock = beginLockTimestamp; + + // Here settings are stored timestamp, in order to ensure an orderly + // global + msg.setStoreTimestamp(beginLockTimestamp); + + if (null == mappedFile || mappedFile.isFull()) { + mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise + } + if (null == mappedFile) { + log.error("create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); + beginTimeInLock = 0; + return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); + } + + result = mappedFile.appendMessage(msg, this.appendMessageCallback); + switch (result.getStatus()) { + case PUT_OK: + break; + case END_OF_FILE: + unlockMappedFile = mappedFile; + // Create a new file, re-write the message + mappedFile = this.mappedFileQueue.getLastMappedFile(0); + if (null == mappedFile) { + // XXX: warn and notify me + log.error("create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); + beginTimeInLock = 0; + return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); + } + result = mappedFile.appendMessage(msg, this.appendMessageCallback); + break; + case MESSAGE_SIZE_EXCEEDED: + case PROPERTIES_SIZE_EXCEEDED: + beginTimeInLock = 0; + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); + case UNKNOWN_ERROR: + beginTimeInLock = 0; + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + default: + beginTimeInLock = 0; + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + + eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp; + beginTimeInLock = 0; + } finally { + releasePutMessageLock(); + } + + if (eclipseTimeInLock > 500) { + log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, result); + } + + if (null != unlockMappedFile && this.defaultMessageStore.getMessageStoreConfig().isWarmMapedFileEnable()) { + this.defaultMessageStore.unlockMappedFile(unlockMappedFile); + } + + + PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result); + + // Statistics + storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet(); + storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); + + GroupCommitRequest request = null; + + // Synchronization flush + if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { + final GroupCommitService service = (GroupCommitService) this.flushCommitLogService; + if (msg.isWaitStoreMsgOK()) { + request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); + service.putRequest(request); + boolean flushOK = request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); + if (!flushOK) { + log.error("do groupcommit, wait for flush failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + + " client address: " + msg.getBornHostString()); + putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); + } + } else { + service.wakeup(); + } + } + // Asynchronous flush + else { + if (!this.defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + flushCommitLogService.wakeup(); + } else { + commitLogService.wakeup(); + } + } + + // Synchronous write double + if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { + HAService service = this.defaultMessageStore.getHaService(); + if (msg.isWaitStoreMsgOK()) { + // Determine whether to wait + if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) { + if (null == request) { + request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); + } + service.putRequest(request); + + service.getWaitNotifyObject().wakeupAll(); + + boolean flushOK = + // TODO + request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); + if (!flushOK) { + log.error("do sync transfer other node, wait return, but failed, topic: " + msg.getTopic() + " tags: " + + msg.getTags() + " client address: " + msg.getBornHostString()); + putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT); + } + } + // Slave problem + else { + // Tell the producer, slave not available + putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE); + } + } + } + + return putMessageResult; + } + + /** + * According to receive certain message or offset storage time if an error + * occurs, it returns -1 + */ + public long pickupStoreTimestamp(final long offset, final int size) { + if (offset >= this.getMinOffset()) { + SelectMappedBufferResult result = this.getMessage(offset, size); + if (null != result) { + try { + return result.getByteBuffer().getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + } finally { + result.release(); + } + } + } + + return -1; + } + + public long getMinOffset() { + MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile(); + if (mappedFile != null) { + if (mappedFile.isAvailable()) { + return mappedFile.getFileFromOffset(); + } else { + return this.rollNextFile(mappedFile.getFileFromOffset()); + } + } + + return -1; + } + + public SelectMappedBufferResult getMessage(final long offset, final int size) { + int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset, offset == 0); + if (mappedFile != null) { + int pos = (int) (offset % mappedFileSize); + return mappedFile.selectMappedBuffer(pos, size); + } + return null; + } + + public long rollNextFile(final long offset) { + int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + return offset + mappedFileSize - offset % mappedFileSize; + } + + public HashMap getTopicQueueTable() { + return topicQueueTable; + } + + + public void setTopicQueueTable(HashMap topicQueueTable) { + this.topicQueueTable = topicQueueTable; + } + + + public void destroy() { + this.mappedFileQueue.destroy(); + } + + + public boolean appendData(long startOffset, byte[] data) { + lockForPutMessage(); //spin... + try { + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(startOffset); + if (null == mappedFile) { + log.error("appendData getLastMappedFile error " + startOffset); + return false; + } + + return mappedFile.appendMessage(data); + } finally { + releasePutMessageLock(); + } + } + + + public boolean retryDeleteFirstFile(final long intervalForcibly) { + return this.mappedFileQueue.retryDeleteFirstFile(intervalForcibly); + } + + public void removeQueueFromTopicQueueTable(final String topic, final int queueId) { + String key = topic + "-" + queueId; + synchronized (this) { + this.topicQueueTable.remove(key); + } + + log.info("removeQueueFromTopicQueueTable OK Topic: {} QueueId: {}", topic, queueId); + } + + public void checkSelf() { + mappedFileQueue.checkSelf(); + } + + abstract class FlushCommitLogService extends ServiceThread { + protected static final int RETRY_TIMES_OVER = 10; + } + + class CommitRealTimeService extends FlushCommitLogService { + + private long lastCommitTimestamp = 0; + + @Override + public String getServiceName() { + return CommitRealTimeService.class.getSimpleName(); + } + + @Override + public void run() { + CommitLog.log.info(this.getServiceName() + " service started"); + while (!this.isStopped()) { + int interval = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitIntervalCommitLog(); + + int commitDataLeastPages = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitCommitLogLeastPages(); + + int commitDataThoroughInterval = + CommitLog.this.defaultMessageStore.getMessageStoreConfig().getCommitCommitLogThoroughInterval(); + + long begin = System.currentTimeMillis(); + if (begin >= (this.lastCommitTimestamp + commitDataThoroughInterval)) { + this.lastCommitTimestamp = begin; + commitDataLeastPages = 0; + } + + try { + boolean result = CommitLog.this.mappedFileQueue.commit(commitDataLeastPages); + long end = System.currentTimeMillis(); + if (!result) { + this.lastCommitTimestamp = end; // result = false means some data committed. + //now wake up flush thread. + flushCommitLogService.wakeup(); + } + + if (end - begin > 500) { + log.info("Commit data to file costs {} ms", end - begin); + } + this.waitForRunning(interval); + } catch (Throwable e) { + CommitLog.log.error(this.getServiceName() + " service has exception. ", e); + } + } + + boolean result = false; + for (int i = 0; i < RETRY_TIMES_OVER && !result; i++) { + result = CommitLog.this.mappedFileQueue.commit(0); + CommitLog.log.info(this.getServiceName() + " service shutdown, retry " + (i + 1) + " times " + (result ? "OK" : "Not OK")); + } + CommitLog.log.info(this.getServiceName() + " service end"); + } + } + + class FlushRealTimeService extends FlushCommitLogService { + private long lastFlushTimestamp = 0; + private long printTimes = 0; + + + public void run() { + CommitLog.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + boolean flushCommitLogTimed = CommitLog.this.defaultMessageStore.getMessageStoreConfig().isFlushCommitLogTimed(); + + int interval = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushIntervalCommitLog(); + int flushPhysicQueueLeastPages = CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushCommitLogLeastPages(); + + int flushPhysicQueueThoroughInterval = + CommitLog.this.defaultMessageStore.getMessageStoreConfig().getFlushCommitLogThoroughInterval(); + + boolean printFlushProgress = false; + + // Print flush progress + long currentTimeMillis = System.currentTimeMillis(); + if (currentTimeMillis >= (this.lastFlushTimestamp + flushPhysicQueueThoroughInterval)) { + this.lastFlushTimestamp = currentTimeMillis; + flushPhysicQueueLeastPages = 0; + printFlushProgress = (printTimes++ % 10) == 0; + } + + try { + if (flushCommitLogTimed) { + Thread.sleep(interval); + } else { + this.waitForRunning(interval); + } + + if (printFlushProgress) { + this.printFlushProgress(); + } + + long begin = System.currentTimeMillis(); + CommitLog.this.mappedFileQueue.flush(flushPhysicQueueLeastPages); + long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp(); + if (storeTimestamp > 0) { + CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp); + } + long past = System.currentTimeMillis() - begin; + if (past > 500) { + log.info("Flush data to disk costs {} ms", past); + } + } catch (Throwable e) { + CommitLog.log.warn(this.getServiceName() + " service has exception. ", e); + this.printFlushProgress(); + } + } + + // Normal shutdown, to ensure that all the flush before exit + boolean result = false; + for (int i = 0; i < RETRY_TIMES_OVER && !result; i++) { + result = CommitLog.this.mappedFileQueue.flush(0); + CommitLog.log.info(this.getServiceName() + " service shutdown, retry " + (i + 1) + " times " + (result ? "OK" : "Not OK")); + } + + this.printFlushProgress(); + + CommitLog.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return FlushRealTimeService.class.getSimpleName(); + } + + + private void printFlushProgress() { + // CommitLog.log.info("how much disk fall behind memory, " + // + CommitLog.this.mappedFileQueue.howMuchFallBehind()); + } + + + @Override + public long getJointime() { + return 1000 * 60 * 5; + } + } + + public static class GroupCommitRequest { + private final long nextOffset; + private final CountDownLatch countDownLatch = new CountDownLatch(1); + private volatile boolean flushOK = false; + + + public GroupCommitRequest(long nextOffset) { + this.nextOffset = nextOffset; + } + + + public long getNextOffset() { + return nextOffset; + } + + + public void wakeupCustomer(final boolean flushOK) { + this.flushOK = flushOK; + this.countDownLatch.countDown(); + } + + + public boolean waitForFlush(long timeout) { + try { + this.countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + return this.flushOK; + } catch (InterruptedException e) { + e.printStackTrace(); + return false; + } + } + } + + /** + * GroupCommit Service + */ + class GroupCommitService extends FlushCommitLogService { + private volatile List requestsWrite = new ArrayList(); + private volatile List requestsRead = new ArrayList(); + + + public void putRequest(final GroupCommitRequest request) { + synchronized (this) { + this.requestsWrite.add(request); + if (hasNotified.compareAndSet(false, true)) { + waitPoint.countDown(); // notify + } + } + } + + + private void swapRequests() { + List tmp = this.requestsWrite; + this.requestsWrite = this.requestsRead; + this.requestsRead = tmp; + } + + + private void doCommit() { + if (!this.requestsRead.isEmpty()) { + for (GroupCommitRequest req : this.requestsRead) { + // There may be a message in the next file, so a maximum of + // two times the flush + boolean flushOK = false; + for (int i = 0; i < 2 && !flushOK; i++) { + flushOK = CommitLog.this.mappedFileQueue.getFlushedWhere() >= req.getNextOffset(); + + if (!flushOK) { + CommitLog.this.mappedFileQueue.flush(0); + } + } + + req.wakeupCustomer(flushOK); + } + + long storeTimestamp = CommitLog.this.mappedFileQueue.getStoreTimestamp(); + if (storeTimestamp > 0) { + CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimestamp); + } + + this.requestsRead.clear(); + } else { + // Because of individual messages is set to not sync flush, it + // will come to this process + CommitLog.this.mappedFileQueue.flush(0); + } + } + + + public void run() { + CommitLog.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.waitForRunning(0); + this.doCommit(); + } catch (Exception e) { + CommitLog.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + // Under normal circumstances shutdown, wait for the arrival of the + // request, and then flush + try { + Thread.sleep(10); + } catch (InterruptedException e) { + CommitLog.log.warn("GroupCommitService Exception, ", e); + } + + synchronized (this) { + this.swapRequests(); + } + + this.doCommit(); + + CommitLog.log.info(this.getServiceName() + " service end"); + } + + + @Override + protected void onWaitEnd() { + this.swapRequests(); + } + + + @Override + public String getServiceName() { + return GroupCommitService.class.getSimpleName(); + } + + + @Override + public long getJointime() { + return 1000 * 60 * 5; + } + } + + class DefaultAppendMessageCallback implements AppendMessageCallback { + // File at the end of the minimum fixed length empty + private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; + private final ByteBuffer msgIdMemory; + // Store the message content + private final ByteBuffer msgStoreItemMemory; + // The maximum length of the message + private final int maxMessageSize; + // Build Message Key + private final StringBuilder keyBuilder = new StringBuilder(); + + private final ByteBuffer hostHolder = ByteBuffer.allocate(8); + + + DefaultAppendMessageCallback(final int size) { + this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); + this.maxMessageSize = size; + } + + + public ByteBuffer getMsgStoreItemMemory() { + return msgStoreItemMemory; + } + + + public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank, final MessageExtBrokerInner msgInner) { + // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
+ + // PHY OFFSET + long wroteOffset = fileFromOffset + byteBuffer.position(); + + this.resetByteBuffer(hostHolder, 8); + String msgId = MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(hostHolder), wroteOffset); + + // Record ConsumeQueue information + keyBuilder.setLength(0); + keyBuilder.append(msgInner.getTopic()); + keyBuilder.append('-'); + keyBuilder.append(msgInner.getQueueId()); + String key = keyBuilder.toString(); + Long queueOffset = CommitLog.this.topicQueueTable.get(key); + if (null == queueOffset) { + queueOffset = 0L; + CommitLog.this.topicQueueTable.put(key, queueOffset); + } + + // Transaction messages that require special handling + final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag()); + switch (tranType) { + // Prepared and Rollback message is not consumed, will not enter the + // consumer queuec + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + queueOffset = 0L; + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + default: + break; + } + + /** + * Serialize message + */ + final byte[] propertiesData = + msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8); + + final short propertiesLength = propertiesData == null ? 0 : (short) propertiesData.length; + + if (propertiesLength > Short.MAX_VALUE) { + log.warn("putMessage message properties length too long. length={}", propertiesData.length); + return new AppendMessageResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED); + } + + final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); + final int topicLength = topicData == null ? 0 : topicData.length; + + final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length; + + final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength); + + // Exceeds the maximum message + if (msgLen > this.maxMessageSize) { + CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength + + ", maxMessageSize: " + this.maxMessageSize); + return new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED); + } + + // Determines whether there is sufficient free space + if ((msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) { + this.resetByteBuffer(this.msgStoreItemMemory, maxBlank); + // 1 TOTALSIZE + this.msgStoreItemMemory.putInt(maxBlank); + // 2 MAGICCODE + this.msgStoreItemMemory.putInt(CommitLog.BLANK_MAGIC_CODE); + // 3 The remaining space may be any value + // + + // Here the length of the specially set maxBlank + final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); + byteBuffer.put(this.msgStoreItemMemory.array(), 0, maxBlank); + return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgId, msgInner.getStoreTimestamp(), + queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); + } + + // Initialization of storage space + this.resetByteBuffer(msgStoreItemMemory, msgLen); + // 1 TOTALSIZE + this.msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + this.msgStoreItemMemory.putInt(CommitLog.MESSAGE_MAGIC_CODE); + // 3 BODYCRC + this.msgStoreItemMemory.putInt(msgInner.getBodyCRC()); + // 4 QUEUEID + this.msgStoreItemMemory.putInt(msgInner.getQueueId()); + // 5 FLAG + this.msgStoreItemMemory.putInt(msgInner.getFlag()); + // 6 QUEUEOFFSET + this.msgStoreItemMemory.putLong(queueOffset); + // 7 PHYSICALOFFSET + this.msgStoreItemMemory.putLong(fileFromOffset + byteBuffer.position()); + // 8 SYSFLAG + this.msgStoreItemMemory.putInt(msgInner.getSysFlag()); + // 9 BORNTIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + // 10 BORNHOST + this.resetByteBuffer(hostHolder, 8); + this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder)); + // 11 STORETIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + this.resetByteBuffer(hostHolder, 8); + this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder)); + //this.msgStoreItemMemory.put(msgInner.getStoreHostBytes()); + // 13 RECONSUMETIMES + this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + // 14 Prepared Transaction Offset + this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + // 15 BODY + this.msgStoreItemMemory.putInt(bodyLength); + if (bodyLength > 0) + this.msgStoreItemMemory.put(msgInner.getBody()); + // 16 TOPIC + this.msgStoreItemMemory.put((byte) topicLength); + this.msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + this.msgStoreItemMemory.putShort(propertiesLength); + if (propertiesLength > 0) + this.msgStoreItemMemory.put(propertiesData); + + final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); + // Write messages to the queue buffer + byteBuffer.put(this.msgStoreItemMemory.array(), 0, msgLen); + + AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgId, + msgInner.getStoreTimestamp(), queueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); + + switch (tranType) { + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + // The next update ConsumeQueue information + CommitLog.this.topicQueueTable.put(key, ++queueOffset); + break; + default: + break; + } + return result; + } + + + private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) { + byteBuffer.flip(); + byteBuffer.limit(limit); + } + } + + public long lockTimeMills() { + long diff = 0; + long begin = this.beginTimeInLock; + if (begin > 0) { + diff = this.defaultMessageStore.now() - begin; + } + + if (diff < 0) { + diff = 0; + } + + return diff; + } + + /** + * Spin util acquired the lock. + */ + private void lockForPutMessage() { + if (this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()) { + putMessageNormalLock.lock(); + } else { + boolean flag; + do { + flag = this.putMessageSpinLock.compareAndSet(true, false); + } while (!flag); + } + } + + private void releasePutMessageLock() { + if (this.defaultMessageStore.getMessageStoreConfig().isUseReentrantLockWhenPutMessage()) { + putMessageNormalLock.unlock(); + } else { + this.putMessageSpinLock.compareAndSet(false, true); + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java new file mode 100644 index 0000000000..f23eb1c449 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java @@ -0,0 +1,495 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class ConsumeQueue { + + public static final int CQ_STORE_UNIT_SIZE = 20; + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + private final DefaultMessageStore defaultMessageStore; + + private final MappedFileQueue mappedFileQueue; + private final String topic; + private final int queueId; + private final ByteBuffer byteBufferIndex; + + private final String storePath; + private final int mappedFileSize; + private long maxPhysicOffset = -1; + private volatile long minLogicOffset = 0; + + + public ConsumeQueue( + final String topic, + final int queueId, + final String storePath, + final int mappedFileSize, + final DefaultMessageStore defaultMessageStore) { + this.storePath = storePath; + this.mappedFileSize = mappedFileSize; + this.defaultMessageStore = defaultMessageStore; + + this.topic = topic; + this.queueId = queueId; + + String queueDir = this.storePath + + File.separator + topic + + File.separator + queueId; + + this.mappedFileQueue = new MappedFileQueue(queueDir, mappedFileSize, null); + + this.byteBufferIndex = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE); + } + + + public boolean load() { + boolean result = this.mappedFileQueue.load(); + log.info("load consume queue " + this.topic + "-" + this.queueId + " " + (result ? "OK" : "Failed")); + return result; + } + + + public void recover() { + final List mappedFiles = this.mappedFileQueue.getMappedFiles(); + if (!mappedFiles.isEmpty()) { + + int index = mappedFiles.size() - 3; + if (index < 0) + index = 0; + + int mapedFileSizeLogics = this.mappedFileSize; + MappedFile mappedFile = mappedFiles.get(index); + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + long processOffset = mappedFile.getFileFromOffset(); + long mapedFileOffset = 0; + while (true) { + for (int i = 0; i < mapedFileSizeLogics; i += CQ_STORE_UNIT_SIZE) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + long tagsCode = byteBuffer.getLong(); + + if (offset >= 0 && size > 0) { + mapedFileOffset = i + CQ_STORE_UNIT_SIZE; + this.maxPhysicOffset = offset; + } else { + log.info("recover current consume queue file over, " + mappedFile.getFileName() + " " + + offset + " " + size + " " + tagsCode); + break; + } + } + + + if (mapedFileOffset == mapedFileSizeLogics) { + index++; + if (index >= mappedFiles.size()) { + + log.info("recover last consume queue file over, last maped file " + + mappedFile.getFileName()); + break; + } else { + mappedFile = mappedFiles.get(index); + byteBuffer = mappedFile.sliceByteBuffer(); + processOffset = mappedFile.getFileFromOffset(); + mapedFileOffset = 0; + log.info("recover next consume queue file, " + mappedFile.getFileName()); + } + } else { + log.info("recover current consume queue queue over " + mappedFile.getFileName() + " " + + (processOffset + mapedFileOffset)); + break; + } + } + + processOffset += mapedFileOffset; + this.mappedFileQueue.setFlushedWhere(processOffset); + this.mappedFileQueue.setCommittedWhere(processOffset); + this.mappedFileQueue.truncateDirtyFiles(processOffset); + } + } + + public long getOffsetInQueueByTime(final long timestamp) { + MappedFile mappedFile = this.mappedFileQueue.getMappedFileByTime(timestamp); + if (mappedFile != null) { + long offset = 0; + int low = + minLogicOffset > mappedFile.getFileFromOffset() ? (int) (minLogicOffset - mappedFile + .getFileFromOffset()) : 0; + int high = 0; + int midOffset = -1, targetOffset = -1, leftOffset = -1, rightOffset = -1; + long leftIndexValue = -1L, rightIndexValue = -1L; + long minPhysicOffset = this.defaultMessageStore.getMinPhyOffset(); + SelectMappedBufferResult sbr = mappedFile.selectMappedBuffer(0); + if (null != sbr) { + ByteBuffer byteBuffer = sbr.getByteBuffer(); + high = byteBuffer.limit() - CQ_STORE_UNIT_SIZE; + try { + while (high >= low) { + midOffset = (low + high) / (2 * CQ_STORE_UNIT_SIZE) * CQ_STORE_UNIT_SIZE; + byteBuffer.position(midOffset); + long phyOffset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + if (phyOffset < minPhysicOffset) { + low = midOffset + CQ_STORE_UNIT_SIZE; + leftOffset = midOffset; + continue; + } + + long storeTime = + this.defaultMessageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); + if (storeTime < 0) { + return 0; + } else if (storeTime == timestamp) { + targetOffset = midOffset; + break; + } else if (storeTime > timestamp) { + high = midOffset - CQ_STORE_UNIT_SIZE; + rightOffset = midOffset; + rightIndexValue = storeTime; + } else { + low = midOffset + CQ_STORE_UNIT_SIZE; + leftOffset = midOffset; + leftIndexValue = storeTime; + } + } + + if (targetOffset != -1) { + + offset = targetOffset; + } else { + if (leftIndexValue == -1) { + + offset = rightOffset; + } else if (rightIndexValue == -1) { + + offset = leftOffset; + } else { + offset = + Math.abs(timestamp - leftIndexValue) > Math.abs(timestamp + - rightIndexValue) ? rightOffset : leftOffset; + } + } + + return (mappedFile.getFileFromOffset() + offset) / CQ_STORE_UNIT_SIZE; + } finally { + sbr.release(); + } + } + } + return 0; + } + + public void truncateDirtyLogicFiles(long phyOffet) { + + int logicFileSize = this.mappedFileSize; + + this.maxPhysicOffset = phyOffet - 1; + + while (true) { + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + if (mappedFile != null) { + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + + mappedFile.setWrotePosition(0); + mappedFile.setCommittedPosition(0); + mappedFile.setFlushedPosition(0); + + for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + byteBuffer.getLong(); + + + if (0 == i) { + if (offset >= phyOffet) { + this.mappedFileQueue.deleteLastMappedFile(); + break; + } else { + int pos = i + CQ_STORE_UNIT_SIZE; + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); + this.maxPhysicOffset = offset; + } + } else { + + if (offset >= 0 && size > 0) { + + if (offset >= phyOffet) { + return; + } + + int pos = i + CQ_STORE_UNIT_SIZE; + mappedFile.setWrotePosition(pos); + mappedFile.setCommittedPosition(pos); + mappedFile.setFlushedPosition(pos); + this.maxPhysicOffset = offset; + + + if (pos == logicFileSize) { + return; + } + } else { + return; + } + } + } + } else { + break; + } + } + } + + public long getLastOffset() { + long lastOffset = -1; + + int logicFileSize = this.mappedFileSize; + + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + if (mappedFile != null) { + + int position = mappedFile.getWrotePosition() - CQ_STORE_UNIT_SIZE; + if (position < 0) + position = 0; + + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + byteBuffer.position(position); + for (int i = 0; i < logicFileSize; i += CQ_STORE_UNIT_SIZE) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + byteBuffer.getLong(); + + + if (offset >= 0 && size > 0) { + lastOffset = offset + size; + } else { + break; + } + } + } + + return lastOffset; + } + + + public boolean flush(final int flushLeastPages) { + return this.mappedFileQueue.flush(flushLeastPages); + } + + + public int deleteExpiredFile(long offset) { + int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(offset, CQ_STORE_UNIT_SIZE); + this.correctMinOffset(offset); + return cnt; + } + + public void correctMinOffset(long phyMinOffset) { + MappedFile mappedFile = this.mappedFileQueue.getFirstMappedFile(); + if (mappedFile != null) { + SelectMappedBufferResult result = mappedFile.selectMappedBuffer(0); + if (result != null) { + try { + + for (int i = 0; i < result.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + long offsetPy = result.getByteBuffer().getLong(); + result.getByteBuffer().getInt(); + result.getByteBuffer().getLong(); + + if (offsetPy >= phyMinOffset) { + this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i; + log.info("compute logics min offset: " + this.getMinOffsetInQuque() + ", topic: " + + this.topic + ", queueId: " + this.queueId); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + result.release(); + } + } + } + } + + + public long getMinOffsetInQuque() { + return this.minLogicOffset / CQ_STORE_UNIT_SIZE; + } + + + public void putMessagePositionInfoWrapper(long offset, int size, long tagsCode, long storeTimestamp, + long logicOffset) { + final int maxRetries = 30; + boolean canWrite = this.defaultMessageStore.getRunningFlags().isWriteable(); + for (int i = 0; i < maxRetries && canWrite; i++) { + boolean result = this.putMessagePositionInfo(offset, size, tagsCode, logicOffset); + if (result) { + this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimestamp); + return; + } else { + // XXX: warn and notify me + log.warn("[BUG]put commit log position info to " + topic + ":" + queueId + " " + offset + + " failed, retry " + i + " times"); + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + log.warn("", e); + } + } + } + + // XXX: warn and notify me + log.error("[BUG]consume queue can not write, {} {}", this.topic, this.queueId); + this.defaultMessageStore.getRunningFlags().makeLogicsQueueError(); + } + + private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + + if (offset <= this.maxPhysicOffset) { + return true; + } + + this.byteBufferIndex.flip(); + this.byteBufferIndex.limit(CQ_STORE_UNIT_SIZE); + this.byteBufferIndex.putLong(offset); + this.byteBufferIndex.putInt(size); + this.byteBufferIndex.putLong(tagsCode); + + final long expectLogicOffset = cqOffset * CQ_STORE_UNIT_SIZE; + + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(expectLogicOffset); + if (mappedFile != null) { + + if (mappedFile.isFirstCreateInQueue() && cqOffset != 0 && mappedFile.getWrotePosition() == 0) { + this.minLogicOffset = expectLogicOffset; + this.mappedFileQueue.setFlushedWhere(expectLogicOffset); + this.mappedFileQueue.setCommittedWhere(expectLogicOffset); + this.fillPreBlank(mappedFile, expectLogicOffset); + log.info("fill pre blank space " + mappedFile.getFileName() + " " + expectLogicOffset + " " + + mappedFile.getWrotePosition()); + } + + if (cqOffset != 0) { + long currentLogicOffset = mappedFile.getWrotePosition() + mappedFile.getFileFromOffset(); + if (expectLogicOffset != currentLogicOffset) { + LOG_ERROR.warn( + "[BUG]logic queue order maybe wrong, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}", + expectLogicOffset, + currentLogicOffset, + this.topic, + this.queueId, + expectLogicOffset - currentLogicOffset + ); + } + } + this.maxPhysicOffset = offset; + return mappedFile.appendMessage(this.byteBufferIndex.array()); + } + return false; + } + + + private void fillPreBlank(final MappedFile mappedFile, final long untilWhere) { + ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_STORE_UNIT_SIZE); + byteBuffer.putLong(0L); + byteBuffer.putInt(Integer.MAX_VALUE); + byteBuffer.putLong(0L); + + int until = (int) (untilWhere % this.mappedFileQueue.getMappedFileSize()); + for (int i = 0; i < until; i += CQ_STORE_UNIT_SIZE) { + mappedFile.appendMessage(byteBuffer.array()); + } + } + + public SelectMappedBufferResult getIndexBuffer(final long startIndex) { + int mappedFileSize = this.mappedFileSize; + long offset = startIndex * CQ_STORE_UNIT_SIZE; + if (offset >= this.getMinLogicOffset()) { + MappedFile mappedFile = this.mappedFileQueue.findMappedFileByOffset(offset); + if (mappedFile != null) { + SelectMappedBufferResult result = mappedFile.selectMappedBuffer((int) (offset % mappedFileSize)); + return result; + } + } + return null; + } + + public long getMinLogicOffset() { + return minLogicOffset; + } + + public void setMinLogicOffset(long minLogicOffset) { + this.minLogicOffset = minLogicOffset; + } + + public long rollNextFile(final long index) { + int mapedFileSize = this.mappedFileSize; + int totalUnitsInFile = mapedFileSize / CQ_STORE_UNIT_SIZE; + return index + totalUnitsInFile - index % totalUnitsInFile; + } + + public String getTopic() { + return topic; + } + + public int getQueueId() { + return queueId; + } + + public long getMaxPhysicOffset() { + return maxPhysicOffset; + } + + public void setMaxPhysicOffset(long maxPhysicOffset) { + this.maxPhysicOffset = maxPhysicOffset; + } + + public void destroy() { + this.maxPhysicOffset = -1; + this.minLogicOffset = 0; + this.mappedFileQueue.destroy(); + } + + public long getMessageTotalInQueue() { + return this.getMaxOffsetInQuque() - this.getMinOffsetInQuque(); + } + + + public long getMaxOffsetInQuque() { + return this.mappedFileQueue.getMaxOffset() / CQ_STORE_UNIT_SIZE; + } + + + public void checkSelf() { + mappedFileQueue.checkSelf(); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java new file mode 100644 index 0000000000..2086438032 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * @author shijia.wxr + */ +public class DefaultMessageFilter implements MessageFilter { + + @Override + public boolean isMessageMatched(SubscriptionData subscriptionData, Long tagsCode) { + if (tagsCode == null) { + return true; + } + + if (null == subscriptionData) { + return true; + } + + if (subscriptionData.isClassFilterMode()) + return true; + + if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) { + return true; + } + + return subscriptionData.getCodeSet().contains(tagsCode.intValue()); + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java new file mode 100644 index 0000000000..c922c8de80 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java @@ -0,0 +1,1748 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.*; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.running.RunningStats; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import com.alibaba.rocketmq.store.ha.HAService; +import com.alibaba.rocketmq.store.index.IndexService; +import com.alibaba.rocketmq.store.index.QueryOffsetResult; +import com.alibaba.rocketmq.store.schedule.ScheduleMessageService; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.rocketmq.store.config.BrokerRole.SLAVE; + + +/** + * @author shijia.wxr + */ +public class DefaultMessageStore implements MessageStore { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + private final MessageFilter messageFilter = new DefaultMessageFilter(); + + private final MessageStoreConfig messageStoreConfig; + // CommitLog + private final CommitLog commitLog; + + private final ConcurrentHashMap> consumeQueueTable; + + private final FlushConsumeQueueService flushConsumeQueueService; + + private final CleanCommitLogService cleanCommitLogService; + + private final CleanConsumeQueueService cleanConsumeQueueService; + + private final IndexService indexService; + + private final AllocateMappedFileService allocateMappedFileService; + + private final ReputMessageService reputMessageService; + + private final HAService haService; + + private final ScheduleMessageService scheduleMessageService; + + private final StoreStatsService storeStatsService; + + private final TransientStorePool transientStorePool; + + private final RunningFlags runningFlags = new RunningFlags(); + private final SystemClock systemClock = new SystemClock(); + + private final ScheduledExecutorService scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread")); + private final BrokerStatsManager brokerStatsManager; + private final MessageArrivingListener messageArrivingListener; + private final BrokerConfig brokerConfig; + + private volatile boolean shutdown = true; + + private StoreCheckpoint storeCheckpoint; + + private AtomicLong printTimes = new AtomicLong(0); + + public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException { + this.messageArrivingListener = messageArrivingListener; + this.brokerConfig = brokerConfig; + this.messageStoreConfig = messageStoreConfig; + this.brokerStatsManager = brokerStatsManager; + this.allocateMappedFileService = new AllocateMappedFileService(this); + this.commitLog = new CommitLog(this); + this.consumeQueueTable = new ConcurrentHashMap<>(32); + + this.flushConsumeQueueService = new FlushConsumeQueueService(); + this.cleanCommitLogService = new CleanCommitLogService(); + this.cleanConsumeQueueService = new CleanConsumeQueueService(); + this.storeStatsService = new StoreStatsService(); + this.indexService = new IndexService(this); + this.haService = new HAService(this); + + this.reputMessageService = new ReputMessageService(); + + this.scheduleMessageService = new ScheduleMessageService(this); + + this.transientStorePool = new TransientStorePool(messageStoreConfig); + + if (messageStoreConfig.isTransientStorePoolEnable()) { + this.transientStorePool.init(); + } + + + this.allocateMappedFileService.start(); + + this.indexService.start(); + } + + + public void truncateDirtyLogicFiles(long phyOffset) { + ConcurrentHashMap> tables = DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.truncateDirtyLogicFiles(phyOffset); + } + } + } + + + /** + * @throws IOException + */ + public boolean load() { + boolean result = true; + + try { + boolean lastExitOK = !this.isTempFileExist(); + log.info("last shutdown {}", lastExitOK ? "normally" : "abnormally"); + + if (null != scheduleMessageService) { + result = result && this.scheduleMessageService.load(); + } + + // load Commit Log + result = result && this.commitLog.load(); + + // load Consume Queue + result = result && this.loadConsumeQueue(); + + if (result) { + this.storeCheckpoint = + new StoreCheckpoint(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); + + this.indexService.load(lastExitOK); + + + this.recover(lastExitOK); + + log.info("load over, and the max phy offset = {}", this.getMaxPhyOffset()); + } + } catch (Exception e) { + log.error("load exception", e); + result = false; + } + + if (!result) { + this.allocateMappedFileService.shutdown(); + } + + return result; + } + + /** + * @throws Exception + */ + public void start() throws Exception { + this.flushConsumeQueueService.start(); + this.commitLog.start(); + this.storeStatsService.start(); + + + if (this.scheduleMessageService != null && SLAVE != messageStoreConfig.getBrokerRole()) { + this.scheduleMessageService.start(); + } + + if (this.getMessageStoreConfig().isDuplicationEnable()) { + this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset()); + } else { + this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset()); + } + this.reputMessageService.start(); + + this.haService.start(); + + this.createTempFile(); + this.addScheduleTask(); + this.shutdown = false; + } + + /** + + */ + public void shutdown() { + if (!this.shutdown) { + this.shutdown = true; + + this.scheduledExecutorService.shutdown(); + + try { + + Thread.sleep(1000 * 3); + } catch (InterruptedException e) { + log.error("shutdown Exception, ", e); + } + + if (this.scheduleMessageService != null) { + this.scheduleMessageService.shutdown(); + } + + this.haService.shutdown(); + + this.storeStatsService.shutdown(); + this.indexService.shutdown(); + this.commitLog.shutdown(); + this.reputMessageService.shutdown(); + this.flushConsumeQueueService.shutdown(); + this.allocateMappedFileService.shutdown(); + this.storeCheckpoint.flush(); + this.storeCheckpoint.shutdown(); + + if (this.runningFlags.isWriteable()) { + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); + } else { + log.warn("the store may be wrong, so shutdown abnormally, and keep abort file."); + } + } + + this.transientStorePool.destroy(); + } + + public void destroy() { + this.destroyLogics(); + this.commitLog.destroy(); + this.indexService.destroy(); + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); + this.deleteFile(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); + } + + public void destroyLogics() { + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.destroy(); + } + } + } + + public PutMessageResult putMessage(MessageExtBrokerInner msg) { + if (this.shutdown) { + log.warn("message store has shutdown, so putMessage is forbidden"); + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } + + if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { + long value = this.printTimes.getAndIncrement(); + if ((value % 50000) == 0) { + log.warn("message store is slave mode, so putMessage is forbidden "); + } + + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } + + if (!this.runningFlags.isWriteable()) { + long value = this.printTimes.getAndIncrement(); + if ((value % 50000) == 0) { + log.warn("message store is not writeable, so putMessage is forbidden " + this.runningFlags.getFlagBits()); + } + + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } else { + this.printTimes.set(0); + } + + + if (msg.getTopic().length() > Byte.MAX_VALUE) { + log.warn("putMessage message topic length too long " + msg.getTopic().length()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + + + if (msg.getPropertiesString() != null && msg.getPropertiesString().length() > Short.MAX_VALUE) { + log.warn("putMessage message properties length too long " + msg.getPropertiesString().length()); + return new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, null); + } + + + if (this.isOSPageCacheBusy()) { + return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, null); + } + + long beginTime = this.getSystemClock().now(); + PutMessageResult result = this.commitLog.putMessage(msg); + + long eclipseTime = this.getSystemClock().now() - beginTime; + if (eclipseTime > 500) { + log.warn("putMessage not in lock eclipse time(ms)={}, bodyLength={}", eclipseTime, msg.getBody().length); + } + this.storeStatsService.setPutMessageEntireTimeMax(eclipseTime); + + if (null == result || !result.isOk()) { + this.storeStatsService.getPutMessageFailedTimes().incrementAndGet(); + } + + return result; + } + + @Override + public boolean isOSPageCacheBusy() { + long begin = this.getCommitLog().getBeginTimeInLock(); + long diff = this.systemClock.now() - begin; + + if (diff < 10000000 // + && diff > this.messageStoreConfig.getOsPageCacheBusyTimeOutMills()) { + return true; + } + + return false; + } + + @Override + public long lockTimeMills() { + return this.commitLog.lockTimeMills(); + } + + public SystemClock getSystemClock() { + return systemClock; + } + + public CommitLog getCommitLog() { + return commitLog; + } + + public GetMessageResult getMessage(final String group, final String topic, final int queueId, final long offset, final int maxMsgNums, + final SubscriptionData subscriptionData) { + if (this.shutdown) { + log.warn("message store has shutdown, so getMessage is forbidden"); + return null; + } + + if (!this.runningFlags.isReadable()) { + log.warn("message store is not readable, so getMessage is forbidden " + this.runningFlags.getFlagBits()); + return null; + } + + long beginTime = this.getSystemClock().now(); + + + GetMessageStatus status = GetMessageStatus.NO_MESSAGE_IN_QUEUE; + long nextBeginOffset = offset; + long minOffset = 0; + long maxOffset = 0; + + GetMessageResult getResult = new GetMessageResult(); + + + final long maxOffsetPy = this.commitLog.getMaxOffset(); + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + minOffset = consumeQueue.getMinOffsetInQuque(); + maxOffset = consumeQueue.getMaxOffsetInQuque(); + + if (maxOffset == 0) { + status = GetMessageStatus.NO_MESSAGE_IN_QUEUE; + nextBeginOffset = nextOffsetCorrection(offset, 0); + } else if (offset < minOffset) { + status = GetMessageStatus.OFFSET_TOO_SMALL; + nextBeginOffset = nextOffsetCorrection(offset, minOffset); + } else if (offset == maxOffset) { + status = GetMessageStatus.OFFSET_OVERFLOW_ONE; + nextBeginOffset = nextOffsetCorrection(offset, offset); + } else if (offset > maxOffset) { + status = GetMessageStatus.OFFSET_OVERFLOW_BADLY; + if (0 == minOffset) { + nextBeginOffset = nextOffsetCorrection(offset, minOffset); + } else { + nextBeginOffset = nextOffsetCorrection(offset, maxOffset); + } + } else { + SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(offset); + if (bufferConsumeQueue != null) { + try { + status = GetMessageStatus.NO_MATCHED_MESSAGE; + + long nextPhyFileStartOffset = Long.MIN_VALUE; + long maxPhyOffsetPulling = 0; + + int i = 0; + final int maxFilterMessageCount = 16000; + final boolean diskFallRecorded = this.messageStoreConfig.isDiskFallRecorded(); + for (; i < bufferConsumeQueue.getSize() && i < maxFilterMessageCount; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + int sizePy = bufferConsumeQueue.getByteBuffer().getInt(); + long tagsCode = bufferConsumeQueue.getByteBuffer().getLong(); + + maxPhyOffsetPulling = offsetPy; + + + if (nextPhyFileStartOffset != Long.MIN_VALUE) { + if (offsetPy < nextPhyFileStartOffset) + continue; + } + + + boolean isInDisk = checkInDiskByCommitOffset(offsetPy, maxOffsetPy); + + if (this.isTheBatchFull(sizePy, maxMsgNums, getResult.getBufferTotalSize(), getResult.getMessageCount(), + isInDisk)) { + break; + } + + + if (this.messageFilter.isMessageMatched(subscriptionData, tagsCode)) { + SelectMappedBufferResult selectResult = this.commitLog.getMessage(offsetPy, sizePy); + if (selectResult != null) { + this.storeStatsService.getGetMessageTransferedMsgCount().incrementAndGet(); + getResult.addMessage(selectResult); + status = GetMessageStatus.FOUND; + nextPhyFileStartOffset = Long.MIN_VALUE; + } else { + if (getResult.getBufferTotalSize() == 0) { + status = GetMessageStatus.MESSAGE_WAS_REMOVING; + } + + + nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy); + } + } else { + if (getResult.getBufferTotalSize() == 0) { + status = GetMessageStatus.NO_MATCHED_MESSAGE; + } + + if (log.isDebugEnabled()) { + log.debug("message type not matched, client: " + subscriptionData + " server: " + tagsCode); + } + } + } + + + if (diskFallRecorded) { + long fallBehind = maxOffsetPy - maxPhyOffsetPulling; + brokerStatsManager.recordDiskFallBehindSize(group, topic, queueId, fallBehind); + } + + nextBeginOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + + + long diff = maxOffsetPy - maxPhyOffsetPulling; + long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE + * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0)); + getResult.setSuggestPullingFromSlave(diff > memory); + } finally { + + bufferConsumeQueue.release(); + } + } else { + status = GetMessageStatus.OFFSET_FOUND_NULL; + nextBeginOffset = nextOffsetCorrection(offset, consumeQueue.rollNextFile(offset)); + log.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: " + + maxOffset + ", but access logic queue failed."); + } + } + } else { + status = GetMessageStatus.NO_MATCHED_LOGIC_QUEUE; + nextBeginOffset = nextOffsetCorrection(offset, 0); + } + + if (GetMessageStatus.FOUND == status) { + this.storeStatsService.getGetMessageTimesTotalFound().incrementAndGet(); + } else { + this.storeStatsService.getGetMessageTimesTotalMiss().incrementAndGet(); + } + long eclipseTime = this.getSystemClock().now() - beginTime; + this.storeStatsService.setGetMessageEntireTimeMax(eclipseTime); + + getResult.setStatus(status); + getResult.setNextBeginOffset(nextBeginOffset); + getResult.setMaxOffset(maxOffset); + getResult.setMinOffset(minOffset); + return getResult; + } + + /** + + */ + public long getMaxOffsetInQuque(String topic, int queueId) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + long offset = logic.getMaxOffsetInQuque(); + return offset; + } + + return 0; + } + + /** + + */ + public long getMinOffsetInQuque(String topic, int queueId) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMinOffsetInQuque(); + } + + return -1; + } + + @Override + public long getCommitLogOffsetInQueue(String topic, int queueId, long cqOffset) { + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(cqOffset); + if (bufferConsumeQueue != null) { + try { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + return offsetPy; + } finally { + bufferConsumeQueue.release(); + } + } + } + + return 0; + } + + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getOffsetInQueueByTime(timestamp); + } + + return 0; + } + + public MessageExt lookMessageByOffset(long commitLogOffset) { + SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4); + if (null != sbr) { + try { + // 1 TOTALSIZE + int size = sbr.getByteBuffer().getInt(); + return lookMessageByOffset(commitLogOffset, size); + } finally { + sbr.release(); + } + } + + return null; + } + + @Override + public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset) { + SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4); + if (null != sbr) { + try { + // 1 TOTALSIZE + int size = sbr.getByteBuffer().getInt(); + return this.commitLog.getMessage(commitLogOffset, size); + } finally { + sbr.release(); + } + } + + return null; + } + + @Override + public SelectMappedBufferResult selectOneMessageByOffset(long commitLogOffset, int msgSize) { + return this.commitLog.getMessage(commitLogOffset, msgSize); + } + + public String getRunningDataInfo() { + return this.storeStatsService.toString(); + } + + @Override + public HashMap getRuntimeInfo() { + HashMap result = this.storeStatsService.getRuntimeInfo(); + + { + String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(physicRatio)); + + } + + + { + + String storePathLogics = StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + result.put(RunningStats.consumeQueueDiskRatio.name(), String.valueOf(logicsRatio)); + } + + + { + if (this.scheduleMessageService != null) { + this.scheduleMessageService.buildRunningStats(result); + } + } + + result.put(RunningStats.commitLogMinOffset.name(), String.valueOf(DefaultMessageStore.this.getMinPhyOffset())); + result.put(RunningStats.commitLogMaxOffset.name(), String.valueOf(DefaultMessageStore.this.getMaxPhyOffset())); + + return result; + } + + @Override + public long getMaxPhyOffset() { + return this.commitLog.getMaxOffset(); + } + + @Override + public long getMinPhyOffset() { + return this.commitLog.getMinOffset(); + } + + @Override + public long getEarliestMessageTime(String topic, int queueId) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + long minLogicOffset = logicQueue.getMinLogicOffset(); + + SelectMappedBufferResult result = logicQueue.getIndexBuffer(minLogicOffset / ConsumeQueue.CQ_STORE_UNIT_SIZE); + if (result != null) { + try { + final long phyOffset = result.getByteBuffer().getLong(); + final int size = result.getByteBuffer().getInt(); + long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); + return storeTime; + } catch (Exception e) { + } finally { + result.release(); + } + } + } + + return -1; + } + + @Override + public long getEarliestMessageTime() { + final long minPhyOffset = this.getMinPhyOffset(); + final int size = this.messageStoreConfig.getMaxMessageSize() * 2; + return this.getCommitLog().pickupStoreTimestamp(minPhyOffset, size); + } + + @Override + public long getMessageStoreTimeStamp(String topic, int queueId, long offset) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + SelectMappedBufferResult result = logicQueue.getIndexBuffer(offset); + if (result != null) { + try { + final long phyOffset = result.getByteBuffer().getLong(); + final int size = result.getByteBuffer().getInt(); + long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); + return storeTime; + } catch (Exception e) { + } finally { + result.release(); + } + } + } + + return -1; + } + + @Override + public long getMessageTotalInQueue(String topic, int queueId) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + return logicQueue.getMessageTotalInQueue(); + } + + return -1; + } + + @Override + public SelectMappedBufferResult getCommitLogData(final long offset) { + if (this.shutdown) { + log.warn("message store has shutdown, so getPhyQueueData is forbidden"); + return null; + } + + return this.commitLog.getData(offset); + } + + @Override + public boolean appendToCommitLog(long startOffset, byte[] data) { + if (this.shutdown) { + log.warn("message store has shutdown, so appendToPhyQueue is forbidden"); + return false; + } + + boolean result = this.commitLog.appendData(startOffset, data); + if (result) { + this.reputMessageService.wakeup(); + } else { + log.error("appendToPhyQueue failed " + startOffset + " " + data.length); + } + + return result; + } + + @Override + public void excuteDeleteFilesManualy() { + this.cleanCommitLogService.excuteDeleteFilesManualy(); + } + + @Override + public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, long end) { + QueryMessageResult queryMessageResult = new QueryMessageResult(); + + long lastQueryMsgTime = end; + + for (int i = 0; i < 3; i++) { + QueryOffsetResult queryOffsetResult = this.indexService.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime); + if (queryOffsetResult.getPhyOffsets().isEmpty()) { + break; + } + + + Collections.sort(queryOffsetResult.getPhyOffsets()); + + queryMessageResult.setIndexLastUpdatePhyoffset(queryOffsetResult.getIndexLastUpdatePhyoffset()); + queryMessageResult.setIndexLastUpdateTimestamp(queryOffsetResult.getIndexLastUpdateTimestamp()); + + for (int m = 0; m < queryOffsetResult.getPhyOffsets().size(); m++) { + long offset = queryOffsetResult.getPhyOffsets().get(m); + + try { + + boolean match = true; + MessageExt msg = this.lookMessageByOffset(offset); + if (0 == m) { + lastQueryMsgTime = msg.getStoreTimestamp(); + } + +// String[] keyArray = msg.getKeys().split(MessageConst.KEY_SEPARATOR); +// if (topic.equals(msg.getTopic())) { +// for (String k : keyArray) { +// if (k.equals(key)) { +// match = true; +// break; +// } +// } +// } + + if (match) { + SelectMappedBufferResult result = this.commitLog.getData(offset, false); + if (result != null) { + int size = result.getByteBuffer().getInt(0); + result.getByteBuffer().limit(size); + result.setSize(size); + queryMessageResult.addMessage(result); + } + } else { + log.warn("queryMessage hash duplicate, {} {}", topic, key); + } + } catch (Exception e) { + log.error("queryMessage exception", e); + } + } + + + if (queryMessageResult.getBufferTotalSize() > 0) { + break; + } + + + if (lastQueryMsgTime < begin) { + break; + } + } + + return queryMessageResult; + } + + @Override + public void updateHaMasterAddress(String newAddr) { + this.haService.updateMasterAddress(newAddr); + } + + @Override + public long slaveFallBehindMuch() { + return this.commitLog.getMaxOffset() - this.haService.getPush2SlaveMaxOffset().get(); + } + + @Override + public long now() { + return this.systemClock.now(); + } + + @Override + public int cleanUnusedTopic(Set topics) { + Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topic = next.getKey(); + + if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + ConcurrentHashMap queueTable = next.getValue(); + for (ConsumeQueue cq : queueTable.values()) { + cq.destroy(); + log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", // + cq.getTopic(), // + cq.getQueueId() // + ); + + this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId()); + } + it.remove(); + + log.info("cleanUnusedTopic: {},topic destroyed", topic); + } + } + + return 0; + } + + public void cleanExpiredConsumerQueue() { + long minCommitLogOffset = this.commitLog.getMinOffset(); + + Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topic = next.getKey(); + if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + ConcurrentHashMap queueTable = next.getValue(); + Iterator> itQT = queueTable.entrySet().iterator(); + while (itQT.hasNext()) { + Entry nextQT = itQT.next(); + long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset(); + + + if (maxCLOffsetInConsumeQueue == -1) { + log.warn("maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.", // + nextQT.getValue().getTopic(), // + nextQT.getValue().getQueueId(), // + nextQT.getValue().getMaxPhysicOffset(), // + nextQT.getValue().getMinLogicOffset()); + } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) { + log.info( + "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}", // + topic, // + nextQT.getKey(), // + minCommitLogOffset, // + maxCLOffsetInConsumeQueue); + + DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(), + nextQT.getValue().getQueueId()); + + nextQT.getValue().destroy(); + itQT.remove(); + } + } + + if (queueTable.isEmpty()) { + log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic); + it.remove(); + } + } + } + } + + public Map getMessageIds(final String topic, final int queueId, long minOffset, long maxOffset, SocketAddress storeHost) { + Map messageIds = new HashMap(); + if (this.shutdown) { + return messageIds; + } + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + minOffset = Math.max(minOffset, consumeQueue.getMinOffsetInQuque()); + maxOffset = Math.min(maxOffset, consumeQueue.getMaxOffsetInQuque()); + + if (maxOffset == 0) { + return messageIds; + } + + long nextOffset = minOffset; + while (nextOffset < maxOffset) { + SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(nextOffset); + if (bufferConsumeQueue != null) { + try { + int i = 0; + for (; i < bufferConsumeQueue.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + final ByteBuffer msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + String msgId = + MessageDecoder.createMessageId(msgIdMemory, MessageExt.socketAddress2ByteBuffer(storeHost), offsetPy); + messageIds.put(msgId, nextOffset++); + if (nextOffset > maxOffset) { + return messageIds; + } + } + } finally { + + bufferConsumeQueue.release(); + } + } else { + return messageIds; + } + } + } + return messageIds; + } + + @Override + public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset) { + + final long maxOffsetPy = this.commitLog.getMaxOffset(); + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(consumeOffset); + if (bufferConsumeQueue != null) { + try { + for (int i = 0; i < bufferConsumeQueue.getSize(); ) { + i += ConsumeQueue.CQ_STORE_UNIT_SIZE; + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + return checkInDiskByCommitOffset(offsetPy, maxOffsetPy); + } + } finally { + + bufferConsumeQueue.release(); + } + } else { + return false; + } + } + return false; + } + + public long dispatchBehindBytes() { + return this.reputMessageService.behind(); + } + + @Override + public long flush() { + return this.commitLog.flush(); + } + + @Override + public boolean resetWriteOffset(long phyOffset) { + return this.commitLog.resetOffset(phyOffset); + } + + @Override + public long getConfirmOffset() { + return this.commitLog.getConfirmOffset(); + } + + @Override + public void setConfirmOffset(long phyOffset) { + this.commitLog.setConfirmOffset(phyOffset); + } + + public MessageExt lookMessageByOffset(long commitLogOffset, int size) { + SelectMappedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, size); + if (null != sbr) { + try { + return MessageDecoder.decode(sbr.getByteBuffer(), true, false); + } finally { + sbr.release(); + } + } + + return null; + } + + public ConsumeQueue findConsumeQueue(String topic, int queueId) { + ConcurrentHashMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentHashMap newMap = new ConcurrentHashMap(128); + ConcurrentHashMap oldMap = consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } else { + map = newMap; + } + } + + ConsumeQueue logic = map.get(queueId); + if (null == logic) { + ConsumeQueue newLogic = new ConsumeQueue(// + topic, // + queueId, // + StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), // + this.getMessageStoreConfig().getMapedFileSizeConsumeQueue(), // + this); + ConsumeQueue oldLogic = map.putIfAbsent(queueId, newLogic); + if (oldLogic != null) { + logic = oldLogic; + } else { + logic = newLogic; + } + } + + return logic; + } + + private long nextOffsetCorrection(long oldOffset, long newOffset) { + long nextOffset = oldOffset; + if (this.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE || this.getMessageStoreConfig().isOffsetCheckInSlave()) { + nextOffset = newOffset; + } + return nextOffset; + } + + private boolean checkInDiskByCommitOffset(long offsetPy, long maxOffsetPy) { + long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryMaxRatio() / 100.0)); + return (maxOffsetPy - offsetPy) > memory; + } + + private boolean isTheBatchFull(int sizePy, int maxMsgNums, int bufferTotal, int messageTotal, boolean isInDisk) { + + if (0 == bufferTotal || 0 == messageTotal) { + return false; + } + + if ((messageTotal + 1) >= maxMsgNums) { + return true; + } + + + if (isInDisk) { + if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInDisk()) { + return true; + } + + if ((messageTotal + 1) > this.messageStoreConfig.getMaxTransferCountOnMessageInDisk()) { + return true; + } + } else { + if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInMemory()) { + return true; + } + + if ((messageTotal + 1) > this.messageStoreConfig.getMaxTransferCountOnMessageInMemory()) { + return true; + } + } + + return false; + } + + private void deleteFile(final String fileName) { + File file = new File(fileName); + boolean result = file.delete(); + log.info(fileName + (result ? " delete OK" : " delete Failed")); + } + + /** + * @throws IOException + */ + private void createTempFile() throws IOException { + String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()); + File file = new File(fileName); + MappedFile.ensureDirOK(file.getParent()); + boolean result = file.createNewFile(); + log.info(fileName + (result ? " create OK" : " already exists")); + } + + private void addScheduleTask() { + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + DefaultMessageStore.this.cleanFilesPeriodically(); + } + }, 1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + DefaultMessageStore.this.checkSelf(); + } + }, 1, 10, TimeUnit.MINUTES); + + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + if (DefaultMessageStore.this.getMessageStoreConfig().isDebugLockEnable()) { + try { + if (DefaultMessageStore.this.commitLog.getBeginTimeInLock() != 0) { + long lockTime = System.currentTimeMillis() - DefaultMessageStore.this.commitLog.getBeginTimeInLock(); + if (lockTime > 1000 && lockTime < 10000000) { + + String stack = UtilAll.jstack(); + final String fileName = System.getProperty("user.home") + File.separator + "debug/lock/stack-" + + DefaultMessageStore.this.commitLog.getBeginTimeInLock() + "-" + lockTime; + MixAll.string2FileNotSafe(stack, fileName); + } + } + } catch (Exception e) { + } + } + } + }, 1, 1, TimeUnit.SECONDS); + + // this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + // @Override + // public void run() { + // DefaultMessageStore.this.cleanExpiredConsumerQueue(); + // } + // }, 1, 1, TimeUnit.HOURS); + } + + private void cleanFilesPeriodically() { + this.cleanCommitLogService.run(); + this.cleanConsumeQueueService.run(); + } + + private void checkSelf() { + this.commitLog.checkSelf(); + + Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + Iterator> itNext = next.getValue().entrySet().iterator(); + while (itNext.hasNext()) { + Entry cq = itNext.next(); + cq.getValue().checkSelf(); + } + } + } + + private boolean isTempFileExist() { + String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()); + File file = new File(fileName); + return file.exists(); + } + + private boolean loadConsumeQueue() { + File dirLogic = new File(StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir())); + File[] fileTopicList = dirLogic.listFiles(); + if (fileTopicList != null) { + + for (File fileTopic : fileTopicList) { + String topic = fileTopic.getName(); + + File[] fileQueueIdList = fileTopic.listFiles(); + if (fileQueueIdList != null) { + for (File fileQueueId : fileQueueIdList) { + int queueId; + try { + queueId = Integer.parseInt(fileQueueId.getName()); + } catch (NumberFormatException e) { + continue; + } + ConsumeQueue logic = new ConsumeQueue( + topic, + queueId, + StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), + this.getMessageStoreConfig().getMapedFileSizeConsumeQueue(), + this); + this.putConsumeQueue(topic, queueId, logic); + if (!logic.load()) { + return false; + } + } + } + } + } + + log.info("load logics queue all over, OK"); + + return true; + } + + private void recover(final boolean lastExitOK) { + this.recoverConsumeQueue(); + + + if (lastExitOK) { + this.commitLog.recoverNormally(); + } else { + this.commitLog.recoverAbnormally(); + } + + this.recoverTopicQueueTable(); + } + + public MessageStoreConfig getMessageStoreConfig() { + return messageStoreConfig; + } + + public TransientStorePool getTransientStorePool() { + return transientStorePool; + } + + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueue consumeQueue) { + ConcurrentHashMap map = this.consumeQueueTable.get(topic); + if (null == map) { + map = new ConcurrentHashMap(); + map.put(queueId, consumeQueue); + this.consumeQueueTable.put(topic, map); + } else { + map.put(queueId, consumeQueue); + } + } + + private void recoverConsumeQueue() { + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.recover(); + } + } + } + + private void recoverTopicQueueTable() { + HashMap table = new HashMap(1024); + long minPhyOffset = this.commitLog.getMinOffset(); + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + String key = logic.getTopic() + "-" + logic.getQueueId(); + table.put(key, logic.getMaxOffsetInQuque()); + logic.correctMinOffset(minPhyOffset); + } + } + + this.commitLog.setTopicQueueTable(table); + } + + public AllocateMappedFileService getAllocateMappedFileService() { + return allocateMappedFileService; + } + + public StoreStatsService getStoreStatsService() { + return storeStatsService; + } + + public RunningFlags getAccessRights() { + return runningFlags; + } + + public ConcurrentHashMap> getConsumeQueueTable() { + return consumeQueueTable; + } + + public StoreCheckpoint getStoreCheckpoint() { + return storeCheckpoint; + } + + public HAService getHaService() { + return haService; + } + + public ScheduleMessageService getScheduleMessageService() { + return scheduleMessageService; + } + + public RunningFlags getRunningFlags() { + return runningFlags; + } + + public void doDispatch(DispatchRequest req) { + final int tranType = MessageSysFlag.getTransactionValue(req.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + DefaultMessageStore.this.putMessagePositionInfo(req.getTopic(), req.getQueueId(), req.getCommitLogOffset(), req.getMsgSize(), + req.getTagsCode(), req.getStoreTimestamp(), req.getConsumeQueueOffset()); + break; + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + } + + if (DefaultMessageStore.this.getMessageStoreConfig().isMessageIndexEnable()) { + DefaultMessageStore.this.indexService.buildIndex(req); + } + } + + public void putMessagePositionInfo(String topic, int queueId, long offset, int size, long tagsCode, long storeTimestamp, + long logicOffset) { + ConsumeQueue cq = this.findConsumeQueue(topic, queueId); + cq.putMessagePositionInfoWrapper(offset, size, tagsCode, storeTimestamp, logicOffset); + } + + public BrokerStatsManager getBrokerStatsManager() { + return brokerStatsManager; + } + + class CleanCommitLogService { + + private final static int MAX_MANUAL_DELETE_FILE_TIMES = 20; + private final double diskSpaceWarningLevelRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); + + private final double diskSpaceCleanForciblyRatio = + Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); + private long lastRedeleteTimestamp = 0; + + private volatile int manualDeleteFileSeveralTimes = 0; + + private volatile boolean cleanImmediately = false; + + + public void excuteDeleteFilesManualy() { + this.manualDeleteFileSeveralTimes = MAX_MANUAL_DELETE_FILE_TIMES; + DefaultMessageStore.log.info("excuteDeleteFilesManualy was invoked"); + } + + + public void run() { + try { + this.deleteExpiredFiles(); + + this.redeleteHangedFile(); + } catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + private void deleteExpiredFiles() { + int deleteCount = 0; + long fileReservedTime = DefaultMessageStore.this.getMessageStoreConfig().getFileReservedTime(); + int deletePhysicFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteCommitLogFilesInterval(); + int destroyMapedFileIntervalForcibly = DefaultMessageStore.this.getMessageStoreConfig().getDestroyMapedFileIntervalForcibly(); + + boolean timeup = this.isTimeToDelete(); + boolean spacefull = this.isSpaceToDelete(); + boolean manualDelete = this.manualDeleteFileSeveralTimes > 0; + + + if (timeup || spacefull || manualDelete) { + + if (manualDelete) + this.manualDeleteFileSeveralTimes--; + + + boolean cleanAtOnce = DefaultMessageStore.this.getMessageStoreConfig().isCleanFileForciblyEnable() && this.cleanImmediately; + + log.info("begin to delete before {} hours file. timeup: {} spacefull: {} manualDeleteFileSeveralTimes: {} cleanAtOnce: {}", // + fileReservedTime, // + timeup, // + spacefull, // + manualDeleteFileSeveralTimes, // + cleanAtOnce); + + + fileReservedTime *= 60 * 60 * 1000; + + deleteCount = DefaultMessageStore.this.commitLog.deleteExpiredFile(fileReservedTime, deletePhysicFilesInterval, + destroyMapedFileIntervalForcibly, cleanAtOnce); + if (deleteCount > 0) { + } else if (spacefull) { + log.warn("disk space will be full soon, but delete file failed."); + } + } + } + + private void redeleteHangedFile() { + int interval = DefaultMessageStore.this.getMessageStoreConfig().getRedeleteHangedFileInterval(); + long currentTimestamp = System.currentTimeMillis(); + if ((currentTimestamp - this.lastRedeleteTimestamp) > interval) { + this.lastRedeleteTimestamp = currentTimestamp; + int destroyMapedFileIntervalForcibly = + DefaultMessageStore.this.getMessageStoreConfig().getDestroyMapedFileIntervalForcibly(); + if (DefaultMessageStore.this.commitLog.retryDeleteFirstFile(destroyMapedFileIntervalForcibly)) { + } + } + } + + public String getServiceName() { + return CleanCommitLogService.class.getSimpleName(); + } + + private boolean isTimeToDelete() { + String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); + if (UtilAll.isItTimeToDo(when)) { + DefaultMessageStore.log.info("it's time to reclaim disk space, " + when); + return true; + } + + return false; + } + + private boolean isSpaceToDelete() { + double ratio = DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; + + cleanImmediately = false; + + + { + String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + if (physicRatio > diskSpaceWarningLevelRatio) { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull(); + if (diskok) { + DefaultMessageStore.log.error("physic disk maybe full soon " + physicRatio + ", so mark disk full"); + } + + cleanImmediately = true; + } else if (physicRatio > diskSpaceCleanForciblyRatio) { + cleanImmediately = true; + } else { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK(); + if (!diskok) { + DefaultMessageStore.log.info("physic disk space OK " + physicRatio + ", so mark disk ok"); + } + } + + if (physicRatio < 0 || physicRatio > ratio) { + DefaultMessageStore.log.info("physic disk maybe full soon, so reclaim space, " + physicRatio); + return true; + } + } + + + { + String storePathLogics = StorePathConfigHelper + .getStorePathConsumeQueue(DefaultMessageStore.this.getMessageStoreConfig().getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + if (logicsRatio > diskSpaceWarningLevelRatio) { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull(); + if (diskok) { + DefaultMessageStore.log.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full"); + } + + cleanImmediately = true; + } else if (logicsRatio > diskSpaceCleanForciblyRatio) { + cleanImmediately = true; + } else { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK(); + if (!diskok) { + DefaultMessageStore.log.info("logics disk space OK " + logicsRatio + ", so mark disk ok"); + } + } + + if (logicsRatio < 0 || logicsRatio > ratio) { + DefaultMessageStore.log.info("logics disk maybe full soon, so reclaim space, " + logicsRatio); + return true; + } + } + + return false; + } + + public int getManualDeleteFileSeveralTimes() { + return manualDeleteFileSeveralTimes; + } + + public void setManualDeleteFileSeveralTimes(int manualDeleteFileSeveralTimes) { + this.manualDeleteFileSeveralTimes = manualDeleteFileSeveralTimes; + } + } + + class CleanConsumeQueueService { + private long lastPhysicalMinOffset = 0; + + public void run() { + try { + this.deleteExpiredFiles(); + } catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + private void deleteExpiredFiles() { + int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); + + long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); + if (minOffset > this.lastPhysicalMinOffset) { + this.lastPhysicalMinOffset = minOffset; + + + ConcurrentHashMap> tables = DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue logic : maps.values()) { + int deleteCount = logic.deleteExpiredFile(minOffset); + + if (deleteCount > 0 && deleteLogicsFilesInterval > 0) { + try { + Thread.sleep(deleteLogicsFilesInterval); + } catch (InterruptedException e) { + } + } + } + } + + + DefaultMessageStore.this.indexService.deleteExpiredFile(minOffset); + } + } + + public String getServiceName() { + return CleanConsumeQueueService.class.getSimpleName(); + } + } + + class FlushConsumeQueueService extends ServiceThread { + private static final int RETRY_TIMES_OVER = 3; + private long lastFlushTimestamp = 0; + + + private void doFlush(int retryTimes) { + int flushConsumeQueueLeastPages = DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueLeastPages(); + + if (retryTimes == RETRY_TIMES_OVER) { + flushConsumeQueueLeastPages = 0; + } + + long logicsMsgTimestamp = 0; + + + int flushConsumeQueueThoroughInterval = DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueThoroughInterval(); + long currentTimeMillis = System.currentTimeMillis(); + if (currentTimeMillis >= (this.lastFlushTimestamp + flushConsumeQueueThoroughInterval)) { + this.lastFlushTimestamp = currentTimeMillis; + flushConsumeQueueLeastPages = 0; + logicsMsgTimestamp = DefaultMessageStore.this.getStoreCheckpoint().getLogicsMsgTimestamp(); + } + + ConcurrentHashMap> tables = DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue cq : maps.values()) { + boolean result = false; + for (int i = 0; i < retryTimes && !result; i++) { + result = cq.flush(flushConsumeQueueLeastPages); + } + } + } + + if (0 == flushConsumeQueueLeastPages) { + if (logicsMsgTimestamp > 0) { + DefaultMessageStore.this.getStoreCheckpoint().setLogicsMsgTimestamp(logicsMsgTimestamp); + } + DefaultMessageStore.this.getStoreCheckpoint().flush(); + } + } + + + public void run() { + DefaultMessageStore.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + int interval = DefaultMessageStore.this.getMessageStoreConfig().getFlushIntervalConsumeQueue(); + this.waitForRunning(interval); + this.doFlush(1); + } catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + + this.doFlush(RETRY_TIMES_OVER); + + DefaultMessageStore.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return FlushConsumeQueueService.class.getSimpleName(); + } + + + @Override + public long getJointime() { + return 1000 * 60; + } + } + + class ReputMessageService extends ServiceThread { + + private volatile long reputFromOffset = 0; + + public long getReputFromOffset() { + return reputFromOffset; + } + + @Override + public void shutdown() { + for (int i = 0; i < 50 && this.isCommitLogAvailable(); i++) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + } + } + + if (this.isCommitLogAvailable()) { + log.warn("shutdown ReputMessageService, but commitlog have not finish to be dispatched, CL: {} reputFromOffset: {}", + DefaultMessageStore.this.commitLog.getMaxOffset(), this.reputFromOffset); + } + + super.shutdown(); + } + + public void setReputFromOffset(long reputFromOffset) { + this.reputFromOffset = reputFromOffset; + } + + public long behind() { + return DefaultMessageStore.this.commitLog.getMaxOffset() - this.reputFromOffset; + } + + + private boolean isCommitLogAvailable() { + return this.reputFromOffset < DefaultMessageStore.this.commitLog.getMaxOffset(); + } + + + private void doReput() { + for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { + + if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() // + && this.reputFromOffset >= DefaultMessageStore.this.getConfirmOffset()) { + break; + } + + SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset); + if (result != null) { + try { + this.reputFromOffset = result.getStartOffset(); + + for (int readSize = 0; readSize < result.getSize() && doNext; ) { + DispatchRequest dispatchRequest = + DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false); + int size = dispatchRequest.getMsgSize(); + + if (dispatchRequest.isSuccess()) { + if (size > 0) { + DefaultMessageStore.this.doDispatch(dispatchRequest); + + if (BrokerRole.SLAVE != DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() + && DefaultMessageStore.this.brokerConfig.isLongPollingEnable()) { + DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), + dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, + dispatchRequest.getTagsCode()); + } + // FIXED BUG By shijia + this.reputFromOffset += size; + readSize += size; + if (DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { + DefaultMessageStore.this.storeStatsService + .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).incrementAndGet(); + DefaultMessageStore.this.storeStatsService + .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) + .addAndGet(dispatchRequest.getMsgSize()); + } + } else if (size == 0) { + this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset); + readSize = result.getSize(); + } + } else if (!dispatchRequest.isSuccess()) { + + + if (size > 0) { + log.error("[BUG]read total count not equals msg total size. reputFromOffset={}", reputFromOffset); + this.reputFromOffset += size; + } else { + doNext = false; + if (DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) { + log.error("[BUG]the master dispatch message to consume queue error, COMMITLOG OFFSET: {}", + this.reputFromOffset); + + this.reputFromOffset += result.getSize() - readSize; + } + } + } + } + } finally { + result.release(); + } + } else { + doNext = false; + } + } + } + + + @Override + public void run() { + DefaultMessageStore.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + Thread.sleep(1); + this.doReput(); + } catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + DefaultMessageStore.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return ReputMessageService.class.getSimpleName(); + } + + + } + + public int remainTransientStoreBufferNumbs() { + return this.transientStorePool.remainBufferNumbs(); + } + + @Override + public boolean isTransientStorePoolDeficient() { + return remainTransientStoreBufferNumbs() == 0; + } + + + public void unlockMappedFile(final MappedFile mappedFile) { + this.scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + mappedFile.munlock(); + } + }, 6, TimeUnit.SECONDS); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java new file mode 100644 index 0000000000..4f2c1a9aa5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * @author shijia.wxr + */ +public class DispatchRequest { + private final String topic; + private final int queueId; + private final long commitLogOffset; + private final int msgSize; + private final long tagsCode; + private final long storeTimestamp; + private final long consumeQueueOffset; + private final String keys; + private final boolean success; + private final String uniqKey; + + private final int sysFlag; + private final long preparedTransactionOffset; + + + public DispatchRequest( + final String topic, + final int queueId, + final long commitLogOffset, + final int msgSize, + final long tagsCode, + final long storeTimestamp, + final long consumeQueueOffset, + final String keys, + final String uniqKey, + final int sysFlag, + final long preparedTransactionOffset + ) { + this.topic = topic; + this.queueId = queueId; + this.commitLogOffset = commitLogOffset; + this.msgSize = msgSize; + this.tagsCode = tagsCode; + this.storeTimestamp = storeTimestamp; + this.consumeQueueOffset = consumeQueueOffset; + this.keys = keys; + this.uniqKey = uniqKey; + + this.sysFlag = sysFlag; + this.preparedTransactionOffset = preparedTransactionOffset; + this.success = true; + } + + public DispatchRequest(int size) { + // 1 + this.topic = ""; + // 2 + this.queueId = 0; + // 3 + this.commitLogOffset = 0; + // 4 + this.msgSize = size; + // 5 + this.tagsCode = 0; + // 6 + this.storeTimestamp = 0; + // 7 + this.consumeQueueOffset = 0; + // 8 + this.keys = ""; + //9 + this.uniqKey = null; + this.sysFlag = 0; + this.preparedTransactionOffset = 0; + this.success = false; + } + + public DispatchRequest(int size, boolean success) { + // 1 + this.topic = ""; + // 2 + this.queueId = 0; + // 3 + this.commitLogOffset = 0; + // 4 + this.msgSize = size; + // 5 + this.tagsCode = 0; + // 6 + this.storeTimestamp = 0; + // 7 + this.consumeQueueOffset = 0; + // 8 + this.keys = ""; + // 9 + this.uniqKey = null; + this.sysFlag = 0; + this.preparedTransactionOffset = 0; + this.success = success; + } + + + public String getTopic() { + return topic; + } + + + public int getQueueId() { + return queueId; + } + + + public long getCommitLogOffset() { + return commitLogOffset; + } + + + public int getMsgSize() { + return msgSize; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public long getConsumeQueueOffset() { + return consumeQueueOffset; + } + + + public String getKeys() { + return keys; + } + + + public long getTagsCode() { + return tagsCode; + } + + + public int getSysFlag() { + return sysFlag; + } + + + public long getPreparedTransactionOffset() { + return preparedTransactionOffset; + } + + + public boolean isSuccess() { + return success; + } + + public String getUniqKey() { + return uniqKey; + } + + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java new file mode 100644 index 0000000000..05a0003d28 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class GetMessageResult { + + private final List messageMapedList = + new ArrayList(100); + + private final List messageBufferList = new ArrayList(100); + + private GetMessageStatus status; + private long nextBeginOffset; + private long minOffset; + private long maxOffset; + + private int bufferTotalSize = 0; + + private boolean suggestPullingFromSlave = false; + + private int msgCount4Commercial = 0; + + + public GetMessageResult() { + } + + + public GetMessageStatus getStatus() { + return status; + } + + + public void setStatus(GetMessageStatus status) { + this.status = status; + } + + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + + public void setNextBeginOffset(long nextBeginOffset) { + this.nextBeginOffset = nextBeginOffset; + } + + + public long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(long minOffset) { + this.minOffset = minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + + public List getMessageMapedList() { + return messageMapedList; + } + + + public List getMessageBufferList() { + return messageBufferList; + } + + + public void addMessage(final SelectMappedBufferResult mapedBuffer) { + this.messageMapedList.add(mapedBuffer); + this.messageBufferList.add(mapedBuffer.getByteBuffer()); + this.bufferTotalSize += mapedBuffer.getSize(); + this.msgCount4Commercial += (int) Math.ceil( + mapedBuffer.getSize() / BrokerStatsManager.SIZE_PER_COUNT); + } + + + public void release() { + for (SelectMappedBufferResult select : this.messageMapedList) { + select.release(); + } + } + + + public int getBufferTotalSize() { + return bufferTotalSize; + } + + + public void setBufferTotalSize(int bufferTotalSize) { + this.bufferTotalSize = bufferTotalSize; + } + + + public int getMessageCount() { + return this.messageMapedList.size(); + } + + + public boolean isSuggestPullingFromSlave() { + return suggestPullingFromSlave; + } + + + public void setSuggestPullingFromSlave(boolean suggestPullingFromSlave) { + this.suggestPullingFromSlave = suggestPullingFromSlave; + } + + public int getMsgCount4Commercial() { + return msgCount4Commercial; + } + + public void setMsgCount4Commercial(int msgCount4Commercial) { + this.msgCount4Commercial = msgCount4Commercial; + } + + + @Override + public String toString() { + return "GetMessageResult [status=" + status + ", nextBeginOffset=" + nextBeginOffset + ", minOffset=" + + minOffset + ", maxOffset=" + maxOffset + ", bufferTotalSize=" + bufferTotalSize + + ", suggestPullingFromSlave=" + suggestPullingFromSlave + "]"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java new file mode 100644 index 0000000000..87d6fe0f54 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * @author shijia.wxr + */ +public enum GetMessageStatus { + + FOUND, + + NO_MATCHED_MESSAGE, + + MESSAGE_WAS_REMOVING, + + OFFSET_FOUND_NULL, + + OFFSET_OVERFLOW_BADLY, + + OFFSET_OVERFLOW_ONE, + + OFFSET_TOO_SMALL, + + NO_MATCHED_LOGIC_QUEUE, + + NO_MESSAGE_IN_QUEUE, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFile.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFile.java new file mode 100644 index 0000000000..ed2afa905f --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFile.java @@ -0,0 +1,591 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.config.FlushDiskType; +import com.alibaba.rocketmq.store.util.LibC; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.nio.ch.DirectBuffer; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public class MappedFile extends ReferenceResource { + public static final int OS_PAGE_SIZE = 1024 * 4; + protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + private static final AtomicLong TOTAL_MAPED_VITUAL_MEMORY = new AtomicLong(0); + + private static final AtomicInteger TOTAL_MAPED_FILES = new AtomicInteger(0); + + private String fileName; + + private long fileFromOffset; + + protected int fileSize; + + private File file; + + private MappedByteBuffer mappedByteBuffer; + + protected final AtomicInteger wrotePosition = new AtomicInteger(0); + + private final AtomicInteger flushedPosition = new AtomicInteger(0); + //ADD BY ChenYang + protected final AtomicInteger committedPosition = new AtomicInteger(0); + + + protected FileChannel fileChannel; + + private volatile long storeTimestamp = 0; + private boolean firstCreateInQueue = false; + + /** + * Message will put to here first, and then reput to FileChannel if writeBuffer is not null. + */ + protected ByteBuffer writeBuffer = null; + protected TransientStorePool transientStorePool = null; + + public MappedFile() { + } + + public MappedFile(final String fileName, final int fileSize) throws IOException { + init(fileName, fileSize); + } + + public MappedFile(final String fileName, final int fileSize, final TransientStorePool transientStorePool) throws IOException { + init(fileName, fileSize, transientStorePool); + } + + public void init(final String fileName, final int fileSize, final TransientStorePool transientStorePool) throws IOException { + init(fileName, fileSize); + this.writeBuffer = transientStorePool.borrowBuffer(); + this.transientStorePool = transientStorePool; + } + + private void init(final String fileName, final int fileSize) throws IOException { + this.fileName = fileName; + this.fileSize = fileSize; + this.file = new File(fileName); + this.fileFromOffset = Long.parseLong(this.file.getName()); + boolean ok = false; + + ensureDirOK(this.file.getParent()); + + try { + this.fileChannel = new RandomAccessFile(this.file, "rw").getChannel(); + this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize); + TOTAL_MAPED_VITUAL_MEMORY.addAndGet(fileSize); + TOTAL_MAPED_FILES.incrementAndGet(); + ok = true; + } catch (FileNotFoundException e) { + log.error("create file channel " + this.fileName + " Failed. ", e); + throw e; + } catch (IOException e) { + log.error("map file " + this.fileName + " Failed. ", e); + throw e; + } finally { + if (!ok && this.fileChannel != null) { + this.fileChannel.close(); + } + } + } + + + public static void ensureDirOK(final String dirName) { + if (dirName != null) { + File f = new File(dirName); + if (!f.exists()) { + boolean result = f.mkdirs(); + log.info(dirName + " mkdir " + (result ? "OK" : "Failed")); + } + } + } + + + public static void clean(final ByteBuffer buffer) { + if (buffer == null || !buffer.isDirect() || buffer.capacity() == 0) + return; + invoke(invoke(viewed(buffer), "cleaner"), "clean"); + } + + + private static Object invoke(final Object target, final String methodName, final Class... args) { + return AccessController.doPrivileged(new PrivilegedAction() { + public Object run() { + try { + Method method = method(target, methodName, args); + method.setAccessible(true); + return method.invoke(target); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + }); + } + + + private static Method method(Object target, String methodName, Class[] args) + throws NoSuchMethodException { + try { + return target.getClass().getMethod(methodName, args); + } catch (NoSuchMethodException e) { + return target.getClass().getDeclaredMethod(methodName, args); + } + } + + + private static ByteBuffer viewed(ByteBuffer buffer) { + String methodName = "viewedBuffer"; + + + Method[] methods = buffer.getClass().getMethods(); + for (int i = 0; i < methods.length; i++) { + if (methods[i].getName().equals("attachment")) { + methodName = "attachment"; + break; + } + } + + ByteBuffer viewedBuffer = (ByteBuffer) invoke(buffer, methodName); + if (viewedBuffer == null) + return buffer; + else + return viewed(viewedBuffer); + } + + + public static int getTotalmapedfiles() { + return TOTAL_MAPED_FILES.get(); + } + + + public static long getTotalMapedVitualMemory() { + return TOTAL_MAPED_VITUAL_MEMORY.get(); + } + + + public long getLastModifiedTimestamp() { + return this.file.lastModified(); + } + + public int getFileSize() { + return fileSize; + } + + public FileChannel getFileChannel() { + return fileChannel; + } + + public AppendMessageResult appendMessage(final MessageExtBrokerInner msg, final AppendMessageCallback cb) { + assert msg != null; + assert cb != null; + + int currentPos = this.wrotePosition.get(); + + + if (currentPos < this.fileSize) { + ByteBuffer byteBuffer = writeBuffer != null ? writeBuffer.slice() : this.mappedByteBuffer.slice(); + byteBuffer.position(currentPos); + AppendMessageResult result = + cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, msg); + this.wrotePosition.addAndGet(result.getWroteBytes()); + this.storeTimestamp = result.getStoreTimestamp(); + return result; + } + + + log.error("MappedFile.appendMessage return null, wrotePosition: " + currentPos + " fileSize: " + + this.fileSize); + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } + + /** + + */ + public long getFileFromOffset() { + return this.fileFromOffset; + } + + /** + + * + + */ + public boolean appendMessage(final byte[] data) { + int currentPos = this.wrotePosition.get(); + + + if ((currentPos + data.length) <= this.fileSize) { + try { + this.fileChannel.position(currentPos); + this.fileChannel.write(ByteBuffer.wrap(data)); + } catch (Throwable e) { + log.error("Error occurred when append message to mappedFile.", e); + } + this.wrotePosition.addAndGet(data.length); + return true; + } + + return false; + } + + /** + + * + * @param flushLeastPages + + * + * @return The current flushed position + */ + public int flush(final int flushLeastPages) { + if (this.isAbleToFlush(flushLeastPages)) { + if (this.hold()) { + int value = getReadPosition(); + + try { + //We only append data to fileChannel or mappedByteBuffer, never both. + if (writeBuffer != null || this.fileChannel.position() != 0) { + this.fileChannel.force(false); + } else { + this.mappedByteBuffer.force(); + } + } catch (Throwable e) { + log.error("Error occurred when force data to disk.", e); + } + + this.flushedPosition.set(value); + this.release(); + } else { + log.warn("in flush, hold failed, flush offset = " + this.flushedPosition.get()); + this.flushedPosition.set(getReadPosition()); + } + } + return this.getFlushedPosition(); + } + + public int commit(final int commitLeastPages) { + if (writeBuffer == null) { + //no need to commit data to file channel, so just regard wrotePosition as committedPosition. + return this.wrotePosition.get(); + } + if (this.isAbleToCommit(commitLeastPages)) { + if (this.hold()) { + commit0(commitLeastPages); + this.release(); + } else { + log.warn("in commit, hold failed, commit offset = " + this.committedPosition.get()); + } + } + + // All dirty data has been committed to FileChannel. + if (writeBuffer != null && this.transientStorePool != null && this.fileSize == this.committedPosition.get()) { + this.transientStorePool.returnBuffer(writeBuffer); + this.writeBuffer = null; + } + + return this.committedPosition.get(); + } + + protected void commit0(final int commitLeastPages) { + int writePos = this.wrotePosition.get(); + int lastCommittedPosition = this.committedPosition.get(); + + if (writePos - this.committedPosition.get() > 0) { + try { + ByteBuffer byteBuffer = writeBuffer.slice(); + byteBuffer.position(lastCommittedPosition); + byteBuffer.limit(writePos); + this.fileChannel.position(lastCommittedPosition); + this.fileChannel.write(byteBuffer); + this.committedPosition.set(writePos); + } catch (Throwable e) { + log.error("Error occurred when commit data to FileChannel.", e); + } + } + } + + private boolean isAbleToFlush(final int flushLeastPages) { + int flush = this.flushedPosition.get(); + int write = getReadPosition(); + + if (this.isFull()) { + return true; + } + + if (flushLeastPages > 0) { + return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= flushLeastPages; + } + + return write > flush; + } + + protected boolean isAbleToCommit(final int commitLeastPages) { + int flush = this.committedPosition.get(); + int write = this.wrotePosition.get(); + + if (this.isFull()) { + return true; + } + + if (commitLeastPages > 0) { + return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= commitLeastPages; + } + + return write > flush; + } + + public int getFlushedPosition() { + return flushedPosition.get(); + } + + + public void setFlushedPosition(int pos) { + this.flushedPosition.set(pos); + } + + + public boolean isFull() { + return this.fileSize == this.wrotePosition.get(); + } + + public SelectMappedBufferResult selectMappedBuffer(int pos, int size) { + int readPosition = getReadPosition(); + if ((pos + size) <= readPosition) { + + if (this.hold()) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(pos); + ByteBuffer byteBufferNew = byteBuffer.slice(); + byteBufferNew.limit(size); + return new SelectMappedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this); + } else { + log.warn("matched, but hold failed, request pos: " + pos + ", fileFromOffset: " + + this.fileFromOffset); + } + } else { + log.warn("selectMappedBuffer request pos invalid, request pos: " + pos + ", size: " + size + + ", fileFromOffset: " + this.fileFromOffset); + } + + + return null; + } + + /** + + */ + public SelectMappedBufferResult selectMappedBuffer(int pos) { + int readPosition = getReadPosition(); + if (pos < readPosition && pos >= 0) { + if (this.hold()) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(pos); + int size = readPosition - pos; + ByteBuffer byteBufferNew = byteBuffer.slice(); + byteBufferNew.limit(size); + return new SelectMappedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this); + } + } + + + return null; + } + + @Override + public boolean cleanup(final long currentRef) { + if (this.isAvailable()) { + log.error("this file[REF:" + currentRef + "] " + this.fileName + + " have not shutdown, stop unmaping."); + return false; + } + + if (this.isCleanupOver()) { + log.error("this file[REF:" + currentRef + "] " + this.fileName + + " have cleanup, do not do it again."); + return true; + } + + clean(this.mappedByteBuffer); + TOTAL_MAPED_VITUAL_MEMORY.addAndGet(this.fileSize * (-1)); + TOTAL_MAPED_FILES.decrementAndGet(); + log.info("unmap file[REF:" + currentRef + "] " + this.fileName + " OK"); + return true; + } + + public boolean destroy(final long intervalForcibly) { + this.shutdown(intervalForcibly); + + if (this.isCleanupOver()) { + try { + this.fileChannel.close(); + log.info("close file channel " + this.fileName + " OK"); + + long beginTime = System.currentTimeMillis(); + boolean result = this.file.delete(); + log.info("delete file[REF:" + this.getRefCount() + "] " + this.fileName + + (result ? " OK, " : " Failed, ") + "W:" + this.getWrotePosition() + " M:" + + this.getFlushedPosition() + ", " + + UtilAll.computeEclipseTimeMilliseconds(beginTime)); + } catch (Exception e) { + log.warn("close file channel " + this.fileName + " Failed. ", e); + } + + return true; + } else { + log.warn("destroy maped file[REF:" + this.getRefCount() + "] " + this.fileName + + " Failed. cleanupOver: " + this.cleanupOver); + } + + return false; + } + + public int getWrotePosition() { + return wrotePosition.get(); + } + + /** + * + * @return The max position which have valid data + */ + public int getReadPosition() { + return this.writeBuffer == null ? this.wrotePosition.get() : this.committedPosition.get(); + } + + public void setWrotePosition(int pos) { + this.wrotePosition.set(pos); + } + + public void setCommittedPosition(int pos) { + this.committedPosition.set(pos); + } + + public void warmMappedFile(FlushDiskType type, int pages) { + long beginTime = System.currentTimeMillis(); + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + int flush = 0; + long time = System.currentTimeMillis(); + for (int i = 0, j = 0; i < this.fileSize; i += MappedFile.OS_PAGE_SIZE, j++) { + byteBuffer.put(i, (byte) 0); + // force flush when flush disk type is sync + if (type == FlushDiskType.SYNC_FLUSH) { + if ((i / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE) >= pages) { + flush = i; + mappedByteBuffer.force(); + } + } + + // prevent gc + if (j % 1000 == 0) { + log.info("j={}, costTime={}", j, System.currentTimeMillis() - time); + time = System.currentTimeMillis(); + try { + Thread.sleep(0); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + // force flush when prepare load finished + if (type == FlushDiskType.SYNC_FLUSH) { + log.info("mapped file worm up done, force to disk, mappedFile={}, costTime={}", + this.getFileName(), System.currentTimeMillis() - beginTime); + mappedByteBuffer.force(); + } + log.info("mapped file worm up done. mappedFile={}, costTime={}", this.getFileName(), + System.currentTimeMillis() - beginTime); + + this.mlock(); + } + + public String getFileName() { + return fileName; + } + + public MappedByteBuffer getMappedByteBuffer() { + return mappedByteBuffer; + } + + public ByteBuffer sliceByteBuffer() { + return this.mappedByteBuffer.slice(); + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public boolean isFirstCreateInQueue() { + return firstCreateInQueue; + } + + + public void setFirstCreateInQueue(boolean firstCreateInQueue) { + this.firstCreateInQueue = firstCreateInQueue; + } + + + public void mlock() { + final long beginTime = System.currentTimeMillis(); + final long address = ((DirectBuffer) (this.mappedByteBuffer)).address(); + Pointer pointer = new Pointer(address); + { + int ret = LibC.INSTANCE.mlock(pointer, new NativeLong(this.fileSize)); + log.info("mlock {} {} {} ret = {} time consuming = {}", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime); + } + + { + int ret = LibC.INSTANCE.madvise(pointer, new NativeLong(this.fileSize), LibC.MADV_WILLNEED); + log.info("madvise {} {} {} ret = {} time consuming = {}", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime); + } + } + + public void munlock() { + final long beginTime = System.currentTimeMillis(); + final long address = ((DirectBuffer) (this.mappedByteBuffer)).address(); + Pointer pointer = new Pointer(address); + int ret = LibC.INSTANCE.munlock(pointer, new NativeLong(this.fileSize)); + log.info("munlock {} {} {} ret = {} time consuming = {}", address, this.fileName, this.fileSize, ret, System.currentTimeMillis() - beginTime); + } + + @Override + public String toString() { + return this.fileName; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFileQueue.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFileQueue.java new file mode 100644 index 0000000000..2b006c0e53 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MappedFileQueue.java @@ -0,0 +1,606 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; + + +/** + * @author shijia.wxr + */ +public class MappedFileQueue { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final Logger LOG_ERROR = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + private static final int DELETE_FILES_BATCH_MAX = 10; + + private final String storePath; + + private final int mappedFileSize; + + private final CopyOnWriteArrayList mappedFiles = new CopyOnWriteArrayList(); + + private final AllocateMappedFileService allocateMappedFileService; + + private long flushedWhere = 0; + private long committedWhere = 0; + + private volatile long storeTimestamp = 0; + + + public MappedFileQueue(final String storePath, int mappedFileSize, + AllocateMappedFileService allocateMappedFileService) { + this.storePath = storePath; + this.mappedFileSize = mappedFileSize; + this.allocateMappedFileService = allocateMappedFileService; + } + + + public void checkSelf() { + + if (!this.mappedFiles.isEmpty()) { + Iterator iterator = mappedFiles.iterator(); + MappedFile pre = null; + while (iterator.hasNext()) { + MappedFile cur = iterator.next(); + + if (pre != null) { + if (cur.getFileFromOffset() - pre.getFileFromOffset() != this.mappedFileSize) { + LOG_ERROR.error("[BUG]The mappedFile queue's data is damaged, the adjacent mappedFile's offset don't match. pre file {}, cur file {}", + pre.getFileName(), cur.getFileName()); + } + } + pre = cur; + } + } + } + + + public MappedFile getMappedFileByTime(final long timestamp) { + Object[] mfs = this.copyMappedFiles(0); + + if (null == mfs) + return null; + + for (int i = 0; i < mfs.length; i++) { + MappedFile mappedFile = (MappedFile) mfs[i]; + if (mappedFile.getLastModifiedTimestamp() >= timestamp) { + return mappedFile; + } + } + + return (MappedFile) mfs[mfs.length - 1]; + } + + + private Object[] copyMappedFiles(final int reservedMappedFiles) { + Object[] mfs; + + if (this.mappedFiles.size() <= reservedMappedFiles) { + return null; + } + + mfs = this.mappedFiles.toArray(); + return mfs; + } + + + public void truncateDirtyFiles(long offset) { + List willRemoveFiles = new ArrayList(); + + for (MappedFile file : this.mappedFiles) { + long fileTailOffset = file.getFileFromOffset() + this.mappedFileSize; + if (fileTailOffset > offset) { + if (offset >= file.getFileFromOffset()) { + file.setWrotePosition((int) (offset % this.mappedFileSize)); + file.setCommittedPosition((int) (offset % this.mappedFileSize)); + file.setFlushedPosition((int) (offset % this.mappedFileSize)); + } else { + file.destroy(1000); + willRemoveFiles.add(file); + } + } + } + + this.deleteExpiredFile(willRemoveFiles); + } + + + private void deleteExpiredFile(List files) { + + if (!files.isEmpty()) { + + Iterator iterator = files.iterator(); + while (iterator.hasNext()) { + MappedFile cur = iterator.next(); + if (!this.mappedFiles.contains(cur)) { + iterator.remove(); + log.info("This mappedFile {} is not contained by mappedFiles, so skip it.", cur.getFileName()); + } + } + + try { + if (!this.mappedFiles.removeAll(files)) { + log.error("deleteExpiredFile remove failed."); + } + } catch (Exception e) { + log.error("deleteExpiredFile has exception.", e); + } + } + } + + + public boolean load() { + File dir = new File(this.storePath); + File[] files = dir.listFiles(); + if (files != null) { + // ascending order + Arrays.sort(files); + for (File file : files) { + + if (file.length() != this.mappedFileSize) { + log.warn(file + "\t" + file.length() + + " length not matched message store config value, ignore it"); + return true; + } + + + try { + MappedFile mappedFile = new MappedFile(file.getPath(), mappedFileSize); + + mappedFile.setWrotePosition(this.mappedFileSize); + mappedFile.setFlushedPosition(this.mappedFileSize); + mappedFile.setCommittedPosition(this.mappedFileSize); + this.mappedFiles.add(mappedFile); + log.info("load " + file.getPath() + " OK"); + } catch (IOException e) { + log.error("load file " + file + " error", e); + return false; + } + } + } + + return true; + } + + + public long howMuchFallBehind() { + if (this.mappedFiles.isEmpty()) + return 0; + + long committed = this.flushedWhere; + if (committed != 0) { + MappedFile mappedFile = this.getLastMappedFile(0, false); + if (mappedFile != null) { + return (mappedFile.getFileFromOffset() + mappedFile.getWrotePosition()) - committed; + } + } + + return 0; + } + + + public MappedFile getLastMappedFile(final long startOffset, boolean needCreate) { + long createOffset = -1; + MappedFile mappedFileLast = getLastMappedFile(); + + if (mappedFileLast == null) { + createOffset = startOffset - (startOffset % this.mappedFileSize); + } + + if (mappedFileLast != null && mappedFileLast.isFull()) { + createOffset = mappedFileLast.getFileFromOffset() + this.mappedFileSize; + } + + if (createOffset != -1 && needCreate) { + String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset); + String nextNextFilePath = this.storePath + File.separator + + UtilAll.offset2FileName(createOffset + this.mappedFileSize); + MappedFile mappedFile = null; + + if (this.allocateMappedFileService != null) { + mappedFile = this.allocateMappedFileService.putRequestAndReturnMappedFile(nextFilePath, + nextNextFilePath, this.mappedFileSize); + } else { + try { + mappedFile = new MappedFile(nextFilePath, this.mappedFileSize); + } catch (IOException e) { + log.error("create mappedFile exception", e); + } + } + + if (mappedFile != null) { + if (this.mappedFiles.isEmpty()) { + mappedFile.setFirstCreateInQueue(true); + } + this.mappedFiles.add(mappedFile); + } + + return mappedFile; + } + + return mappedFileLast; + } + + public MappedFile getLastMappedFile(final long startOffset) { + return getLastMappedFile(startOffset, true); + } + + public MappedFile getLastMappedFile() { + MappedFile mappedFileLast = null; + + while (!this.mappedFiles.isEmpty()) { + try { + mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); + break; + } catch (IndexOutOfBoundsException e) { + //continue; + } catch (Exception e) { + log.error("getLastMappedFile has exception.", e); + break; + } + } + + return mappedFileLast; + } + + public boolean resetOffset(long offset) { + MappedFile mappedFileLast = getLastMappedFile(); + + if (mappedFileLast != null) { + long lastOffset = mappedFileLast.getFileFromOffset() + + mappedFileLast.getWrotePosition(); + long diff = lastOffset - offset; + + final int maxDiff = this.mappedFileSize * 2; + if (diff > maxDiff) return false; + } + + ListIterator iterator = this.mappedFiles.listIterator(); + + while (iterator.hasPrevious()) { + mappedFileLast = iterator.previous(); + if (offset >= mappedFileLast.getFileFromOffset()) { + int where = (int) (offset % mappedFileLast.getFileSize()); + mappedFileLast.setFlushedPosition(where); + mappedFileLast.setWrotePosition(where); + mappedFileLast.setCommittedPosition(where); + break; + } else { + iterator.remove(); + } + } + return true; + } + + public long getMinOffset() { + + if (!this.mappedFiles.isEmpty()) { + try { + return this.mappedFiles.get(0).getFileFromOffset(); + } catch (IndexOutOfBoundsException e) { + //continue; + } catch (Exception e) { + log.error("getMinOffset has exception.", e); + } + } + return -1; + } + + + public long getMaxOffset() { + MappedFile mappedFile = getLastMappedFile(); + if (mappedFile != null) { + return mappedFile.getFileFromOffset() + mappedFile.getReadPosition(); + } + return 0; + } + + public long getMaxWrotePosition() { + MappedFile mappedFile = getLastMappedFile(); + if (mappedFile != null) { + return mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(); + } + return 0; + } + + public long remainHowManyDataToCommit() { + return getMaxWrotePosition() - committedWhere; + } + + public long remainHowManyDataToFlush() { + return getMaxOffset() - flushedWhere; + } + + public void deleteLastMappedFile() { + MappedFile lastMappedFile = getLastMappedFile(); + if (lastMappedFile != null) { + lastMappedFile.destroy(1000); + this.mappedFiles.remove(lastMappedFile); + log.info("on recover, destroy a logic mapped file " + lastMappedFile.getFileName()); + + } + } + + public int deleteExpiredFileByTime(final long expiredTime, + final int deleteFilesInterval, + final long intervalForcibly, + final boolean cleanImmediately) { + Object[] mfs = this.copyMappedFiles(0); + + if (null == mfs) + return 0; + + int mfsLength = mfs.length - 1; + int deleteCount = 0; + List files = new ArrayList(); + if (null != mfs) { + for (int i = 0; i < mfsLength; i++) { + MappedFile mappedFile = (MappedFile) mfs[i]; + long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime; + if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) { + if (mappedFile.destroy(intervalForcibly)) { + files.add(mappedFile); + deleteCount++; + + if (files.size() >= DELETE_FILES_BATCH_MAX) { + break; + } + + if (deleteFilesInterval > 0 && (i + 1) < mfsLength) { + try { + Thread.sleep(deleteFilesInterval); + } catch (InterruptedException e) { + } + } + } else { + break; + } + } + } + } + + deleteExpiredFile(files); + + return deleteCount; + } + + + public int deleteExpiredFileByOffset(long offset, int unitSize) { + Object[] mfs = this.copyMappedFiles(0); + + List files = new ArrayList(); + int deleteCount = 0; + if (null != mfs) { + + int mfsLength = mfs.length - 1; + + for (int i = 0; i < mfsLength; i++) { + boolean destroy; + MappedFile mappedFile = (MappedFile) mfs[i]; + SelectMappedBufferResult result = mappedFile.selectMappedBuffer(this.mappedFileSize - unitSize); + if (result != null) { + long maxOffsetInLogicQueue = result.getByteBuffer().getLong(); + result.release(); + destroy = maxOffsetInLogicQueue < offset; + if (destroy) { + log.info("physic min offset " + offset + ", logics in current mappedFile max offset " + + maxOffsetInLogicQueue + ", delete it"); + } + } else { + log.warn("this being not executed forever."); + break; + } + + if (destroy && mappedFile.destroy(1000 * 60)) { + files.add(mappedFile); + deleteCount++; + } else { + break; + } + } + } + + deleteExpiredFile(files); + + return deleteCount; + } + + + public boolean flush(final int flushLeastPages) { + boolean result = true; + MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, false); + if (mappedFile != null) { + long tmpTimeStamp = mappedFile.getStoreTimestamp(); + int offset = mappedFile.flush(flushLeastPages); + long where = mappedFile.getFileFromOffset() + offset; + result = where == this.flushedWhere; + this.flushedWhere = where; + if (0 == flushLeastPages) { + this.storeTimestamp = tmpTimeStamp; + } + } + + return result; + } + + public boolean commit(final int commitLeastPages) { + boolean result = true; + MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, false); + if (mappedFile != null) { + int offset = mappedFile.commit(commitLeastPages); + long where = mappedFile.getFileFromOffset() + offset; + result = where == this.committedWhere; + this.committedWhere = where; + } + + return result; + } + + + public MappedFile findMappedFileByOffset(final long offset, final boolean returnFirstOnNotFound) { + try { + MappedFile mappedFile = this.getFirstMappedFile(); + if (mappedFile != null) { + int index = (int) ((offset / this.mappedFileSize) - (mappedFile.getFileFromOffset() / this.mappedFileSize)); + if (index < 0 || index >= this.mappedFiles.size()) { + LOG_ERROR.warn("findMappedFileByOffset offset not matched, request Offset: {}, index: {}, mappedFileSize: {}, mappedFiles count: {}, StackTrace: {}", + offset, + index, + this.mappedFileSize, + this.mappedFiles.size(), + UtilAll.currentStackTrace()); + } + + try { + return this.mappedFiles.get(index); + } catch (Exception e) { + if (returnFirstOnNotFound) { + return mappedFile; + } + } + } + } catch (Exception e) { + log.error("findMappedFileByOffset Exception", e); + } + + return null; + } + + + public MappedFile getFirstMappedFile() { + MappedFile mappedFileFirst = null; + + if (!this.mappedFiles.isEmpty()) { + try { + mappedFileFirst = this.mappedFiles.get(0); + } catch (IndexOutOfBoundsException e) { + //ignore + } catch (Exception e) { + log.error("getFirstMappedFile has exception.", e); + } + } + + return mappedFileFirst; + } + + public MappedFile findMappedFileByOffset(final long offset) { + return findMappedFileByOffset(offset, false); + } + + + public long getMappedMemorySize() { + long size = 0; + + Object[] mfs = this.copyMappedFiles(0); + if (mfs != null) { + for (Object mf : mfs) { + if (((ReferenceResource) mf).isAvailable()) { + size += this.mappedFileSize; + } + } + } + + return size; + } + + + public boolean retryDeleteFirstFile(final long intervalForcibly) { + MappedFile mappedFile = this.getFirstMappedFile(); + if (mappedFile != null) { + if (!mappedFile.isAvailable()) { + log.warn("the mappedFile was destroyed once, but still alive, " + mappedFile.getFileName()); + boolean result = mappedFile.destroy(intervalForcibly); + if (result) { + log.info("the mappedFile re delete OK, " + mappedFile.getFileName()); + List tmpFiles = new ArrayList(); + tmpFiles.add(mappedFile); + this.deleteExpiredFile(tmpFiles); + } else { + log.warn("the mappedFile re delete failed, " + mappedFile.getFileName()); + } + + return result; + } + } + + return false; + } + + + public void shutdown(final long intervalForcibly) { + for (MappedFile mf : this.mappedFiles) { + mf.shutdown(intervalForcibly); + } + } + + + public void destroy() { + for (MappedFile mf : this.mappedFiles) { + mf.destroy(1000 * 3); + } + this.mappedFiles.clear(); + this.flushedWhere = 0; + + // delete parent directory + File file = new File(storePath); + if (file.isDirectory()) { + file.delete(); + } + } + + + public long getFlushedWhere() { + return flushedWhere; + } + + + public void setFlushedWhere(long flushedWhere) { + this.flushedWhere = flushedWhere; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public List getMappedFiles() { + return mappedFiles; + } + + + public int getMappedFileSize() { + return mappedFileSize; + } + + public long getCommittedWhere() { + return committedWhere; + } + + public void setCommittedWhere(final long committedWhere) { + this.committedWhere = committedWhere; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageArrivingListener.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageArrivingListener.java new file mode 100644 index 0000000000..6d227e877c --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageArrivingListener.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.store; + +public interface MessageArrivingListener { + void arriving(String topic, int queueId, long logicOffset, long tagsCode); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java new file mode 100644 index 0000000000..1c549569e3 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * @author shijia.wxr + */ +public class MessageExtBrokerInner extends MessageExt { + private static final long serialVersionUID = 7256001576878700634L; + private String propertiesString; + private long tagsCode; + + public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { + if (null == tags || tags.length() == 0) + return 0; + + return tags.hashCode(); + } + + + public String getPropertiesString() { + return propertiesString; + } + + + public void setPropertiesString(String propertiesString) { + this.propertiesString = propertiesString; + } + + + public long getTagsCode() { + return tagsCode; + } + + + public void setTagsCode(long tagsCode) { + this.tagsCode = tagsCode; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java new file mode 100644 index 0000000000..a10e60714e --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * @author shijia.wxr + */ +public interface MessageFilter { + boolean isMessageMatched(final SubscriptionData subscriptionData, final Long tagsCode); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java new file mode 100644 index 0000000000..30f7bf712b --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + +import java.util.HashMap; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public interface MessageStore { + + boolean load(); + + + void start() throws Exception; + + + void shutdown(); + + + void destroy(); + + PutMessageResult putMessage(final MessageExtBrokerInner msg); + + + GetMessageResult getMessage(final String group, final String topic, final int queueId, + final long offset, final int maxMsgNums, final SubscriptionData subscriptionData); + + + long getMaxOffsetInQuque(final String topic, final int queueId); + + + long getMinOffsetInQuque(final String topic, final int queueId); + + + long getCommitLogOffsetInQueue(final String topic, final int queueId, final long cqOffset); + + + long getOffsetInQueueByTime(final String topic, final int queueId, final long timestamp); + + + MessageExt lookMessageByOffset(final long commitLogOffset); + + + SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset); + + + SelectMappedBufferResult selectOneMessageByOffset(final long commitLogOffset, final int msgSize); + + String getRunningDataInfo(); + + + HashMap getRuntimeInfo(); + + + long getMaxPhyOffset(); + + + long getMinPhyOffset(); + + + long getEarliestMessageTime(final String topic, final int queueId); + long getEarliestMessageTime(); + + + long getMessageStoreTimeStamp(final String topic, final int queueId, final long offset); + + + long getMessageTotalInQueue(final String topic, final int queueId); + + SelectMappedBufferResult getCommitLogData(final long offset); + + + boolean appendToCommitLog(final long startOffset, final byte[] data); + + void excuteDeleteFilesManualy(); + + + QueryMessageResult queryMessage(final String topic, final String key, final int maxNum, + final long begin, final long end); + + + void updateHaMasterAddress(final String newAddr); + + + long slaveFallBehindMuch(); + + + long now(); + + + int cleanUnusedTopic(final Set topics); + + + void cleanExpiredConsumerQueue(); + + + boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset); + + + long dispatchBehindBytes(); + + long flush(); + + boolean resetWriteOffset(long phyOffset); + + long getConfirmOffset(); + + void setConfirmOffset(long phyOffset); + + boolean isOSPageCacheBusy(); + + long lockTimeMills(); + + boolean isTransientStorePoolDeficient(); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java new file mode 100644 index 0000000000..b36ae58d05 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * @author shijia.wxr + */ +public class PutMessageResult { + private PutMessageStatus putMessageStatus; + private AppendMessageResult appendMessageResult; + + + public PutMessageResult(PutMessageStatus putMessageStatus, AppendMessageResult appendMessageResult) { + this.putMessageStatus = putMessageStatus; + this.appendMessageResult = appendMessageResult; + } + + + public boolean isOk() { + return this.appendMessageResult != null && this.appendMessageResult.isOk(); + } + + + public AppendMessageResult getAppendMessageResult() { + return appendMessageResult; + } + + + public void setAppendMessageResult(AppendMessageResult appendMessageResult) { + this.appendMessageResult = appendMessageResult; + } + + + public PutMessageStatus getPutMessageStatus() { + return putMessageStatus; + } + + + public void setPutMessageStatus(PutMessageStatus putMessageStatus) { + this.putMessageStatus = putMessageStatus; + } + + + @Override + public String toString() { + return "PutMessageResult [putMessageStatus=" + putMessageStatus + ", appendMessageResult=" + + appendMessageResult + "]"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java new file mode 100644 index 0000000000..4758789862 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * @author shijia.wxr + */ +public enum PutMessageStatus { + PUT_OK, + FLUSH_DISK_TIMEOUT, + FLUSH_SLAVE_TIMEOUT, + SLAVE_NOT_AVAILABLE, + SERVICE_NOT_AVAILABLE, + CREATE_MAPEDFILE_FAILED, + MESSAGE_ILLEGAL, + PROPERTIES_SIZE_EXCEEDED, + OS_PAGECACHE_BUSY, + UNKNOWN_ERROR, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java new file mode 100644 index 0000000000..a8e3fed824 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class QueryMessageResult { + + private final List messageMapedList = + new ArrayList(100); + + private final List messageBufferList = new ArrayList(100); + private long indexLastUpdateTimestamp; + private long indexLastUpdatePhyoffset; + + private int bufferTotalSize = 0; + + + public void addMessage(final SelectMappedBufferResult mapedBuffer) { + this.messageMapedList.add(mapedBuffer); + this.messageBufferList.add(mapedBuffer.getByteBuffer()); + this.bufferTotalSize += mapedBuffer.getSize(); + } + + + public void release() { + for (SelectMappedBufferResult select : this.messageMapedList) { + select.release(); + } + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public void setIndexLastUpdateTimestamp(long indexLastUpdateTimestamp) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + } + + + public long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } + + + public void setIndexLastUpdatePhyoffset(long indexLastUpdatePhyoffset) { + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } + + + public List getMessageBufferList() { + return messageBufferList; + } + + + public int getBufferTotalSize() { + return bufferTotalSize; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java new file mode 100644 index 0000000000..7a50f3a591 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.util.concurrent.atomic.AtomicLong; + + +/** + * @author shijia.wxr + */ +public abstract class ReferenceResource { + protected final AtomicLong refCount = new AtomicLong(1); + protected volatile boolean available = true; + protected volatile boolean cleanupOver = false; + private volatile long firstShutdownTimestamp = 0; + + + public synchronized boolean hold() { + if (this.isAvailable()) { + if (this.refCount.getAndIncrement() > 0) { + return true; + } else { + this.refCount.getAndDecrement(); + } + } + + return false; + } + + + public boolean isAvailable() { + return this.available; + } + + + + public void shutdown(final long intervalForcibly) { + if (this.available) { + this.available = false; + this.firstShutdownTimestamp = System.currentTimeMillis(); + this.release(); + } + + else if (this.getRefCount() > 0) { + if ((System.currentTimeMillis() - this.firstShutdownTimestamp) >= intervalForcibly) { + this.refCount.set(-1000 - this.getRefCount()); + this.release(); + } + } + } + + public void release() { + long value = this.refCount.decrementAndGet(); + if (value > 0) + return; + + synchronized (this) { + + this.cleanupOver = this.cleanup(value); + } + } + + public long getRefCount() { + return this.refCount.get(); + } + + public abstract boolean cleanup(final long currentRef); + + + public boolean isCleanupOver() { + return this.refCount.get() <= 0 && this.cleanupOver; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java new file mode 100644 index 0000000000..9343ed91a9 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * @author shijia.wxr + */ +public class RunningFlags { + + private static final int NOT_READABLE_BIT = 1; + + private static final int NOT_WRITEABLE_BIT = 1 << 1; + + private static final int WRITE_LOGICS_QUEUE_ERROR_BIT = 1 << 2; + + private static final int WRITE_INDEX_FILE_ERROR_BIT = 1 << 3; + + private static final int DISK_FULL_BIT = 1 << 4; + private volatile int flagBits = 0; + + + public RunningFlags() { + } + + + public int getFlagBits() { + return flagBits; + } + + + public boolean getAndMakeReadable() { + boolean result = this.isReadable(); + if (!result) { + this.flagBits &= ~NOT_READABLE_BIT; + } + return result; + } + + + public boolean isReadable() { + if ((this.flagBits & NOT_READABLE_BIT) == 0) { + return true; + } + + return false; + } + + + public boolean getAndMakeNotReadable() { + boolean result = this.isReadable(); + if (result) { + this.flagBits |= NOT_READABLE_BIT; + } + return result; + } + + + public boolean getAndMakeWriteable() { + boolean result = this.isWriteable(); + if (!result) { + this.flagBits &= ~NOT_WRITEABLE_BIT; + } + return result; + } + + + public boolean isWriteable() { + if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) { + return true; + } + + return false; + } + + + public boolean getAndMakeNotWriteable() { + boolean result = this.isWriteable(); + if (result) { + this.flagBits |= NOT_WRITEABLE_BIT; + } + return result; + } + + + public void makeLogicsQueueError() { + this.flagBits |= WRITE_LOGICS_QUEUE_ERROR_BIT; + } + + + public boolean isLogicsQueueError() { + if ((this.flagBits & WRITE_LOGICS_QUEUE_ERROR_BIT) == WRITE_LOGICS_QUEUE_ERROR_BIT) { + return true; + } + + return false; + } + + + public void makeIndexFileError() { + this.flagBits |= WRITE_INDEX_FILE_ERROR_BIT; + } + + + public boolean isIndexFileError() { + if ((this.flagBits & WRITE_INDEX_FILE_ERROR_BIT) == WRITE_INDEX_FILE_ERROR_BIT) { + return true; + } + + return false; + } + + + public boolean getAndMakeDiskFull() { + boolean result = !((this.flagBits & DISK_FULL_BIT) == DISK_FULL_BIT); + this.flagBits |= DISK_FULL_BIT; + return result; + } + + + public boolean getAndMakeDiskOK() { + boolean result = !((this.flagBits & DISK_FULL_BIT) == DISK_FULL_BIT); + this.flagBits &= ~DISK_FULL_BIT; + return result; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMappedBufferResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMappedBufferResult.java new file mode 100644 index 0000000000..dba8072677 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMappedBufferResult.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; + + +/** + * @author shijia.wxr + */ +public class SelectMappedBufferResult { + + private final long startOffset; + + private final ByteBuffer byteBuffer; + + private int size; + + private MappedFile mappedFile; + + + public SelectMappedBufferResult(long startOffset, ByteBuffer byteBuffer, int size, MappedFile mappedFile) { + this.startOffset = startOffset; + this.byteBuffer = byteBuffer; + this.size = size; + this.mappedFile = mappedFile; + } + + + public ByteBuffer getByteBuffer() { + return byteBuffer; + } + + + public int getSize() { + return size; + } + + + public void setSize(final int s) { + this.size = s; + this.byteBuffer.limit(this.size); + } + + + public MappedFile getMappedFile() { + return mappedFile; + } + + +// @Override +// protected void finalize() { +// if (this.mappedFile != null) { +// this.release(); +// } +// } + + + public synchronized void release() { + if (this.mappedFile != null) { + this.mappedFile.release(); + this.mappedFile = null; + } + } + + + public long getStartOffset() { + return startOffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java new file mode 100644 index 0000000000..2d75fd5f4a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; + + +/** + * @author shijia.wxr + */ +public class StoreCheckpoint { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private final RandomAccessFile randomAccessFile; + private final FileChannel fileChannel; + private final MappedByteBuffer mappedByteBuffer; + private volatile long physicMsgTimestamp = 0; + private volatile long logicsMsgTimestamp = 0; + private volatile long indexMsgTimestamp = 0; + + + public StoreCheckpoint(final String scpPath) throws IOException { + File file = new File(scpPath); + MappedFile.ensureDirOK(file.getParent()); + boolean fileExists = file.exists(); + + this.randomAccessFile = new RandomAccessFile(file, "rw"); + this.fileChannel = this.randomAccessFile.getChannel(); + this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MappedFile.OS_PAGE_SIZE); + + if (fileExists) { + log.info("store checkpoint file exists, " + scpPath); + this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0); + this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8); + this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16); + + log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp)); + log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp)); + log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp)); + } else { + log.info("store checkpoint file not exists, " + scpPath); + } + } + + + public void shutdown() { + this.flush(); + + // unmap mappedByteBuffer + MappedFile.clean(this.mappedByteBuffer); + + try { + this.fileChannel.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + public void flush() { + this.mappedByteBuffer.putLong(0, this.physicMsgTimestamp); + this.mappedByteBuffer.putLong(8, this.logicsMsgTimestamp); + this.mappedByteBuffer.putLong(16, this.indexMsgTimestamp); + this.mappedByteBuffer.force(); + } + + + public long getPhysicMsgTimestamp() { + return physicMsgTimestamp; + } + + + public void setPhysicMsgTimestamp(long physicMsgTimestamp) { + this.physicMsgTimestamp = physicMsgTimestamp; + } + + + public long getLogicsMsgTimestamp() { + return logicsMsgTimestamp; + } + + + public void setLogicsMsgTimestamp(long logicsMsgTimestamp) { + this.logicsMsgTimestamp = logicsMsgTimestamp; + } + + + public long getMinTimestampIndex() { + return Math.min(this.getMinTimestamp(), this.indexMsgTimestamp); + } + + + public long getMinTimestamp() { + long min = Math.min(this.physicMsgTimestamp, this.logicsMsgTimestamp); + + + // fixed https://github.com/alibaba/RocketMQ/issues/467 + min -= 1000 * 3; + if (min < 0) + min = 0; + + return min; + } + + + public long getIndexMsgTimestamp() { + return indexMsgTimestamp; + } + + + public void setIndexMsgTimestamp(long indexMsgTimestamp) { + this.indexMsgTimestamp = indexMsgTimestamp; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java new file mode 100644 index 0000000000..b5ed3f3f26 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java @@ -0,0 +1,615 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * @author shijia.wxr + */ +public class StoreStatsService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + private static final int FREQUENCY_OF_SAMPLING = 1000; + + private static final int MAX_RECORDS_OF_SAMPLING = 60 * 10; + private static final String[] PUT_MESSAGE_ENTIRE_TIME_MAX_DESC = new String[]{ + "[<=0ms]", "[0~10ms]", "[10~50ms]", "[50~100ms]", "[100~200ms]", "[200~500ms]", "[500ms~1s]", "[1~2s]", "[2~3s]", "[3~4s]", "[4~5s]", "[5~10s]", "[10s~]", + }; + + private static int printTPSInterval = 60 * 1; + + private final AtomicLong putMessageFailedTimes = new AtomicLong(0); + + private final Map putMessageTopicTimesTotal = + new ConcurrentHashMap(128); + private final Map putMessageTopicSizeTotal = + new ConcurrentHashMap(128); + + private final AtomicLong getMessageTimesTotalFound = new AtomicLong(0); + private final AtomicLong getMessageTransferedMsgCount = new AtomicLong(0); + private final AtomicLong getMessageTimesTotalMiss = new AtomicLong(0); + private final LinkedList putTimesList = new LinkedList(); + + private final LinkedList getTimesFoundList = new LinkedList(); + private final LinkedList getTimesMissList = new LinkedList(); + private final LinkedList transferedMsgCountList = new LinkedList(); + private volatile AtomicLong[] putMessageDistributeTime; + private long messageStoreBootTimestamp = System.currentTimeMillis(); + private volatile long putMessageEntireTimeMax = 0; + private volatile long getMessageEntireTimeMax = 0; + // for putMessageEntireTimeMax + private ReentrantLock lockPut = new ReentrantLock(); + // for getMessageEntireTimeMax + private ReentrantLock lockGet = new ReentrantLock(); + + private volatile long dispatchMaxBuffer = 0; + + private ReentrantLock lockSampling = new ReentrantLock(); + private long lastPrintTimestamp = System.currentTimeMillis(); + + + public StoreStatsService() { + this.initPutMessageDistributeTime(); + } + + private AtomicLong[] initPutMessageDistributeTime() { + AtomicLong[] next = new AtomicLong[13]; + for (int i = 0; i < next.length; i++) { + next[i] = new AtomicLong(0); + } + + AtomicLong[] old = this.putMessageDistributeTime; + + this.putMessageDistributeTime = next; + + return old; + } + + public long getPutMessageEntireTimeMax() { + return putMessageEntireTimeMax; + } + + public void setPutMessageEntireTimeMax(long value) { + final AtomicLong[] times = this.putMessageDistributeTime; + + if (null == times) return; + + // us + if (value <= 0) { + times[0].incrementAndGet(); + } else if (value < 10) { + times[1].incrementAndGet(); + } else if (value < 50) { + times[2].incrementAndGet(); + } else if (value < 100) { + times[3].incrementAndGet(); + } else if (value < 200) { + times[4].incrementAndGet(); + } else if (value < 500) { + times[5].incrementAndGet(); + } else if (value < 1000) { + times[6].incrementAndGet(); + } + // 2s + else if (value < 2000) { + times[7].incrementAndGet(); + } + // 3s + else if (value < 3000) { + times[8].incrementAndGet(); + } + // 4s + else if (value < 4000) { + times[9].incrementAndGet(); + } + // 5s + else if (value < 5000) { + times[10].incrementAndGet(); + } + // 10s + else if (value < 10000) { + times[11].incrementAndGet(); + } else { + times[12].incrementAndGet(); + } + + if (value > this.putMessageEntireTimeMax) { + this.lockPut.lock(); + this.putMessageEntireTimeMax = + value > this.putMessageEntireTimeMax ? value : this.putMessageEntireTimeMax; + this.lockPut.unlock(); + } + } + + + public long getGetMessageEntireTimeMax() { + return getMessageEntireTimeMax; + } + + + public void setGetMessageEntireTimeMax(long value) { + if (value > this.getMessageEntireTimeMax) { + this.lockGet.lock(); + this.getMessageEntireTimeMax = + value > this.getMessageEntireTimeMax ? value : this.getMessageEntireTimeMax; + this.lockGet.unlock(); + } + } + + + public long getDispatchMaxBuffer() { + return dispatchMaxBuffer; + } + + + public void setDispatchMaxBuffer(long value) { + this.dispatchMaxBuffer = value > this.dispatchMaxBuffer ? value : this.dispatchMaxBuffer; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(1024); + Long totalTimes = getPutMessageTimesTotal(); + if (0 == totalTimes) { + totalTimes = 1L; + } + + sb.append("\truntime: " + this.getFormatRuntime() + "\r\n"); + sb.append("\tputMessageEntireTimeMax: " + this.putMessageEntireTimeMax + "\r\n"); + sb.append("\tputMessageTimesTotal: " + totalTimes + "\r\n"); + sb.append("\tputMessageSizeTotal: " + this.getPutMessageSizeTotal() + "\r\n"); + sb.append("\tputMessageDistributeTime: " + this.getPutMessageDistributeTimeStringInfo(totalTimes) + + "\r\n"); + sb.append("\tputMessageAverageSize: " + (this.getPutMessageSizeTotal() / totalTimes.doubleValue()) + + "\r\n"); + sb.append("\tdispatchMaxBuffer: " + this.dispatchMaxBuffer + "\r\n"); + sb.append("\tgetMessageEntireTimeMax: " + this.getMessageEntireTimeMax + "\r\n"); + sb.append("\tputTps: " + this.getPutTps() + "\r\n"); + sb.append("\tgetFoundTps: " + this.getGetFoundTps() + "\r\n"); + sb.append("\tgetMissTps: " + this.getGetMissTps() + "\r\n"); + sb.append("\tgetTotalTps: " + this.getGetTotalTps() + "\r\n"); + sb.append("\tgetTransferedTps: " + this.getGetTransferedTps() + "\r\n"); + return sb.toString(); + } + + public long getPutMessageTimesTotal() { + long rs = 0; + for (AtomicLong data : putMessageTopicTimesTotal.values()) { + rs += data.get(); + } + return rs; + } + + private String getFormatRuntime() { + final long millisecond = 1; + final long second = 1000 * millisecond; + final long minute = 60 * second; + final long hour = 60 * minute; + final long day = 24 * hour; + final MessageFormat messageFormat = new MessageFormat("[ {0} days, {1} hours, {2} minutes, {3} seconds ]"); + + long time = System.currentTimeMillis() - this.messageStoreBootTimestamp; + long days = time / day; + long hours = (time % day) / hour; + long minutes = (time % hour) / minute; + long seconds = (time % minute) / second; + return messageFormat.format(new Long[]{days, hours, minutes, seconds}); + } + + public long getPutMessageSizeTotal() { + long rs = 0; + for (AtomicLong data : putMessageTopicSizeTotal.values()) { + rs += data.get(); + } + return rs; + } + + private String getPutMessageDistributeTimeStringInfo(Long total) { + return this.putMessageDistributeTimeToString(); + } + + private String getPutTps() { + StringBuilder sb = new StringBuilder(); + + sb.append(this.getPutTps(10)); + sb.append(" "); + + + sb.append(this.getPutTps(60)); + sb.append(" "); + + + sb.append(this.getPutTps(600)); + + return sb.toString(); + } + + private String getGetFoundTps() { + StringBuilder sb = new StringBuilder(); + + sb.append(this.getGetFoundTps(10)); + sb.append(" "); + + + sb.append(this.getGetFoundTps(60)); + sb.append(" "); + + + sb.append(this.getGetFoundTps(600)); + + return sb.toString(); + } + + private String getGetMissTps() { + StringBuilder sb = new StringBuilder(); + + sb.append(this.getGetMissTps(10)); + sb.append(" "); + + + sb.append(this.getGetMissTps(60)); + sb.append(" "); + + + sb.append(this.getGetMissTps(600)); + + return sb.toString(); + } + + private String getGetTotalTps() { + StringBuilder sb = new StringBuilder(); + + sb.append(this.getGetTotalTps(10)); + sb.append(" "); + + + sb.append(this.getGetTotalTps(60)); + sb.append(" "); + + + sb.append(this.getGetTotalTps(600)); + + return sb.toString(); + } + + private String getGetTransferedTps() { + StringBuilder sb = new StringBuilder(); + + sb.append(this.getGetTransferedTps(10)); + sb.append(" "); + + + sb.append(this.getGetTransferedTps(60)); + sb.append(" "); + + + sb.append(this.getGetTransferedTps(600)); + + return sb.toString(); + } + + private String putMessageDistributeTimeToString() { + final AtomicLong[] times = this.putMessageDistributeTime; + if (null == times) return null; + + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < times.length; i++) { + long value = times[i].get(); + sb.append(String.format("%s:%d", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value)); + sb.append(" "); + } + + return sb.toString(); + } + + private String getPutTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.putTimesList.getLast(); + + if (this.putTimesList.size() > time) { + CallSnapshot lastBefore = this.putTimesList.get(this.putTimesList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } finally { + this.lockSampling.unlock(); + } + return result; + } + + private String getGetFoundTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.getTimesFoundList.getLast(); + + if (this.getTimesFoundList.size() > time) { + CallSnapshot lastBefore = + this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + } finally { + this.lockSampling.unlock(); + } + + return result; + } + + private String getGetMissTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.getTimesMissList.getLast(); + + if (this.getTimesMissList.size() > time) { + CallSnapshot lastBefore = + this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } finally { + this.lockSampling.unlock(); + } + + return result; + } + + private String getGetTotalTps(int time) { + this.lockSampling.lock(); + double found = 0; + double miss = 0; + try { + { + CallSnapshot last = this.getTimesFoundList.getLast(); + + if (this.getTimesFoundList.size() > time) { + CallSnapshot lastBefore = + this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1)); + found = CallSnapshot.getTPS(lastBefore, last); + } + } + { + CallSnapshot last = this.getTimesMissList.getLast(); + + if (this.getTimesMissList.size() > time) { + CallSnapshot lastBefore = + this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1)); + miss = CallSnapshot.getTPS(lastBefore, last); + } + } + + } finally { + this.lockSampling.unlock(); + } + + return Double.toString(found + miss); + } + + private String getGetTransferedTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.transferedMsgCountList.getLast(); + + if (this.transferedMsgCountList.size() > time) { + CallSnapshot lastBefore = + this.transferedMsgCountList.get(this.transferedMsgCountList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } finally { + this.lockSampling.unlock(); + } + + return result; + } + + public HashMap getRuntimeInfo() { + HashMap result = new HashMap(64); + + Long totalTimes = getPutMessageTimesTotal(); + if (0 == totalTimes) { + totalTimes = 1L; + } + + result.put("bootTimestamp", String.valueOf(this.messageStoreBootTimestamp)); + result.put("runtime", this.getFormatRuntime()); + result.put("putMessageEntireTimeMax", String.valueOf(this.putMessageEntireTimeMax)); + result.put("putMessageTimesTotal", String.valueOf(totalTimes)); + result.put("putMessageSizeTotal", String.valueOf(this.getPutMessageSizeTotal())); + result.put("putMessageDistributeTime", + String.valueOf(this.getPutMessageDistributeTimeStringInfo(totalTimes))); + result.put("putMessageAverageSize", + String.valueOf(this.getPutMessageSizeTotal() / totalTimes.doubleValue())); + result.put("dispatchMaxBuffer", String.valueOf(this.dispatchMaxBuffer)); + result.put("getMessageEntireTimeMax", String.valueOf(this.getMessageEntireTimeMax)); + result.put("putTps", String.valueOf(this.getPutTps())); + result.put("getFoundTps", String.valueOf(this.getGetFoundTps())); + result.put("getMissTps", String.valueOf(this.getGetMissTps())); + result.put("getTotalTps", String.valueOf(this.getGetTotalTps())); + result.put("getTransferedTps", String.valueOf(this.getGetTransferedTps())); + + return result; + } + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.waitForRunning(FREQUENCY_OF_SAMPLING); + + this.sampling(); + + this.printTps(); + } catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + @Override + public String getServiceName() { + return StoreStatsService.class.getSimpleName(); + } + + private void sampling() { + this.lockSampling.lock(); + try { + this.putTimesList.add(new CallSnapshot(System.currentTimeMillis(), getPutMessageTimesTotal())); + if (this.putTimesList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) { + this.putTimesList.removeFirst(); + } + + this.getTimesFoundList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTimesTotalFound.get())); + if (this.getTimesFoundList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) { + this.getTimesFoundList.removeFirst(); + } + + this.getTimesMissList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTimesTotalMiss.get())); + if (this.getTimesMissList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) { + this.getTimesMissList.removeFirst(); + } + + this.transferedMsgCountList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTransferedMsgCount.get())); + if (this.transferedMsgCountList.size() > (MAX_RECORDS_OF_SAMPLING + 1)) { + this.transferedMsgCountList.removeFirst(); + } + + } finally { + this.lockSampling.unlock(); + } + } + + private void printTps() { + if (System.currentTimeMillis() > (this.lastPrintTimestamp + printTPSInterval * 1000)) { + this.lastPrintTimestamp = System.currentTimeMillis(); + + log.info("[STORETPS] put_tps {} get_found_tps {} get_miss_tps {} get_transfered_tps {}", + this.getPutTps(printTPSInterval), + this.getGetFoundTps(printTPSInterval), + this.getGetMissTps(printTPSInterval), + this.getGetTransferedTps(printTPSInterval) + ); + + final AtomicLong[] times = this.initPutMessageDistributeTime(); + if (null == times) return; + + final StringBuilder sb = new StringBuilder(); + long totalPut = 0; + for (int i = 0; i < times.length; i++) { + long value = times[i].get(); + totalPut += value; + sb.append(String.format("%s:%d", PUT_MESSAGE_ENTIRE_TIME_MAX_DESC[i], value)); + sb.append(" "); + } + + log.info("[PAGECACHERT] TotalPut {}, PutMessageDistributeTime {}", totalPut, sb.toString()); + } + } + + public AtomicLong getGetMessageTimesTotalFound() { + return getMessageTimesTotalFound; + } + + + public AtomicLong getGetMessageTimesTotalMiss() { + return getMessageTimesTotalMiss; + } + + + public AtomicLong getGetMessageTransferedMsgCount() { + return getMessageTransferedMsgCount; + } + + + public AtomicLong getPutMessageFailedTimes() { + return putMessageFailedTimes; + } + + + public AtomicLong getSinglePutMessageTopicSizeTotal(String topic) { + AtomicLong rs = putMessageTopicSizeTotal.get(topic); + if (null == rs) { + rs = new AtomicLong(0); + putMessageTopicSizeTotal.put(topic, rs); + } + return rs; + } + + + public AtomicLong getSinglePutMessageTopicTimesTotal(String topic) { + AtomicLong rs = putMessageTopicTimesTotal.get(topic); + if (null == rs) { + rs = new AtomicLong(0); + putMessageTopicTimesTotal.put(topic, rs); + } + return rs; + } + + + public Map getPutMessageTopicTimesTotal() { + return putMessageTopicTimesTotal; + } + + + public Map getPutMessageTopicSizeTotal() { + return putMessageTopicSizeTotal; + } + + static class CallSnapshot { + public final long timestamp; + public final long callTimesTotal; + + + public CallSnapshot(long timestamp, long callTimesTotal) { + this.timestamp = timestamp; + this.callTimesTotal = callTimesTotal; + } + + + public static double getTPS(final CallSnapshot begin, final CallSnapshot end) { + long total = end.callTimesTotal - begin.callTimesTotal; + Long time = end.timestamp - begin.timestamp; + + double tps = total / time.doubleValue(); + + return tps * 1000; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java new file mode 100644 index 0000000000..9e0b565585 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; + + +/** + * @author shijia.wxr + * + */ +public class StoreUtil { + public static final long TOTAL_PHYSICAL_MEMORY_SIZE = getTotalPhysicalMemorySize(); + + + @SuppressWarnings("restriction") + public static long getTotalPhysicalMemorySize() { + long physicalTotal = 1024 * 1024 * 1024 * 24; + OperatingSystemMXBean osmxb = ManagementFactory.getOperatingSystemMXBean(); + if (osmxb instanceof com.sun.management.OperatingSystemMXBean) { + physicalTotal = ((com.sun.management.OperatingSystemMXBean) osmxb).getTotalPhysicalMemorySize(); + } + + return physicalTotal; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/TransientStorePool.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/TransientStorePool.java new file mode 100644 index 0000000000..8abe7e994e --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/TransientStorePool.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.util.LibC; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.nio.ch.DirectBuffer; + +import java.nio.ByteBuffer; +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; + +/** + * @author xinyuzhou.zxy + */ +public class TransientStorePool { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + private final int poolSize; + private final int fileSize; + private final Deque availableBuffers; + private final MessageStoreConfig storeConfig; + + public TransientStorePool(final MessageStoreConfig storeConfig) { + this.storeConfig = storeConfig; + this.poolSize = storeConfig.getTransientStorePoolSize(); + this.fileSize = storeConfig.getMapedFileSizeCommitLog(); + this.availableBuffers = new ConcurrentLinkedDeque<>(); + } + + /** + * It's a heavy init method. + */ + public void init() { + for (int i = 0; i < poolSize; i++) { + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(fileSize); + + final long address = ((DirectBuffer) byteBuffer).address(); + Pointer pointer = new Pointer(address); + LibC.INSTANCE.mlock(pointer, new NativeLong(fileSize)); + + availableBuffers.offer(byteBuffer); + } + } + + public void destroy() { + for (ByteBuffer byteBuffer : availableBuffers) { + final long address = ((DirectBuffer) byteBuffer).address(); + Pointer pointer = new Pointer(address); + LibC.INSTANCE.munlock(pointer, new NativeLong(fileSize)); + } + } + + public void returnBuffer(ByteBuffer byteBuffer) { + byteBuffer.position(0); + byteBuffer.limit(fileSize); + this.availableBuffers.offerFirst(byteBuffer); + } + + public ByteBuffer borrowBuffer() { + ByteBuffer buffer = availableBuffers.pollFirst(); + if (availableBuffers.size() < poolSize * 0.4) { + log.warn("TransientStorePool only remain {} sheets.", availableBuffers.size()); + } + return buffer; + } + + public int remainBufferNumbs() { + if (storeConfig.isTransientStorePoolEnable()) { + return availableBuffers.size(); + } + return Integer.MAX_VALUE; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java new file mode 100644 index 0000000000..06714b21fa --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +/** + * @author shijia.wxr + */ +public enum BrokerRole { + ASYNC_MASTER, + SYNC_MASTER, + SLAVE; +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java new file mode 100644 index 0000000000..48ae4b2ba8 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +/** + * @author shijia.wxr + */ +public enum FlushDiskType { + SYNC_FLUSH, + ASYNC_FLUSH +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java new file mode 100644 index 0000000000..138bc2eacc --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java @@ -0,0 +1,727 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.store.ConsumeQueue; + +import java.io.File; + + +/** + * @author vongosling + * @author shijia.wxr + */ +public class MessageStoreConfig { + //The root directory in which the log data is kept + @ImportantField + private String storePathRootDir = System.getProperty("user.home") + File.separator + "store"; + + //The directory in which the commitlog is kept + @ImportantField + private String storePathCommitLog = System.getProperty("user.home") + File.separator + "store" + + File.separator + "commitlog"; + + // CommitLog file size,default is 1G + private int mapedFileSizeCommitLog = 1024 * 1024 * 1024; + // ConsumeQueue file size,default is 30W + private int mapedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQ_STORE_UNIT_SIZE; + + // CommitLog flush interval + // flush data to disk + @ImportantField + private int flushIntervalCommitLog = 500; + + // Only used if TransientStorePool enabled + // flush data to FileChannel + @ImportantField + private int commitIntervalCommitLog = 200; + + private boolean useReentrantLockWhenPutMessage = false; + + // Whether schedule flush,default is real-time + @ImportantField + private boolean flushCommitLogTimed = false; + // ConsumeQueue flush interval + private int flushIntervalConsumeQueue = 1000; + // Resource reclaim interval + private int cleanResourceInterval = 10000; + // CommitLog removal interval + private int deleteCommitLogFilesInterval = 100; + // ConsumeQueue removal interval + private int deleteConsumeQueueFilesInterval = 100; + private int destroyMapedFileIntervalForcibly = 1000 * 120; + private int redeleteHangedFileInterval = 1000 * 120; + // When to delete,default is at 4 am + @ImportantField + private String deleteWhen = "04"; + private int diskMaxUsedSpaceRatio = 75; + // The number of hours to keep a log file before deleting it (in hours) + @ImportantField + private int fileReservedTime = 72; + // Flow control for ConsumeQueue + private int putMsgIndexHightWater = 600000; + // The maximum size of a single log file,default is 512K + private int maxMessageSize = 1024 * 1024 * 4; + // Whether check the CRC32 of the records consumed. + // This ensures no on-the-wire or on-disk corruption to the messages occurred. + // This check adds some overhead,so it may be disabled in cases seeking extreme performance. + private boolean checkCRCOnRecover = true; + // How many pages are to be flushed when flush CommitLog + private int flushCommitLogLeastPages = 4; + // How many pages are to be committed when commit data to file + private int commitCommitLogLeastPages = 4; + // Flush page size when the disk in warming state + private int flushLeastPagesWhenWarmMapedFile = 1024 / 4 * 16; + // How many pages are to be flushed when flush ConsumeQueue + private int flushConsumeQueueLeastPages = 2; + private int flushCommitLogThoroughInterval = 1000 * 10; + private int commitCommitLogThoroughInterval = 200; + private int flushConsumeQueueThoroughInterval = 1000 * 60; + @ImportantField + private int maxTransferBytesOnMessageInMemory = 1024 * 256; + @ImportantField + private int maxTransferCountOnMessageInMemory = 32; + @ImportantField + private int maxTransferBytesOnMessageInDisk = 1024 * 64; + @ImportantField + private int maxTransferCountOnMessageInDisk = 8; + @ImportantField + private int accessMessageInMemoryMaxRatio = 40; + @ImportantField + private boolean messageIndexEnable = true; + private int maxHashSlotNum = 5000000; + private int maxIndexNum = 5000000 * 4; + private int maxMsgsNumBatch = 64; + @ImportantField + private boolean messageIndexSafe = false; + private int haListenPort = 10912; + private int haSendHeartbeatInterval = 1000 * 5; + private int haHousekeepingInterval = 1000 * 20; + private int haTransferBatchSize = 1024 * 32; + @ImportantField + private String haMasterAddress = null; + private int haSlaveFallbehindMax = 1024 * 1024 * 256; + @ImportantField + private BrokerRole brokerRole = BrokerRole.ASYNC_MASTER; + @ImportantField + private FlushDiskType flushDiskType = FlushDiskType.ASYNC_FLUSH; + private int syncFlushTimeout = 1000 * 5; + private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h"; + private long flushDelayOffsetInterval = 1000 * 10; + @ImportantField + private boolean cleanFileForciblyEnable = true; + private boolean warmMapedFileEnable = false; + private boolean offsetCheckInSlave = false; + private boolean debugLockEnable = false; + private boolean duplicationEnable = false; + private boolean diskFallRecorded = true; + private long osPageCacheBusyTimeOutMills = 1000; + private int defaultQueryMaxNum = 32; + + @ImportantField + private boolean transientStorePoolEnable = false; + private int transientStorePoolSize = 5; + private boolean fastFailIfNoBufferInStorePool = false; + + public boolean isDebugLockEnable() { + return debugLockEnable; + } + + public void setDebugLockEnable(final boolean debugLockEnable) { + this.debugLockEnable = debugLockEnable; + } + + public boolean isDuplicationEnable() { + return duplicationEnable; + } + + public void setDuplicationEnable(final boolean duplicationEnable) { + this.duplicationEnable = duplicationEnable; + } + + public long getOsPageCacheBusyTimeOutMills() { + return osPageCacheBusyTimeOutMills; + } + + public void setOsPageCacheBusyTimeOutMills(final long osPageCacheBusyTimeOutMills) { + this.osPageCacheBusyTimeOutMills = osPageCacheBusyTimeOutMills; + } + + public boolean isDiskFallRecorded() { + return diskFallRecorded; + } + + public void setDiskFallRecorded(final boolean diskFallRecorded) { + this.diskFallRecorded = diskFallRecorded; + } + + public boolean isWarmMapedFileEnable() { + return warmMapedFileEnable; + } + + + public void setWarmMapedFileEnable(boolean warmMapedFileEnable) { + this.warmMapedFileEnable = warmMapedFileEnable; + } + + + public int getMapedFileSizeCommitLog() { + return mapedFileSizeCommitLog; + } + + + public void setMapedFileSizeCommitLog(int mapedFileSizeCommitLog) { + this.mapedFileSizeCommitLog = mapedFileSizeCommitLog; + } + + + public int getMapedFileSizeConsumeQueue() { + + int factor = (int) Math.ceil(this.mapedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0)); + return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE); + } + + + public void setMapedFileSizeConsumeQueue(int mapedFileSizeConsumeQueue) { + this.mapedFileSizeConsumeQueue = mapedFileSizeConsumeQueue; + } + + + public int getFlushIntervalCommitLog() { + return flushIntervalCommitLog; + } + + + public void setFlushIntervalCommitLog(int flushIntervalCommitLog) { + this.flushIntervalCommitLog = flushIntervalCommitLog; + } + + + public int getFlushIntervalConsumeQueue() { + return flushIntervalConsumeQueue; + } + + + public void setFlushIntervalConsumeQueue(int flushIntervalConsumeQueue) { + this.flushIntervalConsumeQueue = flushIntervalConsumeQueue; + } + + + public int getPutMsgIndexHightWater() { + return putMsgIndexHightWater; + } + + + public void setPutMsgIndexHightWater(int putMsgIndexHightWater) { + this.putMsgIndexHightWater = putMsgIndexHightWater; + } + + + public int getCleanResourceInterval() { + return cleanResourceInterval; + } + + + public void setCleanResourceInterval(int cleanResourceInterval) { + this.cleanResourceInterval = cleanResourceInterval; + } + + + public int getMaxMessageSize() { + return maxMessageSize; + } + + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + + public boolean isCheckCRCOnRecover() { + return checkCRCOnRecover; + } + + + public boolean getCheckCRCOnRecover() { + return checkCRCOnRecover; + } + + + public void setCheckCRCOnRecover(boolean checkCRCOnRecover) { + this.checkCRCOnRecover = checkCRCOnRecover; + } + + + public String getStorePathCommitLog() { + return storePathCommitLog; + } + + + public void setStorePathCommitLog(String storePathCommitLog) { + this.storePathCommitLog = storePathCommitLog; + } + + + public String getDeleteWhen() { + return deleteWhen; + } + + + public void setDeleteWhen(String deleteWhen) { + this.deleteWhen = deleteWhen; + } + + + public int getDiskMaxUsedSpaceRatio() { + if (this.diskMaxUsedSpaceRatio < 10) + return 10; + + if (this.diskMaxUsedSpaceRatio > 95) + return 95; + + return diskMaxUsedSpaceRatio; + } + + + public void setDiskMaxUsedSpaceRatio(int diskMaxUsedSpaceRatio) { + this.diskMaxUsedSpaceRatio = diskMaxUsedSpaceRatio; + } + + + public int getDeleteCommitLogFilesInterval() { + return deleteCommitLogFilesInterval; + } + + + public void setDeleteCommitLogFilesInterval(int deleteCommitLogFilesInterval) { + this.deleteCommitLogFilesInterval = deleteCommitLogFilesInterval; + } + + + public int getDeleteConsumeQueueFilesInterval() { + return deleteConsumeQueueFilesInterval; + } + + + public void setDeleteConsumeQueueFilesInterval(int deleteConsumeQueueFilesInterval) { + this.deleteConsumeQueueFilesInterval = deleteConsumeQueueFilesInterval; + } + + + public int getMaxTransferBytesOnMessageInMemory() { + return maxTransferBytesOnMessageInMemory; + } + + + public void setMaxTransferBytesOnMessageInMemory(int maxTransferBytesOnMessageInMemory) { + this.maxTransferBytesOnMessageInMemory = maxTransferBytesOnMessageInMemory; + } + + + public int getMaxTransferCountOnMessageInMemory() { + return maxTransferCountOnMessageInMemory; + } + + + public void setMaxTransferCountOnMessageInMemory(int maxTransferCountOnMessageInMemory) { + this.maxTransferCountOnMessageInMemory = maxTransferCountOnMessageInMemory; + } + + + public int getMaxTransferBytesOnMessageInDisk() { + return maxTransferBytesOnMessageInDisk; + } + + + public void setMaxTransferBytesOnMessageInDisk(int maxTransferBytesOnMessageInDisk) { + this.maxTransferBytesOnMessageInDisk = maxTransferBytesOnMessageInDisk; + } + + + public int getMaxTransferCountOnMessageInDisk() { + return maxTransferCountOnMessageInDisk; + } + + + public void setMaxTransferCountOnMessageInDisk(int maxTransferCountOnMessageInDisk) { + this.maxTransferCountOnMessageInDisk = maxTransferCountOnMessageInDisk; + } + + + public int getFlushCommitLogLeastPages() { + return flushCommitLogLeastPages; + } + + + public void setFlushCommitLogLeastPages(int flushCommitLogLeastPages) { + this.flushCommitLogLeastPages = flushCommitLogLeastPages; + } + + + public int getFlushConsumeQueueLeastPages() { + return flushConsumeQueueLeastPages; + } + + + public void setFlushConsumeQueueLeastPages(int flushConsumeQueueLeastPages) { + this.flushConsumeQueueLeastPages = flushConsumeQueueLeastPages; + } + + + public int getFlushCommitLogThoroughInterval() { + return flushCommitLogThoroughInterval; + } + + + public void setFlushCommitLogThoroughInterval(int flushCommitLogThoroughInterval) { + this.flushCommitLogThoroughInterval = flushCommitLogThoroughInterval; + } + + + public int getFlushConsumeQueueThoroughInterval() { + return flushConsumeQueueThoroughInterval; + } + + + public void setFlushConsumeQueueThoroughInterval(int flushConsumeQueueThoroughInterval) { + this.flushConsumeQueueThoroughInterval = flushConsumeQueueThoroughInterval; + } + + + public int getDestroyMapedFileIntervalForcibly() { + return destroyMapedFileIntervalForcibly; + } + + + public void setDestroyMapedFileIntervalForcibly(int destroyMapedFileIntervalForcibly) { + this.destroyMapedFileIntervalForcibly = destroyMapedFileIntervalForcibly; + } + + + public int getFileReservedTime() { + return fileReservedTime; + } + + + public void setFileReservedTime(int fileReservedTime) { + this.fileReservedTime = fileReservedTime; + } + + + public int getRedeleteHangedFileInterval() { + return redeleteHangedFileInterval; + } + + + public void setRedeleteHangedFileInterval(int redeleteHangedFileInterval) { + this.redeleteHangedFileInterval = redeleteHangedFileInterval; + } + + + public int getAccessMessageInMemoryMaxRatio() { + return accessMessageInMemoryMaxRatio; + } + + + public void setAccessMessageInMemoryMaxRatio(int accessMessageInMemoryMaxRatio) { + this.accessMessageInMemoryMaxRatio = accessMessageInMemoryMaxRatio; + } + + + public boolean isMessageIndexEnable() { + return messageIndexEnable; + } + + + public void setMessageIndexEnable(boolean messageIndexEnable) { + this.messageIndexEnable = messageIndexEnable; + } + + + public int getMaxHashSlotNum() { + return maxHashSlotNum; + } + + + public void setMaxHashSlotNum(int maxHashSlotNum) { + this.maxHashSlotNum = maxHashSlotNum; + } + + + public int getMaxIndexNum() { + return maxIndexNum; + } + + + public void setMaxIndexNum(int maxIndexNum) { + this.maxIndexNum = maxIndexNum; + } + + + public int getMaxMsgsNumBatch() { + return maxMsgsNumBatch; + } + + + public void setMaxMsgsNumBatch(int maxMsgsNumBatch) { + this.maxMsgsNumBatch = maxMsgsNumBatch; + } + + + public int getHaListenPort() { + return haListenPort; + } + + + public void setHaListenPort(int haListenPort) { + this.haListenPort = haListenPort; + } + + + public int getHaSendHeartbeatInterval() { + return haSendHeartbeatInterval; + } + + + public void setHaSendHeartbeatInterval(int haSendHeartbeatInterval) { + this.haSendHeartbeatInterval = haSendHeartbeatInterval; + } + + + public int getHaHousekeepingInterval() { + return haHousekeepingInterval; + } + + + public void setHaHousekeepingInterval(int haHousekeepingInterval) { + this.haHousekeepingInterval = haHousekeepingInterval; + } + + + public BrokerRole getBrokerRole() { + return brokerRole; + } + + public void setBrokerRole(BrokerRole brokerRole) { + this.brokerRole = brokerRole; + } + + public void setBrokerRole(String brokerRole) { + this.brokerRole = BrokerRole.valueOf(brokerRole); + } + + public int getHaTransferBatchSize() { + return haTransferBatchSize; + } + + + public void setHaTransferBatchSize(int haTransferBatchSize) { + this.haTransferBatchSize = haTransferBatchSize; + } + + + public int getHaSlaveFallbehindMax() { + return haSlaveFallbehindMax; + } + + + public void setHaSlaveFallbehindMax(int haSlaveFallbehindMax) { + this.haSlaveFallbehindMax = haSlaveFallbehindMax; + } + + + public FlushDiskType getFlushDiskType() { + return flushDiskType; + } + + public void setFlushDiskType(FlushDiskType flushDiskType) { + this.flushDiskType = flushDiskType; + } + + public void setFlushDiskType(String type) { + this.flushDiskType = FlushDiskType.valueOf(type); + } + + public int getSyncFlushTimeout() { + return syncFlushTimeout; + } + + + public void setSyncFlushTimeout(int syncFlushTimeout) { + this.syncFlushTimeout = syncFlushTimeout; + } + + + public String getHaMasterAddress() { + return haMasterAddress; + } + + + public void setHaMasterAddress(String haMasterAddress) { + this.haMasterAddress = haMasterAddress; + } + + + public String getMessageDelayLevel() { + return messageDelayLevel; + } + + + public void setMessageDelayLevel(String messageDelayLevel) { + this.messageDelayLevel = messageDelayLevel; + } + + + public long getFlushDelayOffsetInterval() { + return flushDelayOffsetInterval; + } + + + public void setFlushDelayOffsetInterval(long flushDelayOffsetInterval) { + this.flushDelayOffsetInterval = flushDelayOffsetInterval; + } + + + public boolean isCleanFileForciblyEnable() { + return cleanFileForciblyEnable; + } + + + public void setCleanFileForciblyEnable(boolean cleanFileForciblyEnable) { + this.cleanFileForciblyEnable = cleanFileForciblyEnable; + } + + + public boolean isMessageIndexSafe() { + return messageIndexSafe; + } + + + public void setMessageIndexSafe(boolean messageIndexSafe) { + this.messageIndexSafe = messageIndexSafe; + } + + + public boolean isFlushCommitLogTimed() { + return flushCommitLogTimed; + } + + + public void setFlushCommitLogTimed(boolean flushCommitLogTimed) { + this.flushCommitLogTimed = flushCommitLogTimed; + } + + + public String getStorePathRootDir() { + return storePathRootDir; + } + + + public void setStorePathRootDir(String storePathRootDir) { + this.storePathRootDir = storePathRootDir; + } + + + public int getFlushLeastPagesWhenWarmMapedFile() { + return flushLeastPagesWhenWarmMapedFile; + } + + + public void setFlushLeastPagesWhenWarmMapedFile(int flushLeastPagesWhenWarmMapedFile) { + this.flushLeastPagesWhenWarmMapedFile = flushLeastPagesWhenWarmMapedFile; + } + + + public boolean isOffsetCheckInSlave() { + return offsetCheckInSlave; + } + + + public void setOffsetCheckInSlave(boolean offsetCheckInSlave) { + this.offsetCheckInSlave = offsetCheckInSlave; + } + + public int getDefaultQueryMaxNum() { + return defaultQueryMaxNum; + } + + public void setDefaultQueryMaxNum(int defaultQueryMaxNum) { + this.defaultQueryMaxNum = defaultQueryMaxNum; + } + + /** + * Enable transient commitLog store poll only if transientStorePoolEnable is true and the FlushDiskType is ASYNC_FLUSH + * @return true or false + */ + public boolean isTransientStorePoolEnable() { + return transientStorePoolEnable && FlushDiskType.ASYNC_FLUSH == getFlushDiskType() + && BrokerRole.SLAVE != getBrokerRole(); + } + + public void setTransientStorePoolEnable(final boolean transientStorePoolEnable) { + this.transientStorePoolEnable = transientStorePoolEnable; + } + + public int getTransientStorePoolSize() { + return transientStorePoolSize; + } + + public void setTransientStorePoolSize(final int transientStorePoolSize) { + this.transientStorePoolSize = transientStorePoolSize; + } + + public int getCommitIntervalCommitLog() { + return commitIntervalCommitLog; + } + + public void setCommitIntervalCommitLog(final int commitIntervalCommitLog) { + this.commitIntervalCommitLog = commitIntervalCommitLog; + } + + public boolean isFastFailIfNoBufferInStorePool() { + return fastFailIfNoBufferInStorePool; + } + + public void setFastFailIfNoBufferInStorePool(final boolean fastFailIfNoBufferInStorePool) { + this.fastFailIfNoBufferInStorePool = fastFailIfNoBufferInStorePool; + } + + public boolean isUseReentrantLockWhenPutMessage() { + return useReentrantLockWhenPutMessage; + } + + public void setUseReentrantLockWhenPutMessage(final boolean useReentrantLockWhenPutMessage) { + this.useReentrantLockWhenPutMessage = useReentrantLockWhenPutMessage; + } + + public int getCommitCommitLogLeastPages() { + return commitCommitLogLeastPages; + } + + public void setCommitCommitLogLeastPages(final int commitCommitLogLeastPages) { + this.commitCommitLogLeastPages = commitCommitLogLeastPages; + } + + public int getCommitCommitLogThoroughInterval() { + return commitCommitLogThoroughInterval; + } + + public void setCommitCommitLogThoroughInterval(final int commitCommitLogThoroughInterval) { + this.commitCommitLogThoroughInterval = commitCommitLogThoroughInterval; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java new file mode 100644 index 0000000000..d3d77c05b5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +import java.io.File; + + +public class StorePathConfigHelper { + + public static String getStorePathConsumeQueue(final String rootDir) { + return rootDir + File.separator + "consumequeue"; + } + + + public static String getStorePathIndex(final String rootDir) { + return rootDir + File.separator + "index"; + } + + + public static String getStoreCheckpoint(final String rootDir) { + return rootDir + File.separator + "checkpoint"; + } + + + public static String getAbortFile(final String rootDir) { + return rootDir + File.separator + "abort"; + } + + + public static String getDelayOffsetStorePath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "delayOffset.json"; + } + + + public static String getTranStateTableStorePath(final String rootDir) { + return rootDir + File.separator + "transaction" + File.separator + "statetable"; + } + + + public static String getTranRedoLogStorePath(final String rootDir) { + return rootDir + File.separator + "transaction" + File.separator + "redolog"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java new file mode 100644 index 0000000000..17730593e7 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java @@ -0,0 +1,429 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; + + +/** + * @author shijia.wxr + */ +public class HAConnection { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private final HAService haService; + private final SocketChannel socketChannel; + private final String clientAddr; + private WriteSocketService writeSocketService; + private ReadSocketService readSocketService; + + private volatile long slaveRequestOffset = -1; + private volatile long slaveAckOffset = -1; + + + public HAConnection(final HAService haService, final SocketChannel socketChannel) throws IOException { + this.haService = haService; + this.socketChannel = socketChannel; + this.clientAddr = this.socketChannel.socket().getRemoteSocketAddress().toString(); + this.socketChannel.configureBlocking(false); + this.socketChannel.socket().setSoLinger(false, -1); + this.socketChannel.socket().setTcpNoDelay(true); + this.socketChannel.socket().setReceiveBufferSize(1024 * 64); + this.socketChannel.socket().setSendBufferSize(1024 * 64); + this.writeSocketService = new WriteSocketService(this.socketChannel); + this.readSocketService = new ReadSocketService(this.socketChannel); + this.haService.getConnectionCount().incrementAndGet(); + } + + + public void start() { + this.readSocketService.start(); + this.writeSocketService.start(); + } + + + public void shutdown() { + this.writeSocketService.shutdown(true); + this.readSocketService.shutdown(true); + this.close(); + } + + + public void close() { + if (this.socketChannel != null) { + try { + this.socketChannel.close(); + } catch (IOException e) { + HAConnection.log.error("", e); + } + } + } + + + public SocketChannel getSocketChannel() { + return socketChannel; + } + + /** + + * + * @author shijia.wxr + */ + class ReadSocketService extends ServiceThread { + private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024; + private final Selector selector; + private final SocketChannel socketChannel; + private final ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE); + private int processPostion = 0; + private volatile long lastReadTimestamp = System.currentTimeMillis(); + + + public ReadSocketService(final SocketChannel socketChannel) throws IOException { + this.selector = RemotingUtil.openSelector(); + this.socketChannel = socketChannel; + this.socketChannel.register(this.selector, SelectionKey.OP_READ); + this.thread.setDaemon(true); + } + + + @Override + public void run() { + HAConnection.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.selector.select(1000); + boolean ok = this.processReadEvent(); + if (!ok) { + HAConnection.log.error("processReadEvent error"); + break; + } + + + long interval = HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() - this.lastReadTimestamp; + if (interval > HAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaHousekeepingInterval()) { + log.warn("ha housekeeping, found this connection[" + HAConnection.this.clientAddr + "] expired, " + interval); + break; + } + } catch (Exception e) { + HAConnection.log.error(this.getServiceName() + " service has exception.", e); + break; + } + } + + this.makeStop(); + + writeSocketService.makeStop(); + + + haService.removeConnection(HAConnection.this); + + + HAConnection.this.haService.getConnectionCount().decrementAndGet(); + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + try { + this.selector.close(); + this.socketChannel.close(); + } catch (IOException e) { + HAConnection.log.error("", e); + } + + HAConnection.log.info(this.getServiceName() + " service end"); + } + + @Override + public String getServiceName() { + return ReadSocketService.class.getSimpleName(); + } + + private boolean processReadEvent() { + int readSizeZeroTimes = 0; + + if (!this.byteBufferRead.hasRemaining()) { + this.byteBufferRead.flip(); + this.processPostion = 0; + } + + while (this.byteBufferRead.hasRemaining()) { + try { + int readSize = this.socketChannel.read(this.byteBufferRead); + if (readSize > 0) { + readSizeZeroTimes = 0; + this.lastReadTimestamp = HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + if ((this.byteBufferRead.position() - this.processPostion) >= 8) { + int pos = this.byteBufferRead.position() - (this.byteBufferRead.position() % 8); + long readOffset = this.byteBufferRead.getLong(pos - 8); + this.processPostion = pos; + + + HAConnection.this.slaveAckOffset = readOffset; + if (HAConnection.this.slaveRequestOffset < 0) { + HAConnection.this.slaveRequestOffset = readOffset; + log.info("slave[" + HAConnection.this.clientAddr + "] request offset " + readOffset); + } + + + HAConnection.this.haService.notifyTransferSome(HAConnection.this.slaveAckOffset); + } + } else if (readSize == 0) { + if (++readSizeZeroTimes >= 3) { + break; + } + } else { + log.error("read socket[" + HAConnection.this.clientAddr + "] < 0"); + return false; + } + } catch (IOException e) { + log.error("processReadEvent exception", e); + return false; + } + } + + return true; + } + } + + /** + + * + * @author shijia.wxr + */ + class WriteSocketService extends ServiceThread { + private final Selector selector; + private final SocketChannel socketChannel; + + private final int headerSize = 8 + 4; + private final ByteBuffer byteBufferHeader = ByteBuffer.allocate(headerSize); + private long nextTransferFromWhere = -1; + private SelectMappedBufferResult selectMappedBufferResult; + private boolean lastWriteOver = true; + private long lastWriteTimestamp = System.currentTimeMillis(); + + + public WriteSocketService(final SocketChannel socketChannel) throws IOException { + this.selector = RemotingUtil.openSelector(); + this.socketChannel = socketChannel; + this.socketChannel.register(this.selector, SelectionKey.OP_WRITE); + this.thread.setDaemon(true); + } + + + @Override + public void run() { + HAConnection.log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.selector.select(1000); + + if (-1 == HAConnection.this.slaveRequestOffset) { + Thread.sleep(10); + continue; + } + + + + if (-1 == this.nextTransferFromWhere) { + if (0 == HAConnection.this.slaveRequestOffset) { + long masterOffset = HAConnection.this.haService.getDefaultMessageStore().getCommitLog().getMaxOffset(); + masterOffset = + masterOffset + - (masterOffset % HAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig() + .getMapedFileSizeCommitLog()); + + if (masterOffset < 0) { + masterOffset = 0; + } + + this.nextTransferFromWhere = masterOffset; + } else { + this.nextTransferFromWhere = HAConnection.this.slaveRequestOffset; + } + + log.info("master transfer data from " + this.nextTransferFromWhere + " to slave[" + HAConnection.this.clientAddr + + "], and slave request " + HAConnection.this.slaveRequestOffset); + } + + if (this.lastWriteOver) { + + long interval = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() - this.lastWriteTimestamp; + + if (interval > HAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig() + .getHaSendHeartbeatInterval()) { + + // Build Header + this.byteBufferHeader.position(0); + this.byteBufferHeader.limit(headerSize); + this.byteBufferHeader.putLong(this.nextTransferFromWhere); + this.byteBufferHeader.putInt(0); + this.byteBufferHeader.flip(); + + this.lastWriteOver = this.transferData(); + if (!this.lastWriteOver) + continue; + } + } + + else { + this.lastWriteOver = this.transferData(); + if (!this.lastWriteOver) + continue; + } + + SelectMappedBufferResult selectResult = + HAConnection.this.haService.getDefaultMessageStore().getCommitLogData(this.nextTransferFromWhere); + if (selectResult != null) { + int size = selectResult.getSize(); + if (size > HAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize()) { + size = HAConnection.this.haService.getDefaultMessageStore().getMessageStoreConfig().getHaTransferBatchSize(); + } + + long thisOffset = this.nextTransferFromWhere; + this.nextTransferFromWhere += size; + + selectResult.getByteBuffer().limit(size); + this.selectMappedBufferResult = selectResult; + + // Build Header + this.byteBufferHeader.position(0); + this.byteBufferHeader.limit(headerSize); + this.byteBufferHeader.putLong(thisOffset); + this.byteBufferHeader.putInt(size); + this.byteBufferHeader.flip(); + + this.lastWriteOver = this.transferData(); + } else { + + HAConnection.this.haService.getWaitNotifyObject().allWaitForRunning(100); + } + } catch (Exception e) { + + HAConnection.log.error(this.getServiceName() + " service has exception.", e); + break; + } + } + + + if (this.selectMappedBufferResult != null) { + this.selectMappedBufferResult.release(); + } + + this.makeStop(); + + readSocketService.makeStop(); + + + haService.removeConnection(HAConnection.this); + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + try { + this.selector.close(); + this.socketChannel.close(); + } catch (IOException e) { + HAConnection.log.error("", e); + } + + HAConnection.log.info(this.getServiceName() + " service end"); + } + + + /** + + */ + private boolean transferData() throws Exception { + int writeSizeZeroTimes = 0; + // Write Header + while (this.byteBufferHeader.hasRemaining()) { + int writeSize = this.socketChannel.write(this.byteBufferHeader); + if (writeSize > 0) { + writeSizeZeroTimes = 0; + this.lastWriteTimestamp = HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + } else if (writeSize == 0) { + if (++writeSizeZeroTimes >= 3) { + break; + } + } else { + throw new Exception("ha master write header error < 0"); + } + } + + if (null == this.selectMappedBufferResult) { + return !this.byteBufferHeader.hasRemaining(); + } + + writeSizeZeroTimes = 0; + + // Write Body + if (!this.byteBufferHeader.hasRemaining()) { + while (this.selectMappedBufferResult.getByteBuffer().hasRemaining()) { + int writeSize = this.socketChannel.write(this.selectMappedBufferResult.getByteBuffer()); + if (writeSize > 0) { + writeSizeZeroTimes = 0; + this.lastWriteTimestamp = HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + } else if (writeSize == 0) { + if (++writeSizeZeroTimes >= 3) { + break; + } + } else { + throw new Exception("ha master write body error < 0"); + } + } + } + + boolean result = !this.byteBufferHeader.hasRemaining() && !this.selectMappedBufferResult.getByteBuffer().hasRemaining(); + + if (!this.selectMappedBufferResult.getByteBuffer().hasRemaining()) { + this.selectMappedBufferResult.release(); + this.selectMappedBufferResult = null; + } + + return result; + } + + + @Override + public String getServiceName() { + return WriteSocketService.class.getSimpleName(); + } + + + @Override + public void shutdown() { + super.shutdown(); + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java new file mode 100644 index 0000000000..00f9833750 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java @@ -0,0 +1,654 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.store.CommitLog.GroupCommitRequest; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.*; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + + +/** + * @author shijia.wxr + */ +public class HAService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + + private final AtomicInteger connectionCount = new AtomicInteger(0); + + private final List connectionList = new LinkedList(); + + private final AcceptSocketService acceptSocketService; + + private final DefaultMessageStore defaultMessageStore; + + private final WaitNotifyObject waitNotifyObject = new WaitNotifyObject(); + private final AtomicLong push2SlaveMaxOffset = new AtomicLong(0); + + private final GroupTransferService groupTransferService; + + private final HAClient haClient; + + + public HAService(final DefaultMessageStore defaultMessageStore) throws IOException { + this.defaultMessageStore = defaultMessageStore; + this.acceptSocketService = + new AcceptSocketService(defaultMessageStore.getMessageStoreConfig().getHaListenPort()); + this.groupTransferService = new GroupTransferService(); + this.haClient = new HAClient(); + } + + + public void updateMasterAddress(final String newAddr) { + if (this.haClient != null) { + this.haClient.updateMasterAddress(newAddr); + } + } + + + public void putRequest(final GroupCommitRequest request) { + this.groupTransferService.putRequest(request); + } + + + public boolean isSlaveOK(final long masterPutWhere) { + boolean result = this.connectionCount.get() > 0; + result = + result + && ((masterPutWhere - this.push2SlaveMaxOffset.get()) < this.defaultMessageStore + .getMessageStoreConfig().getHaSlaveFallbehindMax()); + return result; + } + + + /** + + */ + public void notifyTransferSome(final long offset) { + for (long value = this.push2SlaveMaxOffset.get(); offset > value; ) { + boolean ok = this.push2SlaveMaxOffset.compareAndSet(value, offset); + if (ok) { + this.groupTransferService.notifyTransferSome(); + break; + } else { + value = this.push2SlaveMaxOffset.get(); + } + } + } + + + public AtomicInteger getConnectionCount() { + return connectionCount; + } + + + // public void notifyTransferSome() { + // this.groupTransferService.notifyTransferSome(); + // } + + public void start() { + this.acceptSocketService.beginAccept(); + this.acceptSocketService.start(); + this.groupTransferService.start(); + this.haClient.start(); + } + + + public void addConnection(final HAConnection conn) { + synchronized (this.connectionList) { + this.connectionList.add(conn); + } + } + + + public void removeConnection(final HAConnection conn) { + synchronized (this.connectionList) { + this.connectionList.remove(conn); + } + } + + + public void shutdown() { + this.haClient.shutdown(); + this.acceptSocketService.shutdown(true); + this.destroyConnections(); + this.groupTransferService.shutdown(); + } + + + public void destroyConnections() { + synchronized (this.connectionList) { + for (HAConnection c : this.connectionList) { + c.shutdown(); + } + + this.connectionList.clear(); + } + } + + + public DefaultMessageStore getDefaultMessageStore() { + return defaultMessageStore; + } + + + public WaitNotifyObject getWaitNotifyObject() { + return waitNotifyObject; + } + + public AtomicLong getPush2SlaveMaxOffset() { + return push2SlaveMaxOffset; + } + + class AcceptSocketService extends ServiceThread { + private ServerSocketChannel serverSocketChannel; + private Selector selector; + private SocketAddress socketAddressListen; + + + public AcceptSocketService(final int port) { + this.socketAddressListen = new InetSocketAddress(port); + } + + + public void beginAccept() { + try { + this.serverSocketChannel = ServerSocketChannel.open(); + this.selector = RemotingUtil.openSelector(); + this.serverSocketChannel.socket().setReuseAddress(true); + this.serverSocketChannel.socket().bind(this.socketAddressListen); + this.serverSocketChannel.configureBlocking(false); + this.serverSocketChannel.register(this.selector, SelectionKey.OP_ACCEPT); + } catch (Exception e) { + log.error("beginAccept exception", e); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.selector.select(1000); + Set selected = this.selector.selectedKeys(); + if (selected != null) { + for (SelectionKey k : selected) { + if ((k.readyOps() & SelectionKey.OP_ACCEPT) != 0) { + SocketChannel sc = ((ServerSocketChannel) k.channel()).accept(); + if (sc != null) { + HAService.log.info("HAService receive new connection, " + + sc.socket().getRemoteSocketAddress()); + + try { + HAConnection conn = new HAConnection(HAService.this, sc); + conn.start(); + HAService.this.addConnection(conn); + } catch (Exception e) { + log.error("new HAConnection exception", e); + sc.close(); + } + } + } else { + log.warn("Unexpected ops in select " + k.readyOps()); + } + } + + selected.clear(); + } + + } catch (Exception e) { + log.error(this.getServiceName() + " service has exception.", e); + } + } + + log.error(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return AcceptSocketService.class.getSimpleName(); + } + } + + /** + * GroupTransferService Service + */ + class GroupTransferService extends ServiceThread { + + private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject(); + private volatile List requestsWrite = new ArrayList(); + private volatile List requestsRead = new ArrayList(); + + + public void putRequest(final GroupCommitRequest request) { + synchronized (this) { + this.requestsWrite.add(request); + if (hasNotified.compareAndSet(false, true)) { + waitPoint.countDown(); // notify + } + } + } + + + public void notifyTransferSome() { + this.notifyTransferObject.wakeup(); + } + + + private void swapRequests() { + List tmp = this.requestsWrite; + this.requestsWrite = this.requestsRead; + this.requestsRead = tmp; + } + + + private void doWaitTransfer() { + if (!this.requestsRead.isEmpty()) { + for (GroupCommitRequest req : this.requestsRead) { + boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); + for (int i = 0; !transferOK && i < 5; i++) { + this.notifyTransferObject.waitForRunning(1000); + transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); + } + + if (!transferOK) { + log.warn("transfer messsage to slave timeout, " + req.getNextOffset()); + } + + req.wakeupCustomer(transferOK); + } + + this.requestsRead.clear(); + } + } + + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + this.waitForRunning(0); + this.doWaitTransfer(); + } catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + protected void onWaitEnd() { + this.swapRequests(); + } + + + @Override + public String getServiceName() { + return GroupTransferService.class.getSimpleName(); + } + } + + class HAClient extends ServiceThread { + private static final int READ_MAX_BUFFER_SIZE = 1024 * 1024 * 4; + private final AtomicReference masterAddress = new AtomicReference(); + private final ByteBuffer reportOffset = ByteBuffer.allocate(8); + private SocketChannel socketChannel; + private Selector selector; + private long lastWriteTimestamp = System.currentTimeMillis(); + + private long currentReportedOffset = 0; + private int dispatchPostion = 0; + private ByteBuffer byteBufferRead = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE); + private ByteBuffer byteBufferBackup = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE); + + + public HAClient() throws IOException { + this.selector = RemotingUtil.openSelector(); + } + + + public void updateMasterAddress(final String newAddr) { + String currentAddr = this.masterAddress.get(); + if (currentAddr == null || !currentAddr.equals(newAddr)) { + this.masterAddress.set(newAddr); + log.info("update master address, OLD: " + currentAddr + " NEW: " + newAddr); + } + } + + + private boolean isTimeToReportOffset() { + long interval = + HAService.this.defaultMessageStore.getSystemClock().now() - this.lastWriteTimestamp; + boolean needHeart = interval > HAService.this.defaultMessageStore.getMessageStoreConfig() + .getHaSendHeartbeatInterval(); + + return needHeart; + } + + + private boolean reportSlaveMaxOffset(final long maxOffset) { + this.reportOffset.position(0); + this.reportOffset.limit(8); + this.reportOffset.putLong(maxOffset); + this.reportOffset.position(0); + this.reportOffset.limit(8); + + for (int i = 0; i < 3 && this.reportOffset.hasRemaining(); i++) { + try { + this.socketChannel.write(this.reportOffset); + } catch (IOException e) { + log.error(this.getServiceName() + + "reportSlaveMaxOffset this.socketChannel.write exception", e); + return false; + } + } + + return !this.reportOffset.hasRemaining(); + } + + + // private void reallocateByteBuffer() { + // ByteBuffer bb = ByteBuffer.allocate(READ_MAX_BUFFER_SIZE); + // int remain = this.byteBufferRead.limit() - this.dispatchPostion; + // bb.put(this.byteBufferRead.array(), this.dispatchPostion, remain); + // this.dispatchPostion = 0; + // this.byteBufferRead = bb; + // } + + /** + + */ + private void reallocateByteBuffer() { + int remain = READ_MAX_BUFFER_SIZE - this.dispatchPostion; + if (remain > 0) { + this.byteBufferRead.position(this.dispatchPostion); + + this.byteBufferBackup.position(0); + this.byteBufferBackup.limit(READ_MAX_BUFFER_SIZE); + this.byteBufferBackup.put(this.byteBufferRead); + } + + this.swapByteBuffer(); + + this.byteBufferRead.position(remain); + this.byteBufferRead.limit(READ_MAX_BUFFER_SIZE); + this.dispatchPostion = 0; + } + + + private void swapByteBuffer() { + ByteBuffer tmp = this.byteBufferRead; + this.byteBufferRead = this.byteBufferBackup; + this.byteBufferBackup = tmp; + } + + + private boolean processReadEvent() { + int readSizeZeroTimes = 0; + while (this.byteBufferRead.hasRemaining()) { + try { + int readSize = this.socketChannel.read(this.byteBufferRead); + if (readSize > 0) { + lastWriteTimestamp = HAService.this.defaultMessageStore.getSystemClock().now(); + readSizeZeroTimes = 0; + boolean result = this.dispatchReadRequest(); + if (!result) { + log.error("HAClient, dispatchReadRequest error"); + return false; + } + } else if (readSize == 0) { + if (++readSizeZeroTimes >= 3) { + break; + } + } else { + // TODO ERROR + log.info("HAClient, processReadEvent read socket < 0"); + return false; + } + } catch (IOException e) { + log.info("HAClient, processReadEvent read socket exception", e); + return false; + } + } + + return true; + } + + + private boolean dispatchReadRequest() { + final int msgHeaderSize = 8 + 4; // phyoffset + size + int readSocketPos = this.byteBufferRead.position(); + + while (true) { + int diff = this.byteBufferRead.position() - this.dispatchPostion; + if (diff >= msgHeaderSize) { + long masterPhyOffset = this.byteBufferRead.getLong(this.dispatchPostion); + int bodySize = this.byteBufferRead.getInt(this.dispatchPostion + 8); + + long slavePhyOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + + + if (slavePhyOffset != 0) { + if (slavePhyOffset != masterPhyOffset) { + log.error("master pushed offset not equal the max phy offset in slave, SLAVE: " + + slavePhyOffset + " MASTER: " + masterPhyOffset); + return false; + } + } + + + if (diff >= (msgHeaderSize + bodySize)) { + byte[] bodyData = new byte[bodySize]; + this.byteBufferRead.position(this.dispatchPostion + msgHeaderSize); + this.byteBufferRead.get(bodyData); + + + HAService.this.defaultMessageStore.appendToCommitLog(masterPhyOffset, bodyData); + + this.byteBufferRead.position(readSocketPos); + this.dispatchPostion += msgHeaderSize + bodySize; + + if (!reportSlaveMaxOffsetPlus()) { + return false; + } + + continue; + } + } + + if (!this.byteBufferRead.hasRemaining()) { + this.reallocateByteBuffer(); + } + + break; + } + + return true; + } + + + private boolean reportSlaveMaxOffsetPlus() { + boolean result = true; + long currentPhyOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + if (currentPhyOffset > this.currentReportedOffset) { + this.currentReportedOffset = currentPhyOffset; + result = this.reportSlaveMaxOffset(this.currentReportedOffset); + if (!result) { + this.closeMaster(); + log.error("HAClient, reportSlaveMaxOffset error, " + this.currentReportedOffset); + } + } + + return result; + } + + + private boolean connectMaster() throws ClosedChannelException { + if (null == socketChannel) { + String addr = this.masterAddress.get(); + if (addr != null) { + + SocketAddress socketAddress = RemotingUtil.string2SocketAddress(addr); + if (socketAddress != null) { + this.socketChannel = RemotingUtil.connect(socketAddress); + if (this.socketChannel != null) { + this.socketChannel.register(this.selector, SelectionKey.OP_READ); + } + } + } + + this.currentReportedOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + + this.lastWriteTimestamp = System.currentTimeMillis(); + } + + return this.socketChannel != null; + } + + + private void closeMaster() { + if (null != this.socketChannel) { + try { + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + this.socketChannel.close(); + + this.socketChannel = null; + } catch (IOException e) { + log.warn("closeMaster exception. ", e); + } + + this.lastWriteTimestamp = 0; + this.dispatchPostion = 0; + + this.byteBufferBackup.position(0); + this.byteBufferBackup.limit(READ_MAX_BUFFER_SIZE); + + this.byteBufferRead.position(0); + this.byteBufferRead.limit(READ_MAX_BUFFER_SIZE); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStopped()) { + try { + if (this.connectMaster()) { + + if (this.isTimeToReportOffset()) { + boolean result = this.reportSlaveMaxOffset(this.currentReportedOffset); + if (!result) { + this.closeMaster(); + } + } + + + this.selector.select(1000); + + + boolean ok = this.processReadEvent(); + if (!ok) { + this.closeMaster(); + } + + if (!reportSlaveMaxOffsetPlus()) { + continue; + } + + + long interval = + HAService.this.getDefaultMessageStore().getSystemClock().now() + - this.lastWriteTimestamp; + if (interval > HAService.this.getDefaultMessageStore().getMessageStoreConfig() + .getHaHousekeepingInterval()) { + log.warn("HAClient, housekeeping, found this connection[" + this.masterAddress + + "] expired, " + interval); + this.closeMaster(); + log.warn("HAClient, master not response some time, so close connection"); + } + } else { + this.waitForRunning(1000 * 5); + } + } catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + this.waitForRunning(1000 * 5); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + // + // private void disableWriteFlag() { + // if (this.socketChannel != null) { + // SelectionKey sk = this.socketChannel.keyFor(this.selector); + // if (sk != null) { + // int ops = sk.interestOps(); + // ops &= ~SelectionKey.OP_WRITE; + // sk.interestOps(ops); + // } + // } + // } + // + // + // private void enableWriteFlag() { + // if (this.socketChannel != null) { + // SelectionKey sk = this.socketChannel.keyFor(this.selector); + // if (sk != null) { + // int ops = sk.interestOps(); + // ops |= SelectionKey.OP_WRITE; + // sk.interestOps(ops); + // } + // } + // } + + @Override + public String getServiceName() { + return HAClient.class.getSimpleName(); + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java new file mode 100644 index 0000000000..f540cdb914 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import java.util.HashMap; + + +/** + * @author shijia.wxr + */ +public class WaitNotifyObject { + + protected final HashMap waitingThreadTable = + new HashMap(16); + + protected volatile boolean hasNotified = false; + + + public void wakeup() { + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + + protected void waitForRunning(long interval) { + synchronized (this) { + if (this.hasNotified) { + this.hasNotified = false; + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + this.hasNotified = false; + this.onWaitEnd(); + } + } + } + + + protected void onWaitEnd() { + } + + public void wakeupAll() { + synchronized (this) { + boolean needNotify = false; + + for (Boolean value : this.waitingThreadTable.values()) { + needNotify = needNotify || !value; + value = true; + } + + if (needNotify) { + this.notifyAll(); + } + } + } + + public void allWaitForRunning(long interval) { + long currentThreadId = Thread.currentThread().getId(); + synchronized (this) { + Boolean notified = this.waitingThreadTable.get(currentThreadId); + if (notified != null && notified) { + this.waitingThreadTable.put(currentThreadId, false); + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + this.waitingThreadTable.put(currentThreadId, false); + this.onWaitEnd(); + } + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java new file mode 100644 index 0000000000..f35332014e --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java @@ -0,0 +1,290 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.MappedFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class IndexFile { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static int hashSlotSize = 4; + private static int indexSize = 20; + private static int invalidIndex = 0; + private final int hashSlotNum; + private final int indexNum; + private final MappedFile mappedFile; + private final FileChannel fileChannel; + private final MappedByteBuffer mappedByteBuffer; + private final IndexHeader indexHeader; + + + public IndexFile(final String fileName, final int hashSlotNum, final int indexNum, + final long endPhyOffset, final long endTimestamp) throws IOException { + int fileTotalSize = + IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * hashSlotSize) + (indexNum * indexSize); + this.mappedFile = new MappedFile(fileName, fileTotalSize); + this.fileChannel = this.mappedFile.getFileChannel(); + this.mappedByteBuffer = this.mappedFile.getMappedByteBuffer(); + this.hashSlotNum = hashSlotNum; + this.indexNum = indexNum; + + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + this.indexHeader = new IndexHeader(byteBuffer); + + if (endPhyOffset > 0) { + this.indexHeader.setBeginPhyOffset(endPhyOffset); + this.indexHeader.setEndPhyOffset(endPhyOffset); + } + + if (endTimestamp > 0) { + this.indexHeader.setBeginTimestamp(endTimestamp); + this.indexHeader.setEndTimestamp(endTimestamp); + } + } + + + public String getFileName() { + return this.mappedFile.getFileName(); + } + + + public void load() { + this.indexHeader.load(); + } + + + public void flush() { + long beginTime = System.currentTimeMillis(); + if (this.mappedFile.hold()) { + this.indexHeader.updateByteBuffer(); + this.mappedByteBuffer.force(); + this.mappedFile.release(); + log.info("flush index file eclipse time(ms) " + (System.currentTimeMillis() - beginTime)); + } + } + + public boolean isWriteFull() { + return this.indexHeader.getIndexCount() >= this.indexNum; + } + + + public boolean destroy(final long intervalForcibly) { + return this.mappedFile.destroy(intervalForcibly); + } + + public boolean putKey(final String key, final long phyOffset, final long storeTimestamp) { + if (this.indexHeader.getIndexCount() < this.indexNum) { + int keyHash = indexKeyHashMethod(key); + int slotPos = keyHash % this.hashSlotNum; + int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * hashSlotSize; + + FileLock fileLock = null; + + try { + + // fileLock = this.fileChannel.lock(absSlotPos, hashSlotSize, + // false); + int slotValue = this.mappedByteBuffer.getInt(absSlotPos); + if (slotValue <= invalidIndex || slotValue > this.indexHeader.getIndexCount()) { + slotValue = invalidIndex; + } + + long timeDiff = storeTimestamp - this.indexHeader.getBeginTimestamp(); + + + timeDiff = timeDiff / 1000; + + + if (this.indexHeader.getBeginTimestamp() <= 0) { + timeDiff = 0; + } else if (timeDiff > Integer.MAX_VALUE) { + timeDiff = Integer.MAX_VALUE; + } else if (timeDiff < 0) { + timeDiff = 0; + } + + int absIndexPos = + IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * hashSlotSize + + this.indexHeader.getIndexCount() * indexSize; + + + this.mappedByteBuffer.putInt(absIndexPos, keyHash); + this.mappedByteBuffer.putLong(absIndexPos + 4, phyOffset); + this.mappedByteBuffer.putInt(absIndexPos + 4 + 8, (int) timeDiff); + this.mappedByteBuffer.putInt(absIndexPos + 4 + 8 + 4, slotValue); + + + this.mappedByteBuffer.putInt(absSlotPos, this.indexHeader.getIndexCount()); + + + if (this.indexHeader.getIndexCount() <= 1) { + this.indexHeader.setBeginPhyOffset(phyOffset); + this.indexHeader.setBeginTimestamp(storeTimestamp); + } + + this.indexHeader.incHashSlotCount(); + this.indexHeader.incIndexCount(); + this.indexHeader.setEndPhyOffset(phyOffset); + this.indexHeader.setEndTimestamp(storeTimestamp); + + return true; + } catch (Exception e) { + log.error("putKey exception, Key: " + key + " KeyHashCode: " + key.hashCode(), e); + } finally { + if (fileLock != null) { + try { + fileLock.release(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } else { + log.warn("putKey index count " + this.indexHeader.getIndexCount() + " index max num " + + this.indexNum); + } + + return false; + } + + public int indexKeyHashMethod(final String key) { + int keyHash = key.hashCode(); + int keyHashPositive = Math.abs(keyHash); + if (keyHashPositive < 0) + keyHashPositive = 0; + return keyHashPositive; + } + + public long getBeginTimestamp() { + return this.indexHeader.getBeginTimestamp(); + } + + public long getEndTimestamp() { + return this.indexHeader.getEndTimestamp(); + } + + public long getEndPhyOffset() { + return this.indexHeader.getEndPhyOffset(); + } + + public boolean isTimeMatched(final long begin, final long end) { + boolean result = + begin < this.indexHeader.getBeginTimestamp() && end > this.indexHeader.getEndTimestamp(); + + result = + result + || (begin >= this.indexHeader.getBeginTimestamp() && begin <= this.indexHeader + .getEndTimestamp()); + + result = + result + || (end >= this.indexHeader.getBeginTimestamp() && end <= this.indexHeader + .getEndTimestamp()); + return result; + } + + public void selectPhyOffset(final List phyOffsets, final String key, final int maxNum, + final long begin, final long end, boolean lock) { + if (this.mappedFile.hold()) { + int keyHash = indexKeyHashMethod(key); + int slotPos = keyHash % this.hashSlotNum; + int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * hashSlotSize; + + FileLock fileLock = null; + try { + if (lock) { + // fileLock = this.fileChannel.lock(absSlotPos, + // hashSlotSize, true); + } + + int slotValue = this.mappedByteBuffer.getInt(absSlotPos); + // if (fileLock != null) { + // fileLock.release(); + // fileLock = null; + // } + + if (slotValue <= invalidIndex || slotValue > this.indexHeader.getIndexCount() + || this.indexHeader.getIndexCount() <= 1) { + // TODO NOTFOUND + } else { + for (int nextIndexToRead = slotValue;;) { + if (phyOffsets.size() >= maxNum) { + break; + } + + int absIndexPos = + IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * hashSlotSize + + nextIndexToRead * indexSize; + + int keyHashRead = this.mappedByteBuffer.getInt(absIndexPos); + long phyOffsetRead = this.mappedByteBuffer.getLong(absIndexPos + 4); + + long timeDiff = (long) this.mappedByteBuffer.getInt(absIndexPos + 4 + 8); + int prevIndexRead = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8 + 4); + + + if (timeDiff < 0) { + break; + } + + timeDiff *= 1000L; + + long timeRead = this.indexHeader.getBeginTimestamp() + timeDiff; + boolean timeMatched = (timeRead >= begin) && (timeRead <= end); + + if (keyHash == keyHashRead && timeMatched) { + phyOffsets.add(phyOffsetRead); + } + + if (prevIndexRead <= invalidIndex + || prevIndexRead > this.indexHeader.getIndexCount() + || prevIndexRead == nextIndexToRead || timeRead < begin) { + break; + } + + nextIndexToRead = prevIndexRead; + } + } + } catch (Exception e) { + log.error("selectPhyOffset exception ", e); + } finally { + if (fileLock != null) { + try { + fileLock.release(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + this.mappedFile.release(); + } + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java new file mode 100644 index 0000000000..d6015e38ec --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + + +/** + + * + * @author shijia.wxr + * + */ +public class IndexHeader { + public static final int INDEX_HEADER_SIZE = 40; + private static int beginTimestampIndex = 0; + private static int endTimestampIndex = 8; + private static int beginPhyoffsetIndex = 16; + private static int endPhyoffsetIndex = 24; + private static int hashSlotcountIndex = 32; + private static int indexCountIndex = 36; + private final ByteBuffer byteBuffer; + private AtomicLong beginTimestamp = new AtomicLong(0); + private AtomicLong endTimestamp = new AtomicLong(0); + private AtomicLong beginPhyOffset = new AtomicLong(0); + private AtomicLong endPhyOffset = new AtomicLong(0); + private AtomicInteger hashSlotCount = new AtomicInteger(0); + + private AtomicInteger indexCount = new AtomicInteger(1); + + + public IndexHeader(final ByteBuffer byteBuffer) { + this.byteBuffer = byteBuffer; + } + + + public void load() { + this.beginTimestamp.set(byteBuffer.getLong(beginTimestampIndex)); + this.endTimestamp.set(byteBuffer.getLong(endTimestampIndex)); + this.beginPhyOffset.set(byteBuffer.getLong(beginPhyoffsetIndex)); + this.endPhyOffset.set(byteBuffer.getLong(endPhyoffsetIndex)); + + this.hashSlotCount.set(byteBuffer.getInt(hashSlotcountIndex)); + this.indexCount.set(byteBuffer.getInt(indexCountIndex)); + + if (this.indexCount.get() <= 0) { + this.indexCount.set(1); + } + } + + public void updateByteBuffer() { + this.byteBuffer.putLong(beginTimestampIndex, this.beginTimestamp.get()); + this.byteBuffer.putLong(endTimestampIndex, this.endTimestamp.get()); + this.byteBuffer.putLong(beginPhyoffsetIndex, this.beginPhyOffset.get()); + this.byteBuffer.putLong(endPhyoffsetIndex, this.endPhyOffset.get()); + this.byteBuffer.putInt(hashSlotcountIndex, this.hashSlotCount.get()); + this.byteBuffer.putInt(indexCountIndex, this.indexCount.get()); + } + + + public long getBeginTimestamp() { + return beginTimestamp.get(); + } + + + public void setBeginTimestamp(long beginTimestamp) { + this.beginTimestamp.set(beginTimestamp); + this.byteBuffer.putLong(beginTimestampIndex, beginTimestamp); + } + + + public long getEndTimestamp() { + return endTimestamp.get(); + } + + + public void setEndTimestamp(long endTimestamp) { + this.endTimestamp.set(endTimestamp); + this.byteBuffer.putLong(endTimestampIndex, endTimestamp); + } + + + public long getBeginPhyOffset() { + return beginPhyOffset.get(); + } + + + public void setBeginPhyOffset(long beginPhyOffset) { + this.beginPhyOffset.set(beginPhyOffset); + this.byteBuffer.putLong(beginPhyoffsetIndex, beginPhyOffset); + } + + + public long getEndPhyOffset() { + return endPhyOffset.get(); + } + + + public void setEndPhyOffset(long endPhyOffset) { + this.endPhyOffset.set(endPhyOffset); + this.byteBuffer.putLong(endPhyoffsetIndex, endPhyOffset); + } + + + public AtomicInteger getHashSlotCount() { + return hashSlotCount; + } + + + public void incHashSlotCount() { + int value = this.hashSlotCount.incrementAndGet(); + this.byteBuffer.putInt(hashSlotcountIndex, value); + } + + + public int getIndexCount() { + return indexCount.get(); + } + + + public void incIndexCount() { + int value = this.indexCount.incrementAndGet(); + this.byteBuffer.putInt(indexCountIndex, value); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java new file mode 100644 index 0000000000..f275f80875 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java @@ -0,0 +1,389 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.DispatchRequest; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * @author shijia.wxr + */ +public class IndexService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private final DefaultMessageStore defaultMessageStore; + + private final int hashSlotNum; + private final int indexNum; + private final String storePath; + + private final ArrayList indexFileList = new ArrayList(); + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + + + public IndexService(final DefaultMessageStore store) { + this.defaultMessageStore = store; + this.hashSlotNum = store.getMessageStoreConfig().getMaxHashSlotNum(); + this.indexNum = store.getMessageStoreConfig().getMaxIndexNum(); + this.storePath = + StorePathConfigHelper.getStorePathIndex(store.getMessageStoreConfig().getStorePathRootDir()); + } + + + public boolean load(final boolean lastExitOK) { + File dir = new File(this.storePath); + File[] files = dir.listFiles(); + if (files != null) { + // ascending order + Arrays.sort(files); + for (File file : files) { + try { + IndexFile f = new IndexFile(file.getPath(), this.hashSlotNum, this.indexNum, 0, 0); + f.load(); + + if (!lastExitOK) { + if (f.getEndTimestamp() > this.defaultMessageStore.getStoreCheckpoint() + .getIndexMsgTimestamp()) { + f.destroy(0); + continue; + } + } + + log.info("load index file OK, " + f.getFileName()); + this.indexFileList.add(f); + } catch (IOException e) { + log.error("load file " + file + " error", e); + return false; + } catch (NumberFormatException e) { + continue; + } + } + } + + return true; + } + + public void deleteExpiredFile(long offset) { + Object[] files = null; + try { + this.readWriteLock.readLock().lock(); + if (this.indexFileList.isEmpty()) { + return; + } + + long endPhyOffset = this.indexFileList.get(0).getEndPhyOffset(); + if (endPhyOffset < offset) { + files = this.indexFileList.toArray(); + } + } catch (Exception e) { + log.error("destroy exception", e); + } finally { + this.readWriteLock.readLock().unlock(); + } + + if (files != null) { + List fileList = new ArrayList(); + for (int i = 0; i < (files.length - 1); i++) { + IndexFile f = (IndexFile) files[i]; + if (f.getEndPhyOffset() < offset) { + fileList.add(f); + } else { + break; + } + } + + this.deleteExpiredFile(fileList); + } + } + + private void deleteExpiredFile(List files) { + if (!files.isEmpty()) { + try { + this.readWriteLock.writeLock().lock(); + for (IndexFile file : files) { + boolean destroyed = file.destroy(3000); + destroyed = destroyed && this.indexFileList.remove(file); + if (!destroyed) { + log.error("deleteExpiredFile remove failed."); + break; + } + } + } catch (Exception e) { + log.error("deleteExpiredFile has exception.", e); + } finally { + this.readWriteLock.writeLock().unlock(); + } + } + } + + + public void destroy() { + try { + this.readWriteLock.readLock().lock(); + for (IndexFile f : this.indexFileList) { + f.destroy(1000 * 3); + } + this.indexFileList.clear(); + } catch (Exception e) { + log.error("destroy exception", e); + } finally { + this.readWriteLock.readLock().unlock(); + } + } + + + public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long begin, long end) { + List phyOffsets = new ArrayList(maxNum); + + long indexLastUpdateTimestamp = 0; + long indexLastUpdatePhyoffset = 0; + maxNum = Math.min(maxNum, this.defaultMessageStore.getMessageStoreConfig().getMaxMsgsNumBatch()); + try { + this.readWriteLock.readLock().lock(); + if (!this.indexFileList.isEmpty()) { + for (int i = this.indexFileList.size(); i > 0; i--) { + IndexFile f = this.indexFileList.get(i - 1); + boolean lastFile = i == this.indexFileList.size(); + if (lastFile) { + indexLastUpdateTimestamp = f.getEndTimestamp(); + indexLastUpdatePhyoffset = f.getEndPhyOffset(); + } + + if (f.isTimeMatched(begin, end)) { + + f.selectPhyOffset(phyOffsets, buildKey(topic, key), maxNum, begin, end, lastFile); + } + + + if (f.getBeginTimestamp() < begin) { + break; + } + + if (phyOffsets.size() >= maxNum) { + break; + } + } + } + } catch (Exception e) { + log.error("queryMsg exception", e); + } finally { + this.readWriteLock.readLock().unlock(); + } + + return new QueryOffsetResult(phyOffsets, indexLastUpdateTimestamp, indexLastUpdatePhyoffset); + } + + + private String buildKey(final String topic, final String key) { + return topic + "#" + key; + } + + + public void buildIndex(DispatchRequest req) { + IndexFile indexFile = retryGetAndCreateIndexFile(); + if (indexFile != null) { + long endPhyOffset = indexFile.getEndPhyOffset(); + DispatchRequest msg = req; + String topic = msg.getTopic(); + String keys = msg.getKeys(); + if (msg.getCommitLogOffset() < endPhyOffset) { + return; + } + + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + break; + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + return; + } + + if (req.getUniqKey() != null) { + indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey())); + if (indexFile == null) { + log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey()); + return; + } + } + + if (keys != null && keys.length() > 0) { + String[] keyset = keys.split(MessageConst.KEY_SEPARATOR); + for (int i = 0; i < keyset.length; i++) { + String key = keyset[i]; + if (key.length() > 0) { + indexFile = putKey(indexFile, msg, buildKey(topic, key)); + if (indexFile == null) { + log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey()); + return; + } + } + } + } + } else { + log.error("build index error, stop building index"); + } + } + + + private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) { + for (boolean ok = + indexFile.putKey(idxKey, msg.getCommitLogOffset(), + msg.getStoreTimestamp()); !ok; ) { + log.warn("index file full, so create another one, " + indexFile.getFileName()); + indexFile = retryGetAndCreateIndexFile(); + if (null == indexFile) { + return null; + } + + ok = + indexFile.putKey(idxKey, msg.getCommitLogOffset(), + msg.getStoreTimestamp()); + } + return indexFile; + } + + + public IndexFile retryGetAndCreateIndexFile() { + IndexFile indexFile = null; + + + for (int times = 0; null == indexFile && times < 3; times++) { + indexFile = this.getAndCreateLastIndexFile(); + if (null != indexFile) + break; + + try { + log.error("try to create index file, " + times + " times"); + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + if (null == indexFile) { + this.defaultMessageStore.getAccessRights().makeIndexFileError(); + log.error("mark index file can not build flag"); + } + + return indexFile; + } + + + public IndexFile getAndCreateLastIndexFile() { + IndexFile indexFile = null; + IndexFile prevIndexFile = null; + long lastUpdateEndPhyOffset = 0; + long lastUpdateIndexTimestamp = 0; + + { + this.readWriteLock.readLock().lock(); + if (!this.indexFileList.isEmpty()) { + IndexFile tmp = this.indexFileList.get(this.indexFileList.size() - 1); + if (!tmp.isWriteFull()) { + indexFile = tmp; + } else { + lastUpdateEndPhyOffset = tmp.getEndPhyOffset(); + lastUpdateIndexTimestamp = tmp.getEndTimestamp(); + prevIndexFile = tmp; + } + } + + this.readWriteLock.readLock().unlock(); + } + + + if (indexFile == null) { + try { + String fileName = + this.storePath + File.separator + + UtilAll.timeMillisToHumanString(System.currentTimeMillis()); + indexFile = + new IndexFile(fileName, this.hashSlotNum, this.indexNum, lastUpdateEndPhyOffset, + lastUpdateIndexTimestamp); + this.readWriteLock.writeLock().lock(); + this.indexFileList.add(indexFile); + } catch (Exception e) { + log.error("getLastIndexFile exception ", e); + } finally { + this.readWriteLock.writeLock().unlock(); + } + + + if (indexFile != null) { + final IndexFile flushThisFile = prevIndexFile; + Thread flushThread = new Thread(new Runnable() { + @Override + public void run() { + IndexService.this.flush(flushThisFile); + } + }, "FlushIndexFileThread"); + + flushThread.setDaemon(true); + flushThread.start(); + } + } + + return indexFile; + } + + + public void flush(final IndexFile f) { + if (null == f) + return; + + long indexMsgTimestamp = 0; + + if (f.isWriteFull()) { + indexMsgTimestamp = f.getEndTimestamp(); + } + + f.flush(); + + if (indexMsgTimestamp > 0) { + this.defaultMessageStore.getStoreCheckpoint().setIndexMsgTimestamp(indexMsgTimestamp); + this.defaultMessageStore.getStoreCheckpoint().flush(); + } + } + + + public void start() { + + } + + + public void shutdown() { + + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java new file mode 100644 index 0000000000..89d0755b58 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class QueryOffsetResult { + private final List phyOffsets; + private final long indexLastUpdateTimestamp; + private final long indexLastUpdatePhyoffset; + + + public QueryOffsetResult(List phyOffsets, long indexLastUpdateTimestamp, + long indexLastUpdatePhyoffset) { + this.phyOffsets = phyOffsets; + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } + + + public List getPhyOffsets() { + return phyOffsets; + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java new file mode 100644 index 0000000000..5f4720d451 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.schedule; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + * + */ +public class DelayOffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(32); + + + public ConcurrentHashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java new file mode 100644 index 0000000000..e243a7e59a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java @@ -0,0 +1,386 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.schedule; + +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.running.RunningStats; +import com.alibaba.rocketmq.store.*; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * @author shijia.wxr + * + */ +public class ScheduleMessageService extends ConfigManager { + public static final String SCHEDULE_TOPIC = "SCHEDULE_TOPIC_XXXX"; + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + private static final long FIRST_DELAY_TIME = 1000L; + private static final long DELAY_FOR_A_WHILE = 100L; + private static final long DELAY_FOR_A_PERIOD = 10000L; + + private final ConcurrentHashMap delayLevelTable = + new ConcurrentHashMap(32); + + private final ConcurrentHashMap offsetTable = + new ConcurrentHashMap(32); + + private final Timer timer = new Timer("ScheduleMessageTimerThread", true); + + private final DefaultMessageStore defaultMessageStore; + + private int maxDelayLevel; + + + public ScheduleMessageService(final DefaultMessageStore defaultMessageStore) { + this.defaultMessageStore = defaultMessageStore; + } + + public static int queueId2DelayLevel(final int queueId) { + return queueId + 1; + } + + public void buildRunningStats(HashMap stats) { + Iterator> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + int queueId = delayLevel2QueueId(next.getKey()); + long delayOffset = next.getValue(); + long maxOffset = this.defaultMessageStore.getMaxOffsetInQuque(SCHEDULE_TOPIC, queueId); + String value = String.format("%d,%d", delayOffset, maxOffset); + String key = String.format("%s_%d", RunningStats.scheduleMessageOffset.name(), next.getKey()); + stats.put(key, value); + } + } + + public static int delayLevel2QueueId(final int delayLevel) { + return delayLevel - 1; + } + + + private void updateOffset(int delayLevel, long offset) { + this.offsetTable.put(delayLevel, offset); + } + + + public long computeDeliverTimestamp(final int delayLevel, final long storeTimestamp) { + Long time = this.delayLevelTable.get(delayLevel); + if (time != null) { + return time + storeTimestamp; + } + + return storeTimestamp + 1000; + } + + + public void start() { + + for (Map.Entry entry : this.delayLevelTable.entrySet()) { + Integer level = entry.getKey(); + Long timeDelay = entry.getValue(); + Long offset = this.offsetTable.get(level); + if (null == offset) { + offset = 0L; + } + + if (timeDelay != null) { + this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME); + } + } + + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + ScheduleMessageService.this.persist(); + } catch (Exception e) { + log.error("scheduleAtFixedRate flush exception", e); + } + } + }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval()); + } + + + public void shutdown() { + this.timer.cancel(); + } + + + public int getMaxDelayLevel() { + return maxDelayLevel; + } + + + public String encode() { + return this.encode(false); + } + + public boolean load() { + boolean result = super.load(); + result = result && this.parseDelayLevel(); + return result; + } + + @Override + public String configFilePath() { + return StorePathConfigHelper.getDelayOffsetStorePath(this.defaultMessageStore.getMessageStoreConfig() + .getStorePathRootDir()); + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = + DelayOffsetSerializeWrapper.fromJson(jsonString, DelayOffsetSerializeWrapper.class); + if (delayOffsetSerializeWrapper != null) { + this.offsetTable.putAll(delayOffsetSerializeWrapper.getOffsetTable()); + } + } + } + + public String encode(final boolean prettyFormat) { + DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = new DelayOffsetSerializeWrapper(); + delayOffsetSerializeWrapper.setOffsetTable(this.offsetTable); + return delayOffsetSerializeWrapper.toJson(prettyFormat); + } + + public boolean parseDelayLevel() { + HashMap timeUnitTable = new HashMap(); + timeUnitTable.put("s", 1000L); + timeUnitTable.put("m", 1000L * 60); + timeUnitTable.put("h", 1000L * 60 * 60); + timeUnitTable.put("d", 1000L * 60 * 60 * 24); + + String levelString = this.defaultMessageStore.getMessageStoreConfig().getMessageDelayLevel(); + try { + String[] levelArray = levelString.split(" "); + for (int i = 0; i < levelArray.length; i++) { + String value = levelArray[i]; + String ch = value.substring(value.length() - 1); + Long tu = timeUnitTable.get(ch); + + int level = i + 1; + if (level > this.maxDelayLevel) { + this.maxDelayLevel = level; + } + long num = Long.parseLong(value.substring(0, value.length() - 1)); + long delayTimeMillis = tu * num; + this.delayLevelTable.put(level, delayTimeMillis); + } + } catch (Exception e) { + log.error("parseDelayLevel exception", e); + log.info("levelString String = {}", levelString); + return false; + } + + return true; + } + + class DeliverDelayedMessageTimerTask extends TimerTask { + private final int delayLevel; + private final long offset; + + + public DeliverDelayedMessageTimerTask(int delayLevel, long offset) { + this.delayLevel = delayLevel; + this.offset = offset; + } + + + @Override + public void run() { + try { + this.executeOnTimeup(); + } catch (Exception e) { + // XXX: warn and notify me + log.error("ScheduleMessageService, executeOnTimeup exception", e); + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask( + this.delayLevel, this.offset), DELAY_FOR_A_PERIOD); + } + } + + + /** + + * + * @return + */ + private long correctDeliverTimestamp(final long now, final long deliverTimestamp) { + + long result = deliverTimestamp; + + long maxTimestamp = now + ScheduleMessageService.this.delayLevelTable.get(this.delayLevel); + if (deliverTimestamp > maxTimestamp) { + result = now; + } + + return result; + } + + + public void executeOnTimeup() { + ConsumeQueue cq = + ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(SCHEDULE_TOPIC, + delayLevel2QueueId(delayLevel)); + + long failScheduleOffset = offset; + + if (cq != null) { + SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(this.offset); + if (bufferCQ != null) { + try { + long nextOffset = offset; + int i = 0; + for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { + long offsetPy = bufferCQ.getByteBuffer().getLong(); + int sizePy = bufferCQ.getByteBuffer().getInt(); + long tagsCode = bufferCQ.getByteBuffer().getLong(); + + + long now = System.currentTimeMillis(); + long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode); + + nextOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + + long countdown = deliverTimestamp - now; + + if (countdown <= 0) { + MessageExt msgExt = + ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset( + offsetPy, sizePy); + + if (msgExt != null) { + try { + MessageExtBrokerInner msgInner = this.messageTimeup(msgExt); + PutMessageResult putMessageResult = + ScheduleMessageService.this.defaultMessageStore + .putMessage(msgInner); + + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + continue; + } + + else { + // XXX: warn and notify me + log.error( + "ScheduleMessageService, a message time up, but reput it failed, topic: {} msgId {}", + msgExt.getTopic(), msgExt.getMsgId()); + ScheduleMessageService.this.timer.schedule( + new DeliverDelayedMessageTimerTask(this.delayLevel, + nextOffset), DELAY_FOR_A_PERIOD); + ScheduleMessageService.this.updateOffset(this.delayLevel, + nextOffset); + return; + } + } catch (Exception e) { + /* + * XXX: warn and notify me + + + + */ + log.error( + "ScheduleMessageService, messageTimeup execute error, drop it. msgExt=" + + msgExt + ", nextOffset=" + nextOffset + ",offsetPy=" + + offsetPy + ",sizePy=" + sizePy, e); + } + } + } + + else { + ScheduleMessageService.this.timer.schedule( + new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), + countdown); + ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset); + return; + } + } // end of for + + nextOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask( + this.delayLevel, nextOffset), DELAY_FOR_A_WHILE); + ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset); + return; + } finally { + + bufferCQ.release(); + } + } // end of if (bufferCQ != null) + else { + /* + + + */ + long cqMinOffset = cq.getMinOffsetInQuque(); + if (offset < cqMinOffset) { + failScheduleOffset = cqMinOffset; + log.error("schedule CQ offset invalid. offset=" + offset + ", cqMinOffset=" + + cqMinOffset + ", queueId=" + cq.getQueueId()); + } + } + } // end of if (cq != null) + + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(this.delayLevel, + failScheduleOffset), DELAY_FOR_A_WHILE); + } + + + private MessageExtBrokerInner messageTimeup(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + + TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag()); + long tagsCodeValue = + MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + msgInner.setTagsCode(tagsCodeValue); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + + msgInner.setWaitStoreMsgOK(false); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + + msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC)); + + String queueIdStr = msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID); + int queueId = Integer.parseInt(queueIdStr); + msgInner.setQueueId(queueId); + + return msgInner; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java new file mode 100644 index 0000000000..539d4bed5a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.stats; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author shijia.wxr + */ +public class BrokerStats { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private final DefaultMessageStore defaultMessageStore; + + private volatile long msgPutTotalYesterdayMorning; + + private volatile long msgPutTotalTodayMorning; + + private volatile long msgGetTotalYesterdayMorning; + + private volatile long msgGetTotalTodayMorning; + + + public BrokerStats(DefaultMessageStore defaultMessageStore) { + this.defaultMessageStore = defaultMessageStore; + } + + + /** + + */ + public void record() { + this.msgPutTotalYesterdayMorning = this.msgPutTotalTodayMorning; + this.msgGetTotalYesterdayMorning = this.msgGetTotalTodayMorning; + + this.msgPutTotalTodayMorning = + this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal(); + this.msgGetTotalTodayMorning = + this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get(); + + log.info("yesterday put message total: {}", msgPutTotalTodayMorning - msgPutTotalYesterdayMorning); + log.info("yesterday get message total: {}", msgGetTotalTodayMorning - msgGetTotalYesterdayMorning); + } + + + public long getMsgPutTotalYesterdayMorning() { + return msgPutTotalYesterdayMorning; + } + + + public void setMsgPutTotalYesterdayMorning(long msgPutTotalYesterdayMorning) { + this.msgPutTotalYesterdayMorning = msgPutTotalYesterdayMorning; + } + + + public long getMsgPutTotalTodayMorning() { + return msgPutTotalTodayMorning; + } + + + public void setMsgPutTotalTodayMorning(long msgPutTotalTodayMorning) { + this.msgPutTotalTodayMorning = msgPutTotalTodayMorning; + } + + + public long getMsgGetTotalYesterdayMorning() { + return msgGetTotalYesterdayMorning; + } + + + public void setMsgGetTotalYesterdayMorning(long msgGetTotalYesterdayMorning) { + this.msgGetTotalYesterdayMorning = msgGetTotalYesterdayMorning; + } + + + public long getMsgGetTotalTodayMorning() { + return msgGetTotalTodayMorning; + } + + + public void setMsgGetTotalTodayMorning(long msgGetTotalTodayMorning) { + this.msgGetTotalTodayMorning = msgGetTotalTodayMorning; + } + + + public long getMsgPutTotalTodayNow() { + return this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal(); + } + + + public long getMsgGetTotalTodayNow() { + return this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get(); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java new file mode 100644 index 0000000000..60b3fff686 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.stats; + +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.stats.MomentStatsItemSet; +import com.alibaba.rocketmq.common.stats.StatsItem; +import com.alibaba.rocketmq.common.stats.StatsItemSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + + +public class BrokerStatsManager { + + public static final String TOPIC_PUT_NUMS = "TOPIC_PUT_NUMS"; + public static final String TOPIC_PUT_SIZE = "TOPIC_PUT_SIZE"; + public static final String GROUP_GET_NUMS = "GROUP_GET_NUMS"; + public static final String GROUP_GET_SIZE = "GROUP_GET_SIZE"; + public static final String SNDBCK_PUT_NUMS = "SNDBCK_PUT_NUMS"; + public static final String BROKER_PUT_NUMS = "BROKER_PUT_NUMS"; + public static final String BROKER_GET_NUMS = "BROKER_GET_NUMS"; + public static final String GROUP_GET_FROM_DISK_NUMS = "GROUP_GET_FROM_DISK_NUMS"; + public static final String GROUP_GET_FROM_DISK_SIZE = "GROUP_GET_FROM_DISK_SIZE"; + public static final String BROKER_GET_FROM_DISK_NUMS = "BROKER_GET_FROM_DISK_NUMS"; + public static final String BROKER_GET_FROM_DISK_SIZE = "BROKER_GET_FROM_DISK_SIZE"; + // For commercial + public static final String COMMERCIAL_SEND_TIMES = "COMMERCIAL_SEND_TIMES"; + public static final String COMMERCIAL_SNDBCK_TIMES = "COMMERCIAL_SNDBCK_TIMES"; + public static final String COMMERCIAL_RCV_TIMES = "COMMERCIAL_RCV_TIMES"; + public static final String COMMERCIAL_RCV_EPOLLS = "COMMERCIAL_RCV_EPOLLS"; + public static final String COMMERCIAL_SEND_SIZE = "COMMERCIAL_SEND_SIZE"; + public static final String COMMERCIAL_RCV_SIZE = "COMMERCIAL_RCV_SIZE"; + public static final String COMMERCIAL_PERM_FAILURES = "COMMERCIAL_PERM_FAILURES"; + public static final String COMMERCIAL_OWNER = "Owner"; + // Message Size limit for one api-calling count. + public static final double SIZE_PER_COUNT = 64 * 1024; + + public static final String GROUP_GET_FALL_SIZE = "GROUP_GET_FALL_SIZE"; + public static final String GROUP_GET_FALL_TIME = "GROUP_GET_FALL_TIME"; + // Pull Message Latency + public static final String GROUP_GET_LATENCY = "GROUP_GET_LATENCY"; + + /** + * read disk follow stats + */ + private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_STATS_LOGGER_NAME); + private static final Logger COMMERCIAL_LOG = LoggerFactory.getLogger(LoggerName.COMMERCIAL_LOGGER_NAME); + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "BrokerStatsThread")); + private final ScheduledExecutorService commercialExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "CommercialStatsThread")); + private final HashMap statsTable = new HashMap(); + private final String clusterName; + private final MomentStatsItemSet momentStatsItemSetFallSize = new MomentStatsItemSet(GROUP_GET_FALL_SIZE, scheduledExecutorService, log); + private final MomentStatsItemSet momentStatsItemSetFallTime = new MomentStatsItemSet(GROUP_GET_FALL_TIME, scheduledExecutorService, log); + + public BrokerStatsManager(String clusterName) { + this.clusterName = clusterName; + + this.statsTable.put(TOPIC_PUT_NUMS, new StatsItemSet(TOPIC_PUT_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(TOPIC_PUT_SIZE, new StatsItemSet(TOPIC_PUT_SIZE, this.scheduledExecutorService, log)); + this.statsTable.put(GROUP_GET_NUMS, new StatsItemSet(GROUP_GET_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(GROUP_GET_SIZE, new StatsItemSet(GROUP_GET_SIZE, this.scheduledExecutorService, log)); + this.statsTable.put(GROUP_GET_LATENCY, new StatsItemSet(GROUP_GET_LATENCY, this.scheduledExecutorService, log)); + this.statsTable.put(SNDBCK_PUT_NUMS, new StatsItemSet(SNDBCK_PUT_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(BROKER_PUT_NUMS, new StatsItemSet(BROKER_PUT_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(BROKER_GET_NUMS, new StatsItemSet(BROKER_GET_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(GROUP_GET_FROM_DISK_NUMS, new StatsItemSet(GROUP_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(GROUP_GET_FROM_DISK_SIZE, new StatsItemSet(GROUP_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log)); + this.statsTable.put(BROKER_GET_FROM_DISK_NUMS, new StatsItemSet(BROKER_GET_FROM_DISK_NUMS, this.scheduledExecutorService, log)); + this.statsTable.put(BROKER_GET_FROM_DISK_SIZE, new StatsItemSet(BROKER_GET_FROM_DISK_SIZE, this.scheduledExecutorService, log)); + + + this.statsTable.put(COMMERCIAL_SEND_TIMES, new StatsItemSet(COMMERCIAL_SEND_TIMES, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_RCV_TIMES, new StatsItemSet(COMMERCIAL_RCV_TIMES, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_SEND_SIZE, new StatsItemSet(COMMERCIAL_SEND_SIZE, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_RCV_SIZE, new StatsItemSet(COMMERCIAL_RCV_SIZE, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_RCV_EPOLLS, new StatsItemSet(COMMERCIAL_RCV_EPOLLS, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_SNDBCK_TIMES, new StatsItemSet(COMMERCIAL_SNDBCK_TIMES, this.commercialExecutor, COMMERCIAL_LOG)); + this.statsTable.put(COMMERCIAL_PERM_FAILURES, new StatsItemSet(COMMERCIAL_PERM_FAILURES, this.commercialExecutor, COMMERCIAL_LOG)); + } + + public MomentStatsItemSet getMomentStatsItemSetFallSize() { + return momentStatsItemSetFallSize; + } + + public MomentStatsItemSet getMomentStatsItemSetFallTime() { + return momentStatsItemSetFallTime; + } + + public void start() { + } + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + public StatsItem getStatsItem(final String statsName, final String statsKey) { + try { + return this.statsTable.get(statsName).getStatsItem(statsKey); + } catch (Exception e) { + } + + return null; + } + + public void incTopicPutNums(final String topic) { + this.statsTable.get(TOPIC_PUT_NUMS).addValue(topic, 1, 1); + } + + public void incTopicPutSize(final String topic, final int size) { + this.statsTable.get(TOPIC_PUT_SIZE).addValue(topic, size, 1); + } + + public void incGroupGetNums(final String group, final String topic, final int incValue) { + final String statsKey = buildStatsKey(topic, group); + this.statsTable.get(GROUP_GET_NUMS).addValue(statsKey, incValue, 1); + } + + public String buildStatsKey(String topic, String group) { + StringBuffer strBuilder = new StringBuffer(); + strBuilder.append(topic); + strBuilder.append("@"); + strBuilder.append(group); + return strBuilder.toString(); + } + + public void incGroupGetSize(final String group, final String topic, final int incValue) { + final String statsKey = buildStatsKey(topic, group); + this.statsTable.get(GROUP_GET_SIZE).addValue(statsKey, incValue, 1); + } + + public void incGroupGetLatency(final String group, final String topic, final int queueId, final int incValue) { + final String statsKey = String.format("%d@%s@%s", queueId, topic, group); + this.statsTable.get(GROUP_GET_LATENCY).addValue(statsKey, incValue, 1); + } + + + public void incBrokerPutNums() { + this.statsTable.get(BROKER_PUT_NUMS).getAndCreateStatsItem(this.clusterName).getValue().incrementAndGet(); + } + + + public void incBrokerGetNums(final int incValue) { + this.statsTable.get(BROKER_GET_NUMS).getAndCreateStatsItem(this.clusterName).getValue().addAndGet(incValue); + } + + + public void incSendBackNums(final String group, final String topic) { + final String statsKey = buildStatsKey(topic, group); + this.statsTable.get(SNDBCK_PUT_NUMS).addValue(statsKey, 1, 1); + } + + + public double tpsGroupGetNums(final String group, final String topic) { + final String statsKey = buildStatsKey(topic, group); + return this.statsTable.get(GROUP_GET_NUMS).getStatsDataInMinute(statsKey).getTps(); + } + + + public void recordDiskFallBehindTime(final String group, final String topic, final int queueId, final long fallBehind) { + final String statsKey = String.format("%d@%s@%s", queueId, topic, group); + this.momentStatsItemSetFallTime.getAndCreateStatsItem(statsKey).getValue().set(fallBehind); + } + + + public void recordDiskFallBehindSize(final String group, final String topic, final int queueId, final long fallBehind) { + final String statsKey = String.format("%d@%s@%s", queueId, topic, group); + this.momentStatsItemSetFallSize.getAndCreateStatsItem(statsKey).getValue().set(fallBehind); + } + + public void incCommercialValue(final String key, final String owner, final String group, + final String topic, final String type, final int incValue) { + final String statsKey = buildCommercialStatsKey(owner, topic, group, type); + this.statsTable.get(key).addValue(statsKey, incValue, 1); + } + + public String buildCommercialStatsKey(String owner, String topic, String group, String type) { + StringBuffer strBuilder = new StringBuffer(); + strBuilder.append(owner); + strBuilder.append("@"); + strBuilder.append(topic); + strBuilder.append("@"); + strBuilder.append(group); + strBuilder.append("@"); + strBuilder.append(type); + return strBuilder.toString(); + } + + + public enum StatsType { + SEND_SUCCESS, + SEND_FAILURE, + SEND_BACK, + SEND_TIMER, + SEND_TRANSACTION, + RCV_SUCCESS, + RCV_EPOLLS, + PERM_FAILURE + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/util/LibC.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/util/LibC.java new file mode 100644 index 0000000000..7a3b37cf20 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/util/LibC.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.util; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.NativeLong; +import com.sun.jna.Platform; +import com.sun.jna.Pointer; + + +public interface LibC extends Library { + LibC INSTANCE = (LibC) Native.loadLibrary(Platform.isWindows() ? "msvcrt" : "c", LibC.class); + + int MADV_WILLNEED = 3; + int MADV_DONTNEED = 4; + + int MCL_CURRENT = 1; + int MCL_FUTURE = 2; + int MCL_ONFAULT = 4; + + /* sync memory asynchronously */ + int MS_ASYNC = 0x0001; + /* invalidate mappings & caches */ + int MS_INVALIDATE = 0x0002; + /* synchronous memory sync */ + int MS_SYNC = 0x0004; + + int mlock(Pointer var1, NativeLong var2); + + int munlock(Pointer var1, NativeLong var2); + + int madvise(Pointer var1, NativeLong var2, int var3); + + Pointer memset(Pointer p, int v, long len); + + int mlockall(int flags); + + int msync(Pointer p, NativeLong length, int flags); +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java new file mode 100644 index 0000000000..ac577c7930 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.store.config.FlushDiskType; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class DefaultMessageStoreTest { + private static final String StoreMessage = "Once, there was a chance for me!"; + + private static int QUEUE_TOTAL = 100; + + private static AtomicInteger QueueId = new AtomicInteger(0); + + private static SocketAddress BornHost; + + private static SocketAddress StoreHost; + + private static byte[] MessageBody; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void test_write_read() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 100; + QUEUE_TOTAL = 1; + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8); + messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 4); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(100 * 10); + MessageStore master = new DefaultMessageStore(messageStoreConfig, null, null, null); + + boolean load = master.load(); + assertTrue(load); + + master.start(); + for (long i = 0; i < totalMsgs; i++) { + PutMessageResult result = master.putMessage(buildMessage()); + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } catch (Exception e) { + e.printStackTrace(); + } + + } + master.shutdown(); + master.destroy(); + System.out.println("================================================================"); + } + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("AAA"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + return msg; + } + + @Test + public void test_group_commit() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 100; + QUEUE_TOTAL = 1; + MessageBody = StoreMessage.getBytes(); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8); + messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); + MessageStore master = new DefaultMessageStore(messageStoreConfig, null, null, null); + boolean load = master.load(); + assertTrue(load); + + master.start(); + for (long i = 0; i < totalMsgs; i++) { + PutMessageResult result = master.putMessage(buildMessage()); + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } catch (Exception e) { + e.printStackTrace(); + } + + } + master.shutdown(); + master.destroy(); + System.out.println("================================================================"); + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileQueueTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileQueueTest.java new file mode 100644 index 0000000000..ce7666fb6e --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileQueueTest.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: MappedFileQueueTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import org.junit.*; + +import static org.junit.Assert.*; + + +public class MappedFileQueueTest { + + // private static final String StoreMessage = + // "Once, there was a chance for me! but I did not treasure it. if"; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void test_getLastMapedFile() { + final String fixedMsg = "0123456789abcdef"; + System.out.println("================================================================"); + MappedFileQueue mappedFileQueue = + new MappedFileQueue("target/unit_test_store/a/", 1024, null); + + for (int i = 0; i < 1024; i++) { + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0); + assertTrue(mappedFile != null); + boolean result = mappedFile.appendMessage(fixedMsg.getBytes()); + if (!result) { + System.out.println("appendMessage " + i); + } + assertTrue(result); + } + + mappedFileQueue.shutdown(1000); + mappedFileQueue.destroy(); + System.out.println("MappedFileQueue.getLastMappedFile() OK"); + } + + + @Test + public void test_findMapedFileByOffset() { + final String fixedMsg = "abcd"; + System.out.println("================================================================"); + MappedFileQueue mappedFileQueue = + new MappedFileQueue("target/unit_test_store/b/", 1024, null); + + for (int i = 0; i < 1024; i++) { + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0); + assertTrue(mappedFile != null); + boolean result = mappedFile.appendMessage(fixedMsg.getBytes()); + // System.out.println("appendMessage " + bytes); + assertTrue(result); + } + + MappedFile mappedFile = mappedFileQueue.findMappedFileByOffset(0); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 0); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(100); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 0); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 1024); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024 + 100); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 1024); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 2); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 1024 * 2); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 2 + 100); + assertTrue(mappedFile != null); + assertEquals(mappedFile.getFileFromOffset(), 1024 * 2); + System.out.println(mappedFile.getFileFromOffset()); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 4); + assertTrue(mappedFile == null); + + mappedFile = mappedFileQueue.findMappedFileByOffset(1024 * 4 + 100); + assertTrue(mappedFile == null); + + mappedFileQueue.shutdown(1000); + mappedFileQueue.destroy(); + System.out.println("MappedFileQueue.findMappedFileByOffset() OK"); + } + + @Test + public void test_commit() { + final String fixedMsg = "0123456789abcdef"; + System.out.println("================================================================"); + MappedFileQueue mappedFileQueue = + new MappedFileQueue("target/unit_test_store/c/", 1024, null); + + for (int i = 0; i < 1024; i++) { + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0); + assertTrue(mappedFile != null); + boolean result = mappedFile.appendMessage(fixedMsg.getBytes()); + assertTrue(result); + } + + boolean result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 1, mappedFileQueue.getFlushedWhere()); + System.out.println("1 " + result + " " + mappedFileQueue.getFlushedWhere()); + + result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 2, mappedFileQueue.getFlushedWhere()); + System.out.println("2 " + result + " " + mappedFileQueue.getFlushedWhere()); + + result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 3, mappedFileQueue.getFlushedWhere()); + System.out.println("3 " + result + " " + mappedFileQueue.getFlushedWhere()); + + result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 4, mappedFileQueue.getFlushedWhere()); + System.out.println("4 " + result + " " + mappedFileQueue.getFlushedWhere()); + + result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 5, mappedFileQueue.getFlushedWhere()); + System.out.println("5 " + result + " " + mappedFileQueue.getFlushedWhere()); + + result = mappedFileQueue.flush(0); + assertFalse(result); + assertEquals(1024 * 6, mappedFileQueue.getFlushedWhere()); + System.out.println("6 " + result + " " + mappedFileQueue.getFlushedWhere()); + + mappedFileQueue.shutdown(1000); + mappedFileQueue.destroy(); + System.out.println("MappedFileQueue.flush() OK"); + } + + @Test + public void test_getMapedMemorySize() { + final String fixedMsg = "abcd"; + System.out.println("================================================================"); + MappedFileQueue mappedFileQueue = + new MappedFileQueue("target/unit_test_store/d/", 1024, null); + + for (int i = 0; i < 1024; i++) { + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(0); + assertTrue(mappedFile != null); + boolean result = mappedFile.appendMessage(fixedMsg.getBytes()); + assertTrue(result); + } + + assertEquals(fixedMsg.length() * 1024, mappedFileQueue.getMappedMemorySize()); + mappedFileQueue.shutdown(1000); + mappedFileQueue.destroy(); + System.out.println("MappedFileQueue.getMappedMemorySize() OK"); + } + +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileTest.java new file mode 100644 index 0000000000..94fd5ee3cc --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MappedFileTest.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: MappedFileTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + + +public class MappedFileTest { + + private static final String StoreMessage = "Once, there was a chance for me!"; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void test_write_read() { + try { + MappedFile mappedFile = new MappedFile("target/unit_test_store/MappedFileTest/000", 1024 * 64); + boolean result = mappedFile.appendMessage(StoreMessage.getBytes()); + assertTrue(result); + System.out.println("write OK"); + + SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(0); + byte[] data = new byte[StoreMessage.length()]; + selectMappedBufferResult.getByteBuffer().get(data); + String readString = new String(data); + + System.out.println("Read: " + readString); + assertTrue(readString.equals(StoreMessage)); + + mappedFile.shutdown(1000); + assertTrue(!mappedFile.isAvailable()); + selectMappedBufferResult.release(); + assertTrue(mappedFile.isCleanupOver()); + assertTrue(mappedFile.destroy(1000)); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Ignore + public void test_jvm_crashed() { + try { + MappedFile mappedFile = new MappedFile("target/unit_test_store/MappedFileTest/10086", 1024 * 64); + boolean result = mappedFile.appendMessage(StoreMessage.getBytes()); + assertTrue(result); + System.out.println("write OK"); + + SelectMappedBufferResult selectMappedBufferResult = mappedFile.selectMappedBuffer(0); + selectMappedBufferResult.release(); + mappedFile.shutdown(1000); + + byte[] data = new byte[StoreMessage.length()]; + selectMappedBufferResult.getByteBuffer().get(data); + String readString = new String(data); + System.out.println(readString); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java new file mode 100644 index 0000000000..ea833752d3 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: RecoverTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +public class RecoverTest { + private static final String StoreMessage = "Once, there was a chance for me!aaaaaaaaaaaaaaaaaaaaaaaa"; + + private static int QUEUE_TOTAL = 10; + + private static AtomicInteger QueueId = new AtomicInteger(0); + + private static SocketAddress BornHost; + + private static SocketAddress StoreHost; + + private static byte[] MessageBody; + private MessageStore storeWrite1; + private MessageStore storeWrite2; + private MessageStore storeRead; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void test_recover_normally() throws Exception { + this.writeMessage(true, true); + Thread.sleep(1000 * 3); + this.readMessage(1000); + this.destroy(); + } + + public void writeMessage(boolean normal, boolean first) throws Exception { + System.out.println("================================================================"); + long totalMsgs = 100; + QUEUE_TOTAL = 3; + + MessageBody = StoreMessage.getBytes(); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + messageStoreConfig.setMapedFileSizeConsumeQueue(100 * 20); + messageStoreConfig.setMessageIndexEnable(false); + + MessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null, null, null); + if (first) { + this.storeWrite1 = messageStore; + } else { + this.storeWrite2 = messageStore; + } + + boolean loadResult = messageStore.load(); + assertTrue(loadResult); + messageStore.start(); + for (long i = 0; i < totalMsgs; i++) { + PutMessageResult result = messageStore.putMessage(buildMessage()); + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + if (normal) { + messageStore.shutdown(); + } + System.out.println("========================writeMessage OK========================================"); + } + + public void readMessage(final long msgCnt) throws Exception { + System.out.println("================================================================"); + QUEUE_TOTAL = 3; + MessageBody = StoreMessage.getBytes(); + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + messageStoreConfig.setMapedFileSizeConsumeQueue(100 * 20); + messageStoreConfig.setMessageIndexEnable(false); + storeRead = new DefaultMessageStore(messageStoreConfig, null, null, null); + boolean loadResult = storeRead.load(); + assertTrue(loadResult); + storeRead.start(); + + long readCnt = 0; + for (int queueId = 0; queueId < QUEUE_TOTAL; queueId++) { + for (long offset = 0; ; ) { + GetMessageResult result = storeRead.getMessage("GROUP_A", "TOPIC_A", queueId, offset, 1024 * 1024, null); + if (result.getStatus() == GetMessageStatus.FOUND) { + System.out.println(queueId + "\t" + result.getMessageCount()); + this.veryReadMessage(queueId, offset, result.getMessageBufferList()); + offset += result.getMessageCount(); + readCnt += result.getMessageCount(); + result.release(); + } else { + break; + } + } + } + + System.out.println("readCnt = " + readCnt); + assertTrue(readCnt == msgCnt); + System.out.println("========================readMessage OK========================================"); + } + + private void destroy() { + if (storeWrite1 != null) { + storeWrite1.shutdown(); + storeWrite1.destroy(); + } + + if (storeWrite2 != null) { + storeWrite2.shutdown(); + storeWrite2.destroy(); + } + + if (storeRead != null) { + storeRead.shutdown(); + storeRead.destroy(); + } + } + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("TOPIC_A"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + + return msg; + } + + private void veryReadMessage(int queueId, long queueOffset, List byteBuffers) { + for (ByteBuffer byteBuffer : byteBuffers) { + MessageExt msg = MessageDecoder.decode(byteBuffer); + System.out.println("request queueId " + queueId + ", request queueOffset " + queueOffset + " msg queue offset " + + msg.getQueueOffset()); + + assertTrue(msg.getQueueOffset() == queueOffset); + + queueOffset++; + } + } + + @Test + public void test_recover_normally_write() throws Exception { + this.writeMessage(true, true); + Thread.sleep(1000 * 3); + this.writeMessage(true, false); + Thread.sleep(1000 * 3); + this.readMessage(2000); + this.destroy(); + } + + @Test + public void test_recover_abnormally() throws Exception { + this.writeMessage(false, true); + Thread.sleep(1000 * 3); + this.readMessage(1000); + this.destroy(); + } + + @Test + public void test_recover_abnormally_write() throws Exception { + this.writeMessage(false, true); + Thread.sleep(1000 * 3); + this.writeMessage(false, false); + Thread.sleep(1000 * 3); + this.readMessage(2000); + this.destroy(); + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java new file mode 100644 index 0000000000..e0a550d91e --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: StoreCheckpointTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + + +public class StoreCheckpointTest { + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void test_write_read() { + try { + StoreCheckpoint storeCheckpoint = new StoreCheckpoint("target/checkpoint_test/0000"); + long physicMsgTimestamp = 0xAABB; + long logicsMsgTimestamp = 0xCCDD; + storeCheckpoint.setPhysicMsgTimestamp(physicMsgTimestamp); + storeCheckpoint.setLogicsMsgTimestamp(logicsMsgTimestamp); + storeCheckpoint.flush(); + + long diff = physicMsgTimestamp - storeCheckpoint.getMinTimestamp(); + assertTrue(diff == 3000); + storeCheckpoint.shutdown(); + storeCheckpoint = new StoreCheckpoint("target/checkpoint_test/0000"); + assertTrue(physicMsgTimestamp == storeCheckpoint.getPhysicMsgTimestamp()); + assertTrue(logicsMsgTimestamp == storeCheckpoint.getLogicsMsgTimestamp()); + } catch (Throwable e) { + e.printStackTrace(); + assertTrue(false); + } + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java new file mode 100644 index 0000000000..288b87e9d9 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: IndexFileTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store.index; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +public class IndexFileTest { + private static final int hashSlotNum = 100; + private static final int indexNum = 400; + + @Test + public void test_put_index() { + try { + IndexFile indexFile = new IndexFile("100", hashSlotNum, indexNum, 0, 0); + for (long i = 0; i < (indexNum - 1); i++) { + boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis()); + assertTrue(putResult); + } + + boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis()); + assertFalse(putResult); + + indexFile.destroy(0); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } + } + + + @Test + public void test_put_get_index() { + try { + IndexFile indexFile = new IndexFile("200", hashSlotNum, indexNum, 0, 0); + + for (long i = 0; i < (indexNum - 1); i++) { + boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis()); + assertTrue(putResult); + } + boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis()); + assertFalse(putResult); + + final List phyOffsets = new ArrayList(); + indexFile.selectPhyOffset(phyOffsets, "60", 10, 0, Long.MAX_VALUE, true); + for (Long offset : phyOffsets) { + System.out.println(offset); + } + assertFalse(phyOffsets.isEmpty()); + indexFile.destroy(0); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java new file mode 100644 index 0000000000..d7de738b83 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: ScheduleMessageTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store.schedule; + +import com.alibaba.rocketmq.store.*; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + +@Ignore +public class ScheduleMessageTest { + private static final String StoreMessage = "Once, there was a chance for me!"; + + private static int QUEUE_TOTAL = 100; + + private static AtomicInteger QueueId = new AtomicInteger(0); + + private static SocketAddress BornHost; + + private static SocketAddress StoreHost; + + private static byte[] MessageBody; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void test_delay_message() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 10000; + QUEUE_TOTAL = 32; + + + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 16); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(1000 * 10); + + MessageStore master = new DefaultMessageStore(messageStoreConfig, null, null, null); + + boolean load = master.load(); + assertTrue(load); + + + master.start(); + for (int i = 0; i < totalMsgs; i++) { + MessageExtBrokerInner msg = buildMessage(); + msg.setDelayTimeLevel(i % 4); + + PutMessageResult result = master.putMessage(msg); + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + System.out.println("write message over, wait time up"); + Thread.sleep(1000 * 20); + + + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + Thread.sleep(1000 * 15); + + + master.shutdown(); + + + master.destroy(); + System.out.println("================================================================"); + } + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("AAA"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + + return msg; + } +} diff --git a/rocketmq-tools/pom.xml b/rocketmq-tools/pom.xml new file mode 100644 index 0000000000..5070a68f94 --- /dev/null +++ b/rocketmq-tools/pom.xml @@ -0,0 +1,66 @@ + + + + + com.alibaba.rocketmq + rocketmq-all + 4.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-tools + rocketmq-tools ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-srvutil + + + com.alibaba + fastjson + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + org.apache.commons + commons-lang3 + + + diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java new file mode 100644 index 0000000000..457688647d --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -0,0 +1,458 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { + private final DefaultMQAdminExtImpl defaultMQAdminExtImpl; + private String adminExtGroup = "admin_ext_group"; + private String createTopicKey = MixAll.DEFAULT_TOPIC; + private long timeoutMillis = 5000; + + public DefaultMQAdminExt() { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, null, timeoutMillis); + } + + public DefaultMQAdminExt(long timeoutMillis) { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, null, timeoutMillis); + } + + public DefaultMQAdminExt(RPCHook rpcHook) { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, rpcHook, timeoutMillis); + } + + public DefaultMQAdminExt(RPCHook rpcHook, long timeoutMillis) { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, rpcHook, timeoutMillis); + } + + public DefaultMQAdminExt(final String adminExtGroup) { + this.adminExtGroup = adminExtGroup; + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, timeoutMillis); + } + + public DefaultMQAdminExt(final String adminExtGroup, long timeoutMillis) { + this.adminExtGroup = adminExtGroup; + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, timeoutMillis); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + defaultMQAdminExtImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return defaultMQAdminExtImpl.searchOffset(mq, timestamp); + } + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.minOffset(mq); + } + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.earliestMsgStoreTime(mq); + } + + @Override + public MessageExt viewMessage(String offsetMsgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.viewMessage(offsetMsgId); + } + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, + InterruptedException { + return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); + } + + @Override + public void start() throws MQClientException { + defaultMQAdminExtImpl.start(); + } + + @Override + public void shutdown() { + defaultMQAdminExtImpl.shutdown(); + } + + @Override + public void updateBrokerConfig(String brokerAddr, Properties properties) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException { + defaultMQAdminExtImpl.updateBrokerConfig(brokerAddr, properties); + } + + @Override + public Properties getBrokerConfig(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.getBrokerConfig(brokerAddr); + } + + @Override + public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateTopicConfig(addr, config); + } + + @Override + public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfig(addr, config); + } + + @Override + public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) { + return defaultMQAdminExtImpl.examineSubscriptionGroupConfig(addr, group); + } + + @Override + public TopicConfig examineTopicConfig(String addr, String topic) { + return defaultMQAdminExtImpl.examineTopicConfig(addr, topic); + } + + @Override + public TopicStatsTable examineTopicStats(String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return defaultMQAdminExtImpl.examineTopicStats(topic); + } + + @Override + public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.fetchAllTopicList(); + } + + @Override + public TopicList fetchTopicsByCLuster(String clusterName) throws RemotingException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.fetchTopicsByCLuster(clusterName); + } + + @Override + public KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + return this.defaultMQAdminExtImpl.fetchBrokerRuntimeStats(brokerAddr); + } + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return examineConsumeStats(consumerGroup, null); + } + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup, String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineConsumeStats(consumerGroup, topic); + } + + @Override + public ClusterInfo examineBrokerClusterInfo() throws InterruptedException, RemotingConnectException, RemotingTimeoutException, + RemotingSendRequestException, MQBrokerException { + return defaultMQAdminExtImpl.examineBrokerClusterInfo(); + } + + @Override + public TopicRouteData examineTopicRouteInfo(String topic) throws RemotingException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.examineTopicRouteInfo(topic); + } + + @Override + public ConsumerConnection examineConsumerConnectionInfo(String consumerGroup) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + return defaultMQAdminExtImpl.examineConsumerConnectionInfo(consumerGroup); + } + + @Override + public ProducerConnection examineProducerConnectionInfo(String producerGroup, final String topic) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineProducerConnectionInfo(producerGroup, topic); + } + + @Override + public List getNameServerAddressList() { + return this.defaultMQAdminExtImpl.getNameServerAddressList(); + } + + @Override + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) throws RemotingCommandException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.wipeWritePermOfBroker(namesrvAddr, brokerName); + } + + @Override + public void putKVConfig(String namespace, String key, String value) { + defaultMQAdminExtImpl.putKVConfig(namespace, key, value); + } + + @Override + public String getKVConfig(String namespace, String key) throws RemotingException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.getKVConfig(namespace, key); + } + + @Override + public KVTable getKVListByNamespace(String namespace) throws RemotingException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.getKVListByNamespace(namespace); + } + + @Override + public void deleteTopicInBroker(Set addrs, String topic) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + defaultMQAdminExtImpl.deleteTopicInBroker(addrs, topic); + } + + @Override + public void deleteTopicInNameServer(Set addrs, String topic) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + defaultMQAdminExtImpl.deleteTopicInNameServer(addrs, topic); + } + + @Override + public void deleteSubscriptionGroup(String addr, String groupName) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + defaultMQAdminExtImpl.deleteSubscriptionGroup(addr, groupName); + } + + @Override + public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateKvConfig(namespace, key, value); + } + + @Override + public void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + defaultMQAdminExtImpl.deleteKvConfig(namespace, key); + } + + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, boolean force) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + } + + @Override + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); + } + + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, boolean isC) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(topic, group, timestamp, isForce, isC); + } + + @Override + public void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.defaultMQAdminExtImpl.resetOffsetNew(consumerGroup, topic, timestamp); + } + + @Override + public Map> getConsumeStatus(String topic, String group, String clientAddr) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.getConsumeStatus(topic, group, clientAddr); + } + + @Override + public void createOrUpdateOrderConf(String key, String value, boolean isCluster) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + defaultMQAdminExtImpl.createOrUpdateOrderConf(key, value, isCluster); + } + + @Override + public GroupList queryTopicConsumeByWho(String topic) throws InterruptedException, MQBrokerException, RemotingException, + MQClientException { + return this.defaultMQAdminExtImpl.queryTopicConsumeByWho(topic); + } + + @Override + public List queryConsumeTimeSpan(final String topic, final String group) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + return this.defaultMQAdminExtImpl.queryConsumeTimeSpan(topic, group); + } + + @Override + public boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanExpiredConsumerQueue(cluster); + } + + @Override + public boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanExpiredConsumerQueueByAddr(addr); + } + + @Override + public boolean cleanUnusedTopic(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanUnusedTopicByAddr(cluster); + } + + @Override + public boolean cleanUnusedTopicByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanUnusedTopicByAddr(addr); + } + + @Override + public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String clientId, boolean jstack) throws RemotingException, + MQClientException, InterruptedException { + return defaultMQAdminExtImpl.getConsumerRunningInfo(consumerGroup, clientId, jstack); + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId, String msgId) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, msgId); + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + } + + @Override + public List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return this.defaultMQAdminExtImpl.messageTrackDetail(msg); + } + + @Override + public void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + this.defaultMQAdminExtImpl.cloneGroupOffset(srcGroup, destGroup, topic, isOffline); + } + + @Override + public BrokerStatsData viewBrokerStatsData(String brokerAddr, String statsName, String statsKey) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.viewBrokerStatsData(brokerAddr, statsName, statsKey); + } + + @Override + public Set getClusterList(String topic) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.getClusterList(topic); + } + + @Override + public ConsumeStatsList fetchConsumeStatsInBroker(final String brokerAddr, boolean isOrder, long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.fetchConsumeStatsInBroker(brokerAddr, isOrder, timeoutMillis); + } + + @Override + public Set getTopicClusterList(final String topic) throws InterruptedException, MQBrokerException, MQClientException, RemotingException { + return this.defaultMQAdminExtImpl.getTopicClusterList(topic); + } + + @Override + public SubscriptionGroupWrapper getAllSubscriptionGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException { + return this.defaultMQAdminExtImpl.getAllSubscriptionGroup(brokerAddr, timeoutMillis); + } + + @Override + public TopicConfigSerializeWrapper getAllTopicGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException { + return this.defaultMQAdminExtImpl.getAllTopicGroup(brokerAddr, timeoutMillis); + } + + + /* (non-Javadoc) + * @see com.alibaba.rocketmq.client.MQAdmin#queryMessageByUniqKey(java.lang.String, java.lang.String) + */ + @Override + public MessageExt viewMessage(String topic, String msgId) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.defaultMQAdminExtImpl.viewMessage(topic, msgId); + } + + public String getAdminExtGroup() { + return adminExtGroup; + } + + public void setAdminExtGroup(String adminExtGroup) { + this.adminExtGroup = adminExtGroup; + } + + public String getCreateTopicKey() { + return createTopicKey; + } + + public void setCreateTopicKey(String createTopicKey) { + this.createTopicKey = createTopicKey; + } + + @Override + public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { + this.defaultMQAdminExtImpl.updateConsumeOffset(brokerAddr, consumeGroup, mq, offset); + } + + @Override + public void updateNameServerConfig(final Properties properties, final List nameServers) + throws InterruptedException, RemotingConnectException, + UnsupportedEncodingException, MQBrokerException, RemotingTimeoutException, + MQClientException, RemotingSendRequestException { + this.defaultMQAdminExtImpl.updateNameServerConfig(properties, nameServers); + } + + @Override + public Map getNameServerConfig(final List nameServers) + throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQClientException, + UnsupportedEncodingException { + return this.defaultMQAdminExtImpl.getNameServerConfig(nameServers); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java new file mode 100644 index 0000000000..1de96db4fc --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -0,0 +1,933 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.admin.MQAdminExtInner; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.*; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; +import com.alibaba.rocketmq.tools.admin.api.TrackType; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.util.*; +import java.util.Map.Entry; + + +/** + * @author shijia.wxr + */ +public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { + private final Logger log = ClientLogger.getLog(); + private final DefaultMQAdminExt defaultMQAdminExt; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mqClientInstance; + private RPCHook rpcHook; + private long timeoutMillis = 20000; + private Random random = new Random(); + + + public DefaultMQAdminExtImpl(DefaultMQAdminExt defaultMQAdminExt, long timeoutMillis) { + this(defaultMQAdminExt, null, timeoutMillis); + } + + + public DefaultMQAdminExtImpl(DefaultMQAdminExt defaultMQAdminExt, RPCHook rpcHook, long timeoutMillis) { + this.defaultMQAdminExt = defaultMQAdminExt; + this.rpcHook = rpcHook; + this.timeoutMillis = timeoutMillis; + } + + + @Override + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.defaultMQAdminExt.changeInstanceNameToPID(); + + this.mqClientInstance = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQAdminExt, rpcHook); + + boolean registerOK = mqClientInstance.registerAdminExt(this.defaultMQAdminExt.getAdminExtGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The adminExt group[" + this.defaultMQAdminExt.getAdminExtGroup() + + "] has created already, specifed another name please."// + + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); + } + + mqClientInstance.start(); + + log.info("the adminExt [{}] start OK", this.defaultMQAdminExt.getAdminExtGroup()); + + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The AdminExt service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + default: + break; + } + } + + + @Override + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mqClientInstance.unregisterAdminExt(this.defaultMQAdminExt.getAdminExtGroup()); + this.mqClientInstance.shutdown(); + + log.info("the adminExt [{}] shutdown OK", this.defaultMQAdminExt.getAdminExtGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + @Override + public void updateBrokerConfig(String brokerAddr, Properties properties) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().updateBrokerConfig(brokerAddr, properties, timeoutMillis); + } + + @Override + public Properties getBrokerConfig(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerConfig(brokerAddr, timeoutMillis); + } + + @Override + public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createTopic(addr, this.defaultMQAdminExt.getCreateTopicKey(), config, timeoutMillis); + } + + @Override + public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroup(addr, config, timeoutMillis); + } + + @Override + public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) { + // TODO Auto-generated method stub + return null; + } + + @Override + public TopicConfig examineTopicConfig(String addr, String topic) { + // TODO Auto-generated method stub + return null; + } + + @Override + public TopicStatsTable examineTopicStats(String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = this.mqClientInstance.getMQClientAPIImpl().getTopicStatsInfo(addr, topic, timeoutMillis); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + + if (topicStatsTable.getOffsetTable().isEmpty()) { + throw new MQClientException("Not found the topic stats info", null); + } + + return topicStatsTable; + } + + @Override + public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicListFromNameServer(timeoutMillis); + } + + @Override + public TopicList fetchTopicsByCLuster(String clusterName) throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicsByCluster(clusterName, timeoutMillis); + } + + @Override + public KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerRuntimeInfo(brokerAddr, timeoutMillis); + } + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return examineConsumeStats(consumerGroup, null); + } + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup, String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + String retryTopic = MixAll.getRetryTopic(consumerGroup); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(retryTopic); + ConsumeStats result = new ConsumeStats(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + ConsumeStats consumeStats = + this.mqClientInstance.getMQClientAPIImpl().getConsumeStats(addr, consumerGroup, topic, timeoutMillis * 3); + result.getOffsetTable().putAll(consumeStats.getOffsetTable()); + double value = result.getConsumeTps() + consumeStats.getConsumeTps(); + result.setConsumeTps(value); + } + } + + if (result.getOffsetTable().isEmpty()) { + throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, + "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); + } + + return result; + } + + @Override + public ClusterInfo examineBrokerClusterInfo() throws InterruptedException, MQBrokerException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterInfo(timeoutMillis); + } + + @Override + public TopicRouteData examineTopicRouteInfo(String topic) throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis); + } + + + @Override + public MessageExt viewMessage(String topic, String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + MessageDecoder.decodeMessageId(msgId); + return this.viewMessage(msgId); + } catch (Exception e) { + log.warn("the msgId maybe created by new client. msgId={}", msgId, e); + } + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, msgId); + } + + @Override + public ConsumerConnection examineConsumerConnectionInfo(String consumerGroup) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + ConsumerConnection result = new ConsumerConnection(); + String topic = MixAll.getRetryTopic(consumerGroup); + List brokers = this.examineTopicRouteInfo(topic).getBrokerDatas(); + BrokerData brokerData = brokers.get(random.nextInt(brokers.size())); + String addr = null; + if (brokerData != null) { + addr = brokerData.selectBrokerAddr(); + if (StringUtils.isNotBlank(addr)) { + result = this.mqClientInstance.getMQClientAPIImpl().getConsumerConnectionList(addr, consumerGroup, timeoutMillis); + } + } + + if (result.getConnectionSet().isEmpty()) { + log.warn("the consumer group not online. brokerAddr={}, group={}", addr, consumerGroup); + throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group connection"); + } + + return result; + } + + @Override + public ProducerConnection examineProducerConnectionInfo(String producerGroup, final String topic) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + ProducerConnection result = new ProducerConnection(); + List brokers = this.examineTopicRouteInfo(topic).getBrokerDatas(); + BrokerData brokerData = brokers.get(random.nextInt(brokers.size())); + String addr = null; + if (brokerData != null) { + addr = brokerData.selectBrokerAddr(); + if (StringUtils.isNotBlank(addr)) { + result = this.mqClientInstance.getMQClientAPIImpl().getProducerConnectionList(addr, producerGroup, timeoutMillis); + } + } + + if (result.getConnectionSet().isEmpty()) { + log.warn("the producer group not online. brokerAddr={}, group={}", addr, producerGroup); + throw new MQClientException("Not found the producer group connection", null); + } + + return result; + } + + @Override + public List getNameServerAddressList() { + return this.mqClientInstance.getMQClientAPIImpl().getNameServerAddressList(); + } + + @Override + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) throws RemotingCommandException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl().wipeWritePermOfBroker(namesrvAddr, brokerName, timeoutMillis); + } + + @Override + public void putKVConfig(String namespace, String key, String value) { + // TODO Auto-generated method stub + + } + + @Override + public String getKVConfig(String namespace, String key) throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getKVConfigValue(namespace, key, timeoutMillis); + } + + @Override + public KVTable getKVListByNamespace(String namespace) throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getKVListByNamespace(namespace, timeoutMillis); + } + + @Override + public void deleteTopicInBroker(Set addrs, String topic) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + for (String addr : addrs) { + this.mqClientInstance.getMQClientAPIImpl().deleteTopicInBroker(addr, topic, timeoutMillis); + } + } + + @Override + public void deleteTopicInNameServer(Set addrs, String topic) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + if (addrs == null) { + String ns = this.mqClientInstance.getMQClientAPIImpl().fetchNameServerAddr(); + addrs = new HashSet(Arrays.asList(ns.split(";"))); + } + for (String addr : addrs) { + this.mqClientInstance.getMQClientAPIImpl().deleteTopicInNameServer(addr, topic, timeoutMillis); + } + } + + @Override + public void deleteSubscriptionGroup(String addr, String groupName) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + this.mqClientInstance.getMQClientAPIImpl().deleteSubscriptionGroup(addr, groupName, timeoutMillis); + } + + @Override + public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().putKVConfigValue(namespace, key, value, timeoutMillis); + } + + @Override + public void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigValue(namespace, key, timeoutMillis); + } + + @Override + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, boolean force) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List rollbackStatsList = new ArrayList(); + Map topicRouteMap = new HashMap(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + for (QueueData queueData : topicRouteData.getQueueDatas()) { + topicRouteMap.put(bd.selectBrokerAddr(), queueData.getReadQueueNums()); + } + } + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + ConsumeStats consumeStats = this.mqClientInstance.getMQClientAPIImpl().getConsumeStats(addr, consumerGroup, timeoutMillis); + + boolean hasConsumed = false; + for (Map.Entry entry : consumeStats.getOffsetTable().entrySet()) { + MessageQueue queue = entry.getKey(); + OffsetWrapper offsetWrapper = entry.getValue(); + if (topic.equals(queue.getTopic())) { + hasConsumed = true; + RollbackStats rollbackStats = resetOffsetConsumeOffset(addr, consumerGroup, queue, offsetWrapper, timestamp, force); + rollbackStatsList.add(rollbackStats); + } + } + + if (!hasConsumed) { + HashMap topicStatus = + this.mqClientInstance.getMQClientAPIImpl().getTopicStatsInfo(addr, topic, timeoutMillis).getOffsetTable(); + for (int i = 0; i < topicRouteMap.get(addr); i++) { + MessageQueue queue = new MessageQueue(topic, bd.getBrokerName(), i); + OffsetWrapper offsetWrapper = new OffsetWrapper(); + offsetWrapper.setBrokerOffset(topicStatus.get(queue).getMaxOffset()); + offsetWrapper.setConsumerOffset(topicStatus.get(queue).getMinOffset()); + + RollbackStats rollbackStats = resetOffsetConsumeOffset(addr, consumerGroup, queue, offsetWrapper, timestamp, force); + rollbackStatsList.add(rollbackStats); + } + } + } + } + return rollbackStatsList; + } + + @Override + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return resetOffsetByTimestamp(topic, group, timestamp, isForce, false); + } + + @Override + public void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + try { + this.resetOffsetByTimestamp(topic, consumerGroup, timestamp, true); + } catch (MQClientException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + this.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, true); + return; + } + throw e; + } + } + + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce, boolean isC) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + Map allOffsetTable = new HashMap(); + if (brokerDatas != null) { + for (BrokerData brokerData : brokerDatas) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + Map offsetTable = + this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToResetOffset(addr, topic, group, timestamp, isForce, + timeoutMillis, isC); + if (offsetTable != null) { + allOffsetTable.putAll(offsetTable); + } + } + } + } + return allOffsetTable; + } + + private RollbackStats resetOffsetConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue queue, OffsetWrapper offsetWrapper, + long timestamp, boolean force) throws RemotingException, InterruptedException, MQBrokerException { + long resetOffset; + if (timestamp == -1) { + + resetOffset = this.mqClientInstance.getMQClientAPIImpl().getMaxOffset(brokerAddr, queue.getTopic(), queue.getQueueId(), timeoutMillis); + } else { + resetOffset = + this.mqClientInstance.getMQClientAPIImpl().searchOffset(brokerAddr, queue.getTopic(), queue.getQueueId(), timestamp, + timeoutMillis); + } + + + RollbackStats rollbackStats = new RollbackStats(); + rollbackStats.setBrokerName(queue.getBrokerName()); + rollbackStats.setQueueId(queue.getQueueId()); + rollbackStats.setBrokerOffset(offsetWrapper.getBrokerOffset()); + rollbackStats.setConsumerOffset(offsetWrapper.getConsumerOffset()); + rollbackStats.setTimestampOffset(resetOffset); + rollbackStats.setRollbackOffset(offsetWrapper.getConsumerOffset()); + + if (force || resetOffset <= offsetWrapper.getConsumerOffset()) { + rollbackStats.setRollbackOffset(resetOffset); + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumeGroup); + requestHeader.setTopic(queue.getTopic()); + requestHeader.setQueueId(queue.getQueueId()); + requestHeader.setCommitOffset(resetOffset); + this.mqClientInstance.getMQClientAPIImpl().updateConsumerOffset(brokerAddr, requestHeader, timeoutMillis); + } + return rollbackStats; + } + + @Override + public Map> getConsumeStatus(String topic, String group, String clientAddr) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + if (brokerDatas != null && brokerDatas.size() > 0) { + String addr = brokerDatas.get(0).selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToGetConsumerStatus(addr, topic, group, clientAddr, + timeoutMillis); + } + } + return Collections.EMPTY_MAP; + } + + public void createOrUpdateOrderConf(String key, String value, boolean isCluster) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + + if (isCluster) { + this.mqClientInstance.getMQClientAPIImpl() + .putKVConfigValue(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, value, timeoutMillis); + } else { + String oldOrderConfs = null; + try { + oldOrderConfs = + this.mqClientInstance.getMQClientAPIImpl().getKVConfigValue(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, + timeoutMillis); + } catch (Exception e) { + e.printStackTrace(); + } + + Map orderConfMap = new HashMap(); + if (!UtilAll.isBlank(oldOrderConfs)) { + String[] oldOrderConfArr = oldOrderConfs.split(";"); + for (String oldOrderConf : oldOrderConfArr) { + String[] items = oldOrderConf.split(":"); + orderConfMap.put(items[0], oldOrderConf); + } + } + String[] items = value.split(":"); + orderConfMap.put(items[0], value); + + StringBuilder newOrderConf = new StringBuilder(); + String splitor = ""; + for (Map.Entry entry : orderConfMap.entrySet()) { + newOrderConf.append(splitor).append(entry.getValue()); + splitor = ";"; + } + this.mqClientInstance.getMQClientAPIImpl().putKVConfigValue(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, + newOrderConf.toString(), timeoutMillis); + } + } + + @Override + public GroupList queryTopicConsumeByWho(String topic) throws InterruptedException, MQBrokerException, RemotingException, + MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().queryTopicConsumeByWho(addr, topic, timeoutMillis); + } + + break; + } + + return null; + } + + @Override + public List queryConsumeTimeSpan(final String topic, final String group) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + List spanSet = new ArrayList(); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + spanSet.addAll(this.mqClientInstance.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, group, timeoutMillis)); + } + } + return spanSet; + } + + @Override + public boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = false; + try { + ClusterInfo clusterInfo = examineBrokerClusterInfo(); + if (null == cluster || "".equals(cluster)) { + for (String targetCluster : clusterInfo.retrieveAllClusterNames()) { + result = cleanExpiredConsumerQueueByCluster(clusterInfo, targetCluster); + } + } else { + result = cleanExpiredConsumerQueueByCluster(clusterInfo, cluster); + } + } catch (MQBrokerException e) { + log.error("cleanExpiredConsumerQueue error.", e); + } + + return result; + } + + public boolean cleanExpiredConsumerQueueByCluster(ClusterInfo clusterInfo, String cluster) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = false; + String[] addrs = clusterInfo.retrieveAllAddrByCluster(cluster); + for (String addr : addrs) { + result = cleanExpiredConsumerQueueByAddr(addr); + } + return result; + } + + @Override + public boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = mqClientInstance.getMQClientAPIImpl().cleanExpiredConsumeQueue(addr, timeoutMillis); + log.warn("clean expired ConsumeQueue on target " + addr + " broker " + result); + return result; + } + + @Override + public boolean cleanUnusedTopic(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = false; + try { + ClusterInfo clusterInfo = examineBrokerClusterInfo(); + if (null == cluster || "".equals(cluster)) { + for (String targetCluster : clusterInfo.retrieveAllClusterNames()) { + result = cleanUnusedTopicByCluster(clusterInfo, targetCluster); + } + } else { + result = cleanUnusedTopicByCluster(clusterInfo, cluster); + } + } catch (MQBrokerException e) { + log.error("cleanExpiredConsumerQueue error.", e); + } + + return result; + } + + public boolean cleanUnusedTopicByCluster(ClusterInfo clusterInfo, String cluster) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = false; + String[] addrs = clusterInfo.retrieveAllAddrByCluster(cluster); + for (String addr : addrs) { + result = cleanUnusedTopicByAddr(addr); + } + return result; + } + + @Override + public boolean cleanUnusedTopicByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = mqClientInstance.getMQClientAPIImpl().cleanUnusedTopicByAddr(addr, timeoutMillis); + log.warn("clean expired ConsumeQueue on target " + addr + " broker " + result); + return result; + } + + @Override + public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String clientId, boolean jstack) throws RemotingException, + MQClientException, InterruptedException { + String topic = MixAll.RETRY_GROUP_TOPIC_PREFIX + consumerGroup; + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + if (brokerDatas != null) { + for (BrokerData brokerData : brokerDatas) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().getConsumerRunningInfo(addr, consumerGroup, clientId, jstack, + timeoutMillis * 3); + } + } + } + return null; + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId, String msgId) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + MessageExt msg = this.viewMessage(msgId); + + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(RemotingUtil.socketAddress2String(msg.getStoreHost()), + consumerGroup, clientId, msgId, timeoutMillis * 3); + } + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String consumerGroup, final String clientId, final String topic, final String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + MessageExt msg = this.viewMessage(topic, msgId); + if (msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX) == null) { + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(RemotingUtil.socketAddress2String(msg.getStoreHost()), + consumerGroup, clientId, msgId, timeoutMillis * 3); + } else { + MessageClientExt msgClient = (MessageClientExt) msg; + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly(RemotingUtil.socketAddress2String(msg.getStoreHost()), + consumerGroup, clientId, msgClient.getOffsetMsgId(), timeoutMillis * 3); + } + } + + @Override + public List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + List result = new ArrayList(); + + GroupList groupList = this.queryTopicConsumeByWho(msg.getTopic()); + + for (String group : groupList.getGroupList()) { + + MessageTrack mt = new MessageTrack(); + mt.setConsumerGroup(group); + mt.setTrackType(TrackType.UNKNOWN); + ConsumerConnection cc = null; + try { + cc = this.examineConsumerConnectionInfo(group); + } catch (MQBrokerException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + mt.setTrackType(TrackType.NOT_ONLINE); + } + mt.setExceptionDesc("CODE:" + e.getResponseCode() + " DESC:" + e.getErrorMessage()); + result.add(mt); + continue; + } catch (Exception e) { + mt.setExceptionDesc(RemotingHelper.exceptionSimpleDesc(e)); + result.add(mt); + continue; + } + + switch (cc.getConsumeType()) { + case CONSUME_ACTIVELY: + mt.setTrackType(TrackType.PULL); + break; + case CONSUME_PASSIVELY: + boolean ifConsumed = false; + try { + ifConsumed = this.consumed(msg, group); + } catch (MQClientException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + mt.setTrackType(TrackType.NOT_ONLINE); + } + mt.setExceptionDesc("CODE:" + e.getResponseCode() + " DESC:" + e.getErrorMessage()); + result.add(mt); + continue; + } catch (MQBrokerException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + mt.setTrackType(TrackType.NOT_ONLINE); + } + mt.setExceptionDesc("CODE:" + e.getResponseCode() + " DESC:" + e.getErrorMessage()); + result.add(mt); + continue; + } catch (Exception e) { + mt.setExceptionDesc(RemotingHelper.exceptionSimpleDesc(e)); + result.add(mt); + continue; + } + + if (ifConsumed) { + mt.setTrackType(TrackType.CONSUMED); + Iterator> it = cc.getSubscriptionTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getKey().equals(msg.getTopic())) { + if (next.getValue().getTagsSet().contains(msg.getTags()) + || next.getValue().getTagsSet().contains("*") + || next.getValue().getTagsSet().isEmpty()) { + } else { + mt.setTrackType(TrackType.CONSUMED_BUT_FILTERED); + } + } + } + } else { + mt.setTrackType(TrackType.NOT_CONSUME_YET); + } + break; + default: + break; + } + result.add(mt); + } + + return result; + } + + public boolean consumed(final MessageExt msg, final String group) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + + ConsumeStats cstats = this.examineConsumeStats(group); + + ClusterInfo ci = this.examineBrokerClusterInfo(); + + Iterator> it = cstats.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + if (mq.getTopic().equals(msg.getTopic()) && mq.getQueueId() == msg.getQueueId()) { + BrokerData brokerData = ci.getBrokerAddrTable().get(mq.getBrokerName()); + if (brokerData != null) { + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr.equals(RemotingUtil.socketAddress2String(msg.getStoreHost()))) { + if (next.getValue().getConsumerOffset() > msg.getQueueOffset()) { + return true; + } + } + } + } + } + + return false; + } + + @Override + public void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + String retryTopic = MixAll.getRetryTopic(srcGroup); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(retryTopic); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + this.mqClientInstance.getMQClientAPIImpl().cloneGroupOffset(addr, srcGroup, destGroup, topic, isOffline, timeoutMillis * 3); + } + } + } + + @Override + public BrokerStatsData viewBrokerStatsData(String brokerAddr, String statsName, String statsKey) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().viewBrokerStatsData(brokerAddr, statsName, statsKey, timeoutMillis); + } + + @Override + public Set getClusterList(String topic) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getClusterList(topic, timeoutMillis); + } + + @Override + public ConsumeStatsList fetchConsumeStatsInBroker(final String brokerAddr, boolean isOrder, long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, + InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().fetchConsumeStatsInBroker(brokerAddr, isOrder, timeoutMillis); + } + + @Override + public Set getTopicClusterList(final String topic) throws InterruptedException, MQBrokerException, MQClientException, + RemotingException { + Set clusterSet = new HashSet(); + ClusterInfo clusterInfo = examineBrokerClusterInfo(); + TopicRouteData topicRouteData = examineTopicRouteInfo(topic); + BrokerData brokerData = topicRouteData.getBrokerDatas().get(0); + String brokerName = brokerData.getBrokerName(); + Iterator>> it = clusterInfo.getClusterAddrTable().entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> next = it.next(); + if (next.getValue().contains(brokerName)) { + clusterSet.add(next.getKey()); + } + } + return clusterSet; + } + + @Override + public SubscriptionGroupWrapper getAllSubscriptionGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getAllSubscriptionGroup(brokerAddr, timeoutMillis); + } + + @Override + public TopicConfigSerializeWrapper getAllTopicGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getAllTopicConfig(brokerAddr, timeoutMillis); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.mqClientInstance.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().searchOffset(mq, timestamp); + } + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().maxOffset(mq); + } + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().minOffset(mq); + } + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return this.mqClientInstance.getMQAdminImpl().viewMessage(msgId); + } + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, + InterruptedException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + @Override + public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumeGroup); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setCommitOffset(offset); + this.mqClientInstance.getMQClientAPIImpl().updateConsumerOffset(brokerAddr, requestHeader, timeoutMillis); + } + + @Override + public void updateNameServerConfig(final Properties properties, final List nameServers) + throws InterruptedException, RemotingConnectException, + UnsupportedEncodingException, RemotingSendRequestException, RemotingTimeoutException, + MQClientException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().updateNameServerConfig(properties, nameServers, timeoutMillis); + } + + @Override + public Map getNameServerConfig(final List nameServers) + throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQClientException, + UnsupportedEncodingException { + return this.mqClientInstance.getMQClientAPIImpl().getNameServerConfig(nameServers, timeoutMillis); + } + +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java new file mode 100644 index 0000000000..0075983765 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.*; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.exception.*; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +public interface MQAdminExt extends MQAdmin { + void start() throws MQClientException; + + void shutdown(); + + void updateBrokerConfig(final String brokerAddr, final Properties properties) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException; + + Properties getBrokerConfig(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, InterruptedException, MQBrokerException; + + void createAndUpdateTopicConfig(final String addr, final TopicConfig config) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + void createAndUpdateSubscriptionGroupConfig(final String addr, final SubscriptionGroupConfig config) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr, final String group); + + TopicConfig examineTopicConfig(final String addr, final String topic); + + TopicStatsTable examineTopicStats(final String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException; + + TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException; + + TopicList fetchTopicsByCLuster(String clusterName) throws RemotingException, MQClientException, InterruptedException; + + KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException; + + ConsumeStats examineConsumeStats(final String consumerGroup) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException; + + ConsumeStats examineConsumeStats(final String consumerGroup, final String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException; + + ClusterInfo examineBrokerClusterInfo() throws InterruptedException, MQBrokerException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException; + + TopicRouteData examineTopicRouteInfo(final String topic) throws RemotingException, MQClientException, InterruptedException; + + ConsumerConnection examineConsumerConnectionInfo(final String consumerGroup) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, RemotingException, + MQClientException; + + ProducerConnection examineProducerConnectionInfo(final String producerGroup, final String topic) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException; + + List getNameServerAddressList(); + + int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) throws RemotingCommandException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException; + + void putKVConfig(final String namespace, final String key, final String value); + + String getKVConfig(final String namespace, final String key) throws RemotingException, MQClientException, InterruptedException; + + KVTable getKVListByNamespace(final String namespace) throws RemotingException, MQClientException, InterruptedException; + + void deleteTopicInBroker(final Set addrs, final String topic) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + void deleteTopicInNameServer(final Set addrs, final String topic) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + void deleteSubscriptionGroup(final String addr, String groupName) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException; + + List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, boolean force) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + Map resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + Map> getConsumeStatus(String topic, String group, String clientAddr) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + void createOrUpdateOrderConf(String key, String value, boolean isCluster) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + GroupList queryTopicConsumeByWho(final String topic) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException, RemotingException, MQClientException; + + List queryConsumeTimeSpan(final String topic, final String group) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException; + + boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + boolean cleanUnusedTopic(String cluster) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + + boolean cleanUnusedTopicByAddr(String addr) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + ConsumerRunningInfo getConsumerRunningInfo(final String consumerGroup, final String clientId, final boolean jstack) + throws RemotingException, MQClientException, InterruptedException; + + ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, + String clientId, + String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + + ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, + String clientId, + String topic, + String msgId) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + + List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException; + + void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException; + + BrokerStatsData viewBrokerStatsData(final String brokerAddr, final String statsName, final String statsKey) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, MQClientException, + InterruptedException; + + Set getClusterList(final String topic) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + ConsumeStatsList fetchConsumeStatsInBroker(final String brokerAddr, boolean isOrder, long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQClientException, InterruptedException; + + Set getTopicClusterList(final String topic) throws InterruptedException, MQBrokerException, MQClientException, RemotingException; + + SubscriptionGroupWrapper getAllSubscriptionGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException; + + TopicConfigSerializeWrapper getAllTopicGroup(final String brokerAddr, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException; + + void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException; + + /** + * Update name server config. + *
+ * Command Code : RequestCode.UPDATE_NAMESRV_CONFIG + * + *
If param(nameServers) is null or empty, will use name servers from ns! + * + * @param properties + * @param nameServers + * + * @throws InterruptedException + * @throws RemotingConnectException + * @throws UnsupportedEncodingException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws MQClientException + * @throws MQBrokerException + */ + void updateNameServerConfig(final Properties properties, final List nameServers) throws InterruptedException, RemotingConnectException, + UnsupportedEncodingException, RemotingSendRequestException, RemotingTimeoutException, + MQClientException, MQBrokerException; + + /** + * Get name server config. + *
+ * Command Code : RequestCode.GET_NAMESRV_CONFIG + *
If param(nameServers) is null or empty, will use name servers from ns! + * + * @param nameServers + * + * @return The fetched name server config + * + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + * @throws MQClientException + * @throws UnsupportedEncodingException + */ + Map getNameServerConfig(final List nameServers) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, + MQClientException, UnsupportedEncodingException; +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java new file mode 100644 index 0000000000..bdc72882ea --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.admin.api; + +public class MessageTrack { + private String consumerGroup; + private TrackType trackType; + private String exceptionDesc; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public TrackType getTrackType() { + return trackType; + } + + + public void setTrackType(TrackType trackType) { + this.trackType = trackType; + } + + + public String getExceptionDesc() { + return exceptionDesc; + } + + + public void setExceptionDesc(String exceptionDesc) { + this.exceptionDesc = exceptionDesc; + } + + + @Override + public String toString() { + return "MessageTrack [consumerGroup=" + consumerGroup + ", trackType=" + trackType + + ", exceptionDesc=" + exceptionDesc + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java new file mode 100644 index 0000000000..ca475acbe8 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.admin.api; + +public enum TrackType { + CONSUMED, + CONSUMED_BUT_FILTERED, + PULL, + NOT_CONSUME_YET, + NOT_ONLINE, + UNKNOWN +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java new file mode 100644 index 0000000000..1b5d26438c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.MQAdminExt; + +import java.util.*; + + +/** + * @author shijia.wxr + */ +public class CommandUtil { + + public static Map/*slave addr*/> fetchMasterAndSlaveDistinguish( + final MQAdminExt adminExt, final String clusterName) + throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingSendRequestException, + MQBrokerException { + Map> masterAndSlaveMap = new HashMap>(4); + + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + + if (brokerNameSet == null) { + System.out + .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + return masterAndSlaveMap; + } + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + + if (brokerData == null || brokerData.getBrokerAddrs() == null) { + continue; + } + + String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + masterAndSlaveMap.put(masterAddr, new ArrayList()); + + for (Long id : brokerData.getBrokerAddrs().keySet()) { + if (brokerData.getBrokerAddrs().get(id) == null + || id.longValue() == MixAll.MASTER_ID) { + continue; + } + + masterAndSlaveMap.get(masterAddr).add(brokerData.getBrokerAddrs().get(id)); + } + } + + return masterAndSlaveMap; + } + + public static Set fetchMasterAddrByClusterName(final MQAdminExt adminExt, final String clusterName) + throws InterruptedException, RemotingConnectException, RemotingTimeoutException, + RemotingSendRequestException, MQBrokerException { + Set masterSet = new HashSet(); + + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + + if (brokerNameSet != null) { + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr != null) { + masterSet.add(addr); + } + } + } + } else { + System.out + .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + } + + return masterSet; + } + + public static Set fetchMasterAndSlaveAddrByClusterName(final MQAdminExt adminExt, final String clusterName) + throws InterruptedException, RemotingConnectException, RemotingTimeoutException, + RemotingSendRequestException, MQBrokerException { + Set masterSet = new HashSet(); + + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + + if (brokerNameSet != null) { + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + final Collection addrs = brokerData.getBrokerAddrs().values(); + masterSet.addAll(addrs); + } + } + } else { + System.out + .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + } + + return masterSet; + } + + + public static Set fetchBrokerNameByClusterName(final MQAdminExt adminExt, final String clusterName) + throws Exception { + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + if (brokerNameSet.isEmpty()) { + throw new Exception( + "Make sure the specified clusterName exists or the nameserver which connected is correct."); + } + return brokerNameSet; + } + + + public static String fetchBrokerNameByAddr(final MQAdminExt adminExt, final String addr) throws Exception { + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + HashMap brokerAddrTable = + clusterInfoSerializeWrapper.getBrokerAddrTable(); + Iterator> it = brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + HashMap brokerAddrs = entry.getValue().getBrokerAddrs(); + if (brokerAddrs.containsValue(addr)) + return entry.getKey(); + } + throw new Exception( + "Make sure the specified broker addr exists or the nameserver which connected is correct."); + } + + +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java new file mode 100644 index 0000000000..56ecc59364 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.command.broker.*; +import com.alibaba.rocketmq.tools.command.cluster.CLusterSendMsgRTCommand; +import com.alibaba.rocketmq.tools.command.cluster.ClusterListSubCommand; +import com.alibaba.rocketmq.tools.command.connection.ConsumerConnectionSubCommand; +import com.alibaba.rocketmq.tools.command.connection.ProducerConnectionSubCommand; +import com.alibaba.rocketmq.tools.command.consumer.*; +import com.alibaba.rocketmq.tools.command.message.*; +import com.alibaba.rocketmq.tools.command.namesrv.*; +import com.alibaba.rocketmq.tools.command.offset.CloneGroupOffsetCommand; +import com.alibaba.rocketmq.tools.command.offset.ResetOffsetByTimeCommand; +import com.alibaba.rocketmq.tools.command.stats.StatsAllSubCommand; +import com.alibaba.rocketmq.tools.command.topic.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class MQAdminStartup { + protected static List subCommandList = new ArrayList(); + + public static void main(String[] args) { + main0(args, null); + } + + public static void main0(String[] args, RPCHook rpcHook) { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + + + //PackageConflictDetect.detectFastjson(); + + initCommand(); + + try { + initLogback(); + switch (args.length) { + case 0: + printHelp(); + break; + case 2: + if (args[0].equals("help")) { + SubCommand cmd = findSubCommand(args[1]); + if (cmd != null) { + Options options = ServerUtil.buildCommandlineOptions(new Options()); + options = cmd.buildCommandlineOptions(options); + if (options != null) { + ServerUtil.printCommandLineHelp("mqadmin " + cmd.commandName(), options); + } + } else { + System.out.printf("The sub command \'" + args[1] + "\' not exist.%n"); + } + break; + } + case 1: + default: + SubCommand cmd = findSubCommand(args[0]); + if (cmd != null) { + String[] subargs = parseSubArgs(args); + + + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, cmd.buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return; + } + + if (commandLine.hasOption('n')) { + String namesrvAddr = commandLine.getOptionValue('n'); + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr); + } + + cmd.execute(commandLine, options, rpcHook); + } else { + System.out.printf("The sub command \'" + args[0] + "\' not exist.%n"); + } + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void initCommand() { + initCommand(new UpdateTopicSubCommand()); + initCommand(new DeleteTopicSubCommand()); + initCommand(new UpdateSubGroupSubCommand()); + initCommand(new DeleteSubscriptionGroupCommand()); + initCommand(new UpdateBrokerConfigSubCommand()); + initCommand(new UpdateTopicPermSubCommand()); + + initCommand(new TopicRouteSubCommand()); + initCommand(new TopicStatusSubCommand()); + initCommand(new TopicClusterSubCommand()); + + + initCommand(new BrokerStatusSubCommand()); + initCommand(new QueryMsgByIdSubCommand()); + initCommand(new QueryMsgByKeySubCommand()); + initCommand(new QueryMsgByUniqueKeySubCommand()); + initCommand(new QueryMsgByOffsetSubCommand()); + initCommand(new QueryMsgByUniqueKeySubCommand()); + initCommand(new PrintMessageSubCommand()); + initCommand(new PrintMessageByQueueCommand()); + initCommand(new SendMsgStatusCommand()); + initCommand(new BrokerConsumeStatsSubCommad()); + + + initCommand(new ProducerConnectionSubCommand()); + initCommand(new ConsumerConnectionSubCommand()); + initCommand(new ConsumerProgressSubCommand()); + initCommand(new ConsumerStatusSubCommand()); + initCommand(new CloneGroupOffsetCommand()); + + initCommand(new ClusterListSubCommand()); + initCommand(new TopicListSubCommand()); + + initCommand(new UpdateKvConfigCommand()); + initCommand(new DeleteKvConfigCommand()); + + initCommand(new WipeWritePermSubCommand()); + initCommand(new ResetOffsetByTimeCommand()); + + initCommand(new UpdateOrderConfCommand()); + initCommand(new CleanExpiredCQSubCommand()); + initCommand(new CleanUnusedTopicCommand()); + + initCommand(new StartMonitoringSubCommand()); + initCommand(new StatsAllSubCommand()); + + initCommand(new AllocateMQSubCommand()); + + initCommand(new CheckMsgSendRTCommand()); + initCommand(new CLusterSendMsgRTCommand()); + + initCommand(new GetNamesrvConfigCommand()); + initCommand(new UpdateNamesrvConfigCommand()); + initCommand(new GetBrokerConfigCommand()); + } + + private static void initLogback() throws JoranException { + String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(rocketmqHome + "/conf/logback_tools.xml"); + } + + private static void printHelp() { + System.out.printf("The most commonly used mqadmin commands are:%n"); + + for (SubCommand cmd : subCommandList) { + System.out.printf(" %-20s %s%n", cmd.commandName(), cmd.commandDesc()); + } + + System.out.printf("%nSee 'mqadmin help ' for more information on a specific command."); + } + + private static SubCommand findSubCommand(final String name) { + for (SubCommand cmd : subCommandList) { + if (cmd.commandName().toUpperCase().equals(name.toUpperCase())) { + return cmd; + } + } + + return null; + } + + private static String[] parseSubArgs(String[] args) { + if (args.length > 1) { + String[] result = new String[args.length - 1]; + for (int i = 0; i < args.length - 1; i++) { + result[i] = args[i + 1]; + } + return result; + } + return null; + } + + public static void initCommand(SubCommand command) { + subCommandList.add(command); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java new file mode 100644 index 0000000000..e28b7bc7f7 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import com.alibaba.rocketmq.remoting.RPCHook; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; + + +/** + * @author shijia.wxr + */ +public interface SubCommand { + public String commandName(); + + + public String commandDesc(); + + + public Options buildCommandlineOptions(final Options options); + + + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook); +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java new file mode 100644 index 0000000000..0f24df728b --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatsList; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.*; + + +/** + * @author shijia.wxr + */ +public class BrokerConsumeStatsSubCommad implements SubCommand { + + @Override + public String commandName() { + return "brokerConsumeStats"; + } + + @Override + public String commandDesc() { + return "Fetch broker consume stats data"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "timeoutMillis", true, "request timeout Millis"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("l", "level", true, "threshold of print diff"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("o", "order", true, "order topic"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + defaultMQAdminExt.start(); + String brokerAddr = commandLine.getOptionValue('b').trim(); + boolean isOrder = false; + long timeoutMillis = 50000; + long diffLevel = 0; + if (commandLine.hasOption('o')) { + isOrder = Boolean.parseBoolean(commandLine.getOptionValue('o').trim()); + } + if (commandLine.hasOption('t')) { + timeoutMillis = Long.parseLong(commandLine.getOptionValue('t').trim()); + } + if (commandLine.hasOption('l')) { + diffLevel = Long.parseLong(commandLine.getOptionValue('l').trim()); + } + + ConsumeStatsList consumeStatsList = defaultMQAdminExt.fetchConsumeStatsInBroker(brokerAddr, isOrder, timeoutMillis); + System.out.printf("%-32s %-32s %-32s %-4s %-20s %-20s %-20s %s%n", + "#Topic", + "#Group", + "#Broker Name", + "#QID", + "#Broker Offset", + "#Consumer Offset", + "#Diff", + "#LastTime"); + for (Map> map : consumeStatsList.getConsumeStatsList()) { + for (Map.Entry> entry : map.entrySet()) { + String group = entry.getKey(); + List consumeStatsArray = entry.getValue(); + for (ConsumeStats consumeStats : consumeStatsArray) { + List mqList = new LinkedList(); + mqList.addAll(consumeStats.getOffsetTable().keySet()); + Collections.sort(mqList); + for (MessageQueue mq : mqList) { + OffsetWrapper offsetWrapper = consumeStats.getOffsetTable().get(mq); + long diff = offsetWrapper.getBrokerOffset() - offsetWrapper.getConsumerOffset(); + + if (diff < diffLevel) { + continue; + } + String lastTime = "-"; + try { + lastTime = UtilAll.formatDate(new Date(offsetWrapper.getLastTimestamp()), UtilAll.YYYY_MM_DD_HH_MM_SS); + } catch (Exception e) { + + } + if (offsetWrapper.getLastTimestamp() > 0) + System.out.printf("%-32s %-32s %-32s %-4d %-20d %-20d %-20d %s%n", + UtilAll.frontStringAtLeast(mq.getTopic(), 32), + group, + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), + mq.getQueueId(), + offsetWrapper.getBrokerOffset(), + offsetWrapper.getConsumerOffset(), + diff, + lastTime + ); + } + } + } + } + System.out.printf("%nDiff Total: %d%n", consumeStatsList.getTotalDiff()); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java new file mode 100644 index 0000000000..5f5409b51e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + + +/** + * @author shijia.wxr + */ +public class BrokerStatusSubCommand implements SubCommand { + + @Override + public String commandName() { + return "brokerStatus"; + } + + + @Override + public String commandDesc() { + return "Fetch broker runtime status data"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "which cluster"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String brokerAddr = commandLine.hasOption('b') ? commandLine.getOptionValue('b').trim() : null; + String clusterName = commandLine.hasOption('c') ? commandLine.getOptionValue('c').trim() : null; + if (brokerAddr != null) { + printBrokerRuntimeStats(defaultMQAdminExt, brokerAddr, false); + } else if (clusterName != null) { + Set masterSet = + CommandUtil.fetchMasterAndSlaveAddrByClusterName(defaultMQAdminExt, clusterName); + for (String ba : masterSet) { + try { + printBrokerRuntimeStats(defaultMQAdminExt, ba, true); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + public void printBrokerRuntimeStats(final DefaultMQAdminExt defaultMQAdminExt, final String brokerAddr, final boolean printBroker) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(brokerAddr); + + TreeMap tmp = new TreeMap(); + tmp.putAll(kvTable.getTable()); + + Iterator> it = tmp.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (printBroker) { + System.out.printf("%-24s %-32s: %s%n", brokerAddr, next.getKey(), next.getValue()); + } else { + System.out.printf("%-32s: %s%n", next.getKey(), next.getValue()); + } + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java new file mode 100644 index 0000000000..c2918c12de --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * @author lansheng.zj + */ +public class CleanExpiredCQSubCommand implements SubCommand { + + @Override + public String commandName() { + return "cleanExpiredCQ"; + } + + + @Override + public String commandDesc() { + return "Clean expired ConsumeQueue on broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "clustername"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + boolean result = false; + defaultMQAdminExt.start(); + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + result = defaultMQAdminExt.cleanExpiredConsumerQueueByAddr(addr); + + } else { + String cluster = commandLine.getOptionValue('c'); + if (null != cluster) + cluster = cluster.trim(); + result = defaultMQAdminExt.cleanExpiredConsumerQueue(cluster); + } + System.out.printf(result ? "success" : "false"); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanUnusedTopicCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanUnusedTopicCommand.java new file mode 100644 index 0000000000..f7b543f739 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanUnusedTopicCommand.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * @author lansheng.zj + */ +public class CleanUnusedTopicCommand implements SubCommand { + + @Override + public String commandName() { + return "cleanUnusedTopic"; + } + + + @Override + public String commandDesc() { + return "Clean unused topic on broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "cluster name"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + boolean result = false; + defaultMQAdminExt.start(); + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + result = defaultMQAdminExt.cleanUnusedTopicByAddr(addr); + + } else { + String cluster = commandLine.getOptionValue('c'); + if (null != cluster) + cluster = cluster.trim(); + result = defaultMQAdminExt.cleanUnusedTopicByAddr(cluster); + } + System.out.printf(result ? "success" : "false"); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/GetBrokerConfigCommand.java new file mode 100644 index 0000000000..703d69bebe --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/GetBrokerConfigCommand.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.admin.MQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * @author xigu.lx + */ +public class GetBrokerConfigCommand implements SubCommand { + @Override + public String commandName() { + return "getBrokerConfig"; + } + + @Override + public String commandDesc() { + return "Get broker config by cluster or special broker!"; + } + + @Override + public Options buildCommandlineOptions(final Options options) { + Option opt = new Option("b", "brokerAddr", true, "update which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "update which cluster"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, final RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + + if (commandLine.hasOption('b')) { + String brokerAddr = commandLine.getOptionValue('b').trim(); + defaultMQAdminExt.start(); + + getAndPrint(defaultMQAdminExt, + String.format("============%s============\n", brokerAddr), + brokerAddr); + + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + defaultMQAdminExt.start(); + + Map> masterAndSlaveMap + = CommandUtil.fetchMasterAndSlaveDistinguish(defaultMQAdminExt, clusterName); + + for (String masterAddr : masterAndSlaveMap.keySet()) { + + getAndPrint( + defaultMQAdminExt, + String.format("============Master: %s============\n", masterAddr), + masterAddr + ); + for (String slaveAddr : masterAndSlaveMap.get(masterAddr)) { + + getAndPrint( + defaultMQAdminExt, + String.format("============My Master: %s=====Slave: %s============\n", masterAddr, slaveAddr), + slaveAddr + ); + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + protected void getAndPrint(final MQAdminExt defaultMQAdminExt, final String printPrefix, final String addr) + throws InterruptedException, RemotingConnectException, + UnsupportedEncodingException, RemotingTimeoutException, + MQBrokerException, RemotingSendRequestException { + + System.out.print(printPrefix); + + Properties properties = defaultMQAdminExt.getBrokerConfig(addr); + if (properties == null) { + System.out.printf("Broker[%s] has no config property!\n", addr); + return; + } + + for (Object key : properties.keySet()) { + System.out.printf("%-50s= %s\n", key, properties.get(key)); + } + + System.out.printf("%n"); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/SendMsgStatusCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/SendMsgStatusCommand.java new file mode 100644 index 0000000000..165e397a6d --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/SendMsgStatusCommand.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.UnsupportedEncodingException; + + +/** + * @author lansheng.zj + */ +public class SendMsgStatusCommand implements SubCommand { + + @Override + public String commandName() { + return "sendMsgStatus"; + } + + + @Override + public String commandDesc() { + return "send msg to broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerName", true, "Broker Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "messageSize", true, "Message Size, Default: 128"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "count", true, "send message count, Default: 50"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + final DefaultMQProducer producer = new DefaultMQProducer("PID_SMSC", rpcHook); + producer.setInstanceName("PID_SMSC_" + System.currentTimeMillis()); + + try { + producer.start(); + String brokerName = commandLine.getOptionValue('b').trim(); + int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 128; + int count = commandLine.hasOption('c') ? Integer.parseInt(commandLine.getOptionValue('c')) : 50; + + producer.send(buildMessage(brokerName, 16)); + + for (int i = 0; i < count; i++) { + long begin = System.currentTimeMillis(); + SendResult result = producer.send(buildMessage(brokerName, messageSize)); + System.out.printf("rt:" + (System.currentTimeMillis() - begin) + "ms, SendResult=" + result); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + producer.shutdown(); + } + } + + + private static Message buildMessage(final String topic, final int messageSize) throws UnsupportedEncodingException { + Message msg = new Message(); + msg.setTopic(topic); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 11) { + sb.append("hello jodie"); + } + msg.setBody(sb.toString().getBytes(MixAll.DEFAULT_CHARSET)); + return msg; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java new file mode 100644 index 0000000000..86938a7b8b --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Properties; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class UpdateBrokerConfigSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateBrokerConfig"; + } + + + @Override + public String commandDesc() { + return "Update broker's config"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "update which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "update which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("k", "key", true, "config key"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "value", true, "config value"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String key = commandLine.getOptionValue('k').trim(); + String value = commandLine.getOptionValue('v').trim(); + Properties properties = new Properties(); + properties.put(key, value); + + if (commandLine.hasOption('b')) { + String brokerAddr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + + defaultMQAdminExt.updateBrokerConfig(brokerAddr, properties); + System.out.printf("update broker config success, %s\n", brokerAddr); + return; + + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String brokerAddr : masterSet) { + try { + defaultMQAdminExt.updateBrokerConfig(brokerAddr, properties); + System.out.printf("update broker config success, %s\n", brokerAddr); + } catch (Exception e) { + e.printStackTrace(); + } + } + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java new file mode 100644 index 0000000000..3a285225a7 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.cluster; + +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * @author fengliang.hfl + */ +public class CLusterSendMsgRTCommand implements SubCommand { + + public static void main(String args[]) { + } + + @Override + public String commandName() { + return "clusterRT"; + } + + @Override + public String commandDesc() { + return "List All clusters Message Send RT"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("a", "amout", true, "message amout | default 100"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "size", true, "message size | default 128 Byte"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "cluster name | default display all cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "print log", true, "print as tlog | default false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "machine room", true, "machine room name | default noname"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "interval", true, "print interval | default 10 seconds"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + DefaultMQProducer producer = new DefaultMQProducer(rpcHook); + producer.setProducerGroup(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + producer.start(); + + ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo(); + HashMap> clusterAddr = clusterInfoSerializeWrapper + .getClusterAddrTable(); + + Set clusterNames = null; + + long amount = !commandLine.hasOption('a') ? 50 : Long.parseLong(commandLine + .getOptionValue('a').trim()); + + long size = !commandLine.hasOption('s') ? 128 : Long.parseLong(commandLine + .getOptionValue('s').trim()); + + long interval = !commandLine.hasOption('i') ? 10 : Long.parseLong(commandLine + .getOptionValue('i').trim()); + + boolean printAsTlog = !commandLine.hasOption('p') ? false : Boolean + .parseBoolean(commandLine.getOptionValue('p').trim()); + + String machineRoom = !commandLine.hasOption('m') ? "noname" : commandLine + .getOptionValue('m').trim(); + + if (commandLine.hasOption('c')) { + clusterNames = new TreeSet(); + clusterNames.add(commandLine.getOptionValue('c').trim()); + } else { + clusterNames = clusterAddr.keySet(); + } + + if (!printAsTlog) { + System.out.printf("%-24s %-24s %-4s %-8s %-8s%n", + "#Cluster Name", + "#Broker Name", + "#RT", + "#successCount", + "#failCount" + ); + } + + while (true) { + for (String clusterName : clusterNames) { + Set brokerNames = clusterAddr.get(clusterName); + if (brokerNames == null) { + System.out.printf("cluster [%s] not exist", clusterName); + break; + } + + for (String brokerName : brokerNames) { + Message msg = new Message(brokerName, getStringBySize(size).getBytes(MixAll.DEFAULT_CHARSET)); + long start = 0; + long end = 0; + long elapsed = 0; + int successCount = 0; + int failCount = 0; + + for (int i = 0; i < amount; i++) { + start = System.currentTimeMillis(); + try { + producer.send(msg); + successCount++; + end = System.currentTimeMillis(); + } catch (Exception e) { + failCount++; + end = System.currentTimeMillis(); + } + + if (i != 0) { + elapsed += end - start; + } + } + + double rt = (double) elapsed / (amount - 1); + if (!printAsTlog) { + System.out.printf("%-24s %-24s %-8s %-16s %-16s%n", + clusterName, + brokerName, + String.format("%.2f", rt), + successCount, + failCount + ); + } else { + System.out.printf(String.format("%s|%s|%s|%s|%s%n", getCurTime(), + machineRoom, clusterName, brokerName, + new BigDecimal(rt).setScale(0, BigDecimal.ROUND_HALF_UP))); + } + + } + + } + + Thread.sleep(interval * 1000); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + producer.shutdown(); + } + } + + public String getStringBySize(long size) { + StringBuilder res = new StringBuilder(); + for (int i = 0; i < size; i++) { + res.append('a'); + } + return res.toString(); + } + + public String getCurTime() { + String fromTimeZone = "GMT+8"; + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date date = new Date(); + format.setTimeZone(TimeZone.getTimeZone(fromTimeZone)); + String chinaDate = format.format(date); + return chinaDate; + } + +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java new file mode 100644 index 0000000000..baf4f3c9f5 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java @@ -0,0 +1,280 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.cluster; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + + +/** + * @author shijia.wxr + */ +public class ClusterListSubCommand implements SubCommand { + + @Override + public String commandName() { + return "clusterList"; + } + + + @Override + public String commandDesc() { + return "List all of clusters"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("m", "moreStats", false, "Print more stats"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "interval", true, "specify intervals numbers, it is in seconds"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + long printInterval = 1; + boolean enableInterval = commandLine.hasOption('i'); + + if (enableInterval) { + printInterval = Long.parseLong(commandLine.getOptionValue('i')) * 1000; + } + + try { + defaultMQAdminExt.start(); + long i = 0; + + do { + if (i++ > 0) { + Thread.sleep(printInterval); + } + if (commandLine.hasOption('m')) { + this.printClusterMoreStats(defaultMQAdminExt); + } else { + this.printClusterBaseInfo(defaultMQAdminExt); + } + } while (enableInterval); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + private void printClusterMoreStats(final DefaultMQAdminExt defaultMQAdminExt) throws RemotingConnectException, + RemotingTimeoutException, RemotingSendRequestException, InterruptedException, MQBrokerException { + + ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-16s %-32s %14s %14s %14s %14s%n", + "#Cluster Name", + "#Broker Name", + "#InTotalYest", + "#OutTotalYest", + "#InTotalToday", + "#OutTotalToday" + ); + + Iterator>> itCluster = clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator(); + while (itCluster.hasNext()) { + Map.Entry> next = itCluster.next(); + String clusterName = next.getKey(); + TreeSet brokerNameSet = new TreeSet(); + brokerNameSet.addAll(next.getValue()); + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + Iterator> itAddr = brokerData.getBrokerAddrs().entrySet().iterator(); + while (itAddr.hasNext()) { + Map.Entry next1 = itAddr.next(); + long inTotalYest = 0; + long outTotalYest = 0; + long inTotalToday = 0; + long outTotalToday = 0; + + try { + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue()); + String msgPutTotalYesterdayMorning = kvTable.getTable().get("msgPutTotalYesterdayMorning"); + String msgPutTotalTodayMorning = kvTable.getTable().get("msgPutTotalTodayMorning"); + String msgPutTotalTodayNow = kvTable.getTable().get("msgPutTotalTodayNow"); + String msgGetTotalYesterdayMorning = kvTable.getTable().get("msgGetTotalYesterdayMorning"); + String msgGetTotalTodayMorning = kvTable.getTable().get("msgGetTotalTodayMorning"); + String msgGetTotalTodayNow = kvTable.getTable().get("msgGetTotalTodayNow"); + + inTotalYest = Long.parseLong(msgPutTotalTodayMorning) - Long.parseLong(msgPutTotalYesterdayMorning); + outTotalYest = Long.parseLong(msgGetTotalTodayMorning) - Long.parseLong(msgGetTotalYesterdayMorning); + + inTotalToday = Long.parseLong(msgPutTotalTodayNow) - Long.parseLong(msgPutTotalTodayMorning); + outTotalToday = Long.parseLong(msgGetTotalTodayNow) - Long.parseLong(msgGetTotalTodayMorning); + + } catch (Exception e) { + } + + System.out.printf("%-16s %-32s %14d %14d %14d %14d%n", + clusterName, + brokerName, + inTotalYest, + outTotalYest, + inTotalToday, + outTotalToday + ); + } + } + } + + if (itCluster.hasNext()) { + System.out.printf(""); + } + } + } + + private void printClusterBaseInfo(final DefaultMQAdminExt defaultMQAdminExt) throws RemotingConnectException, RemotingTimeoutException, + RemotingSendRequestException, InterruptedException, MQBrokerException { + + ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-16s %-22s %-4s %-22s %-16s %19s %19s %10s %5s %6s%n", + "#Cluster Name", + "#Broker Name", + "#BID", + "#Addr", + "#Version", + "#InTPS(LOAD)", + "#OutTPS(LOAD)", + "#PCWait(ms)", + "#Hour", + "#SPACE" + ); + + Iterator>> itCluster = clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator(); + while (itCluster.hasNext()) { + Map.Entry> next = itCluster.next(); + String clusterName = next.getKey(); + TreeSet brokerNameSet = new TreeSet(); + brokerNameSet.addAll(next.getValue()); + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + Iterator> itAddr = brokerData.getBrokerAddrs().entrySet().iterator(); + while (itAddr.hasNext()) { + Map.Entry next1 = itAddr.next(); + double in = 0; + double out = 0; + String version = ""; + String sendThreadPoolQueueSize = ""; + String pullThreadPoolQueueSize = ""; + String sendThreadPoolQueueHeadWaitTimeMills = ""; + String pullThreadPoolQueueHeadWaitTimeMills = ""; + String pageCacheLockTimeMills = ""; + String earliestMessageTimeStamp = ""; + String commitLogDiskRatio = ""; + try { + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue()); + String putTps = kvTable.getTable().get("putTps"); + String getTransferedTps = kvTable.getTable().get("getTransferedTps"); + sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize"); + pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize"); + + sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize"); + pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize"); + + sendThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("sendThreadPoolQueueHeadWaitTimeMills"); + pullThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("pullThreadPoolQueueHeadWaitTimeMills"); + pageCacheLockTimeMills = kvTable.getTable().get("pageCacheLockTimeMills"); + earliestMessageTimeStamp = kvTable.getTable().get("earliestMessageTimeStamp"); + commitLogDiskRatio = kvTable.getTable().get("commitLogDiskRatio"); + + version = kvTable.getTable().get("brokerVersionDesc"); + { + String[] tpss = putTps.split(" "); + if (tpss != null && tpss.length > 0) { + in = Double.parseDouble(tpss[0]); + } + } + + { + String[] tpss = getTransferedTps.split(" "); + if (tpss != null && tpss.length > 0) { + out = Double.parseDouble(tpss[0]); + } + } + } catch (Exception e) { + } + + double hour = 0.0; + double space = 0.0; + + if (earliestMessageTimeStamp != null && earliestMessageTimeStamp.length() > 0) { + long mills = System.currentTimeMillis() - Long.valueOf(earliestMessageTimeStamp); + hour = mills / 1000.0 / 60.0 / 60.0; + } + + if (commitLogDiskRatio != null && commitLogDiskRatio.length() > 0) { + space = Double.valueOf(commitLogDiskRatio); + } + + System.out.printf("%-16s %-22s %-4s %-22s %-16s %19s %19s %10s %5s %6s%n", + clusterName, + brokerName, + next1.getKey().longValue(), + next1.getValue(), + version, + String.format("%9.2f(%s,%sms)", in, sendThreadPoolQueueSize, sendThreadPoolQueueHeadWaitTimeMills), + String.format("%9.2f(%s,%sms)", out, pullThreadPoolQueueSize, pullThreadPoolQueueHeadWaitTimeMills), + pageCacheLockTimeMills, + String.format("%2.2f", hour), + String.format("%.4f", space) + ); + } + } + } + + if (itCluster.hasNext()) { + System.out.printf(""); + } + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java new file mode 100644 index 0000000000..aa0598e433 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.connection; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map.Entry; + + +/** + * @author shijia.wxr + */ +public class ConsumerConnectionSubCommand implements SubCommand { + + @Override + public String commandName() { + return "consumerConnection"; + } + + @Override + public String commandDesc() { + return "Query consumer's socket connection, client version and subscription"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String group = commandLine.getOptionValue('g').trim(); + + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(group); + + + int i = 1; + for (Connection conn : cc.getConnectionSet()) { + System.out.printf("%03d %-32s %-22s %-8s %s%n", + i++, + conn.getClientId(), + conn.getClientAddr(), + conn.getLanguage(), + MQVersion.getVersionDesc(conn.getVersion()) + ); + } + + System.out.printf("%nBelow is subscription:"); + Iterator> it = cc.getSubscriptionTable().entrySet().iterator(); + i = 1; + while (it.hasNext()) { + Entry entry = it.next(); + SubscriptionData sd = entry.getValue(); + System.out.printf("%03d Topic: %-40s SubExpression: %s%n", + i++, + sd.getTopic(), + sd.getSubString() + ); + } + + System.out.printf(""); + System.out.printf("ConsumeType: %s%n", cc.getConsumeType()); + System.out.printf("MessageModel: %s%n", cc.getMessageModel()); + System.out.printf("ConsumeFromWhere: %s%n", cc.getConsumeFromWhere()); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java new file mode 100644 index 0000000000..97ba792f97 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.connection; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * @author shijia.wxr + */ +public class ProducerConnectionSubCommand implements SubCommand { + + @Override + public String commandName() { + return "producerConnection"; + } + + @Override + public String commandDesc() { + return "Query producer's socket connection and client version"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "producerGroup", true, "producer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String group = commandLine.getOptionValue('g').trim(); + String topic = commandLine.getOptionValue('t').trim(); + + ProducerConnection pc = defaultMQAdminExt.examineProducerConnectionInfo(group, topic); + + int i = 1; + for (Connection conn : pc.getConnectionSet()) { + System.out.printf("%04d %-32s %-22s %-8s %s%n", + i++, + conn.getClientId(), + conn.getClientAddr(), + conn.getLanguage(), + MQVersion.getVersionDesc(conn.getVersion()) + ); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java new file mode 100644 index 0000000000..d09b74a3ce --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class ConsumerProgressSubCommand implements SubCommand { + private final Logger log = ClientLogger.getLog(); + + @Override + public String commandName() { + return "consumerProgress"; + } + + @Override + public String commandDesc() { + return "Query consumers's progress, speed"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "groupName", true, "consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + if (commandLine.hasOption('g')) { + String consumerGroup = commandLine.getOptionValue('g').trim(); + ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); + List mqList = new LinkedList(); + mqList.addAll(consumeStats.getOffsetTable().keySet()); + Collections.sort(mqList); + + System.out.printf("%-32s %-32s %-4s %-20s %-20s %-20s %s%n", + "#Topic", + "#Broker Name", + "#QID", + "#Broker Offset", + "#Consumer Offset", + "#Diff", + "#LastTime"); + + long diffTotal = 0L; + for (MessageQueue mq : mqList) { + OffsetWrapper offsetWrapper = consumeStats.getOffsetTable().get(mq); + long diff = offsetWrapper.getBrokerOffset() - offsetWrapper.getConsumerOffset(); + diffTotal += diff; + String lastTime = ""; + try { + lastTime = UtilAll.formatDate(new Date(offsetWrapper.getLastTimestamp()), UtilAll.YYYY_MM_DD_HH_MM_SS); + } catch (Exception e) { + } + System.out.printf("%-32s %-32s %-4d %-20d %-20d %-20d %s%n", + UtilAll.frontStringAtLeast(mq.getTopic(), 32), + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), + mq.getQueueId(), + offsetWrapper.getBrokerOffset(), + offsetWrapper.getConsumerOffset(), + diff, + lastTime + ); + } + + System.out.printf("%n"); + System.out.printf("Consume TPS: %s%n", consumeStats.getConsumeTps()); + System.out.printf("Diff Total: %d%n", diffTotal); + } else { + System.out.printf("%-32s %-6s %-24s %-5s %-14s %-7s %s%n", + "#Group", + "#Count", + "#Version", + "#Type", + "#Model", + "#TPS", + "#Diff Total" + ); + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String consumerGroup = topic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + try { + ConsumeStats consumeStats = null; + try { + consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); + } catch (Exception e) { + log.warn("examineConsumeStats exception, " + consumerGroup, e); + } + + ConsumerConnection cc = null; + try { + cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + } catch (Exception e) { + log.warn("examineConsumerConnectionInfo exception, " + consumerGroup, e); + } + + GroupConsumeInfo groupConsumeInfo = new GroupConsumeInfo(); + groupConsumeInfo.setGroup(consumerGroup); + + if (consumeStats != null) { + groupConsumeInfo.setConsumeTps((int) consumeStats.getConsumeTps()); + groupConsumeInfo.setDiffTotal(consumeStats.computeTotalDiff()); + } + + if (cc != null) { + groupConsumeInfo.setCount(cc.getConnectionSet().size()); + groupConsumeInfo.setMessageModel(cc.getMessageModel()); + groupConsumeInfo.setConsumeType(cc.getConsumeType()); + groupConsumeInfo.setVersion(cc.computeMinVersion()); + } + + System.out.printf("%-32s %-6d %-24s %-5s %-14s %-7d %d%n", + UtilAll.frontStringAtLeast(groupConsumeInfo.getGroup(), 32), + groupConsumeInfo.getCount(), + groupConsumeInfo.getCount() > 0 ? groupConsumeInfo.versionDesc() : "OFFLINE", + groupConsumeInfo.consumeTypeDesc(), + groupConsumeInfo.messageModelDesc(), + groupConsumeInfo.getConsumeTps(), + groupConsumeInfo.getDiffTotal() + ); + } catch (Exception e) { + log.warn("examineConsumeStats or examineConsumerConnectionInfo exception, " + consumerGroup, e); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} + + +class GroupConsumeInfo implements Comparable { + private String group; + private int version; + private int count; + private ConsumeType consumeType; + private MessageModel messageModel; + private int consumeTps; + private long diffTotal; + + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String consumeTypeDesc() { + if (this.count != 0) { + return this.getConsumeType() == ConsumeType.CONSUME_ACTIVELY ? "PULL" : "PUSH"; + } + return ""; + } + + public ConsumeType getConsumeType() { + return consumeType; + } + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + public String messageModelDesc() { + if (this.count != 0 && this.getConsumeType() == ConsumeType.CONSUME_PASSIVELY) { + return this.getMessageModel().toString(); + } + return ""; + } + + public MessageModel getMessageModel() { + return messageModel; + } + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + public String versionDesc() { + if (this.count != 0) { + return MQVersion.getVersionDesc(this.version); + } + return ""; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + + public long getDiffTotal() { + return diffTotal; + } + + + public void setDiffTotal(long diffTotal) { + this.diffTotal = diffTotal; + } + + + @Override + public int compareTo(GroupConsumeInfo o) { + if (this.count != o.count) { + return o.count - this.count; + } + + return (int) (o.diffTotal - diffTotal); + } + + + public int getConsumeTps() { + return consumeTps; + } + + + public void setConsumeTps(int consumeTps) { + this.consumeTps = consumeTps; + } + + + public int getVersion() { + return version; + } + + + public void setVersion(int version) { + this.version = version; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java new file mode 100644 index 0000000000..cf796d8f4f --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java @@ -0,0 +1,143 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.MQAdminStartup; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerStatusSubCommand implements SubCommand { + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + MQAdminStartup.main(new String[]{new ConsumerStatusSubCommand().commandName(), "-g", "benchmark_consumer"}); + } + + @Override + public String commandName() { + return "consumerStatus"; + } + + @Override + public String commandDesc() { + return "Query consumer's internal data structure"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "clientId", true, "The consumer's client id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "jstack", false, "Run jstack command in the consumer progress"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String group = commandLine.getOptionValue('g').trim(); + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(group); + boolean jstack = commandLine.hasOption('s'); + if (!commandLine.hasOption('i')) { + int i = 1; + long now = System.currentTimeMillis(); + final TreeMap criTable = new TreeMap(); + for (Connection conn : cc.getConnectionSet()) { + try { + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, conn.getClientId(), jstack); + if (consumerRunningInfo != null) { + criTable.put(conn.getClientId(), consumerRunningInfo); + String filePath = now + "/" + conn.getClientId(); + MixAll.string2FileNotSafe(consumerRunningInfo.formatString(), filePath); + System.out.printf("%03d %-40s %-20s %s%n", + i++, + conn.getClientId(), + MQVersion.getVersionDesc(conn.getVersion()), + filePath); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + if (!criTable.isEmpty()) { + boolean subSame = ConsumerRunningInfo.analyzeSubscription(criTable); + + boolean rebalanceOK = subSame && ConsumerRunningInfo.analyzeRebalance(criTable); + + if (subSame) { + System.out.printf("%n%nSame subscription in the same group of consumer"); + System.out.printf("%n%nRebalance %s%n", rebalanceOK ? "OK" : "Failed"); + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String result = + ConsumerRunningInfo.analyzeProcessQueue(next.getKey(), next.getValue()); + if (result.length() > 0) { + System.out.printf(result); + } + } + } else { + System.out.printf("%n%nWARN: Different subscription in the same group of consumer!!!"); + } + } + } else { + String clientId = commandLine.getOptionValue('i').trim(); + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, clientId, jstack); + if (consumerRunningInfo != null) { + System.out.printf("%s", consumerRunningInfo.formatString()); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerSubCommand.java new file mode 100644 index 0000000000..373da1e96e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerSubCommand.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.MQAdminStartup; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + + +/** + * @author shijia.wxr + */ +public class ConsumerSubCommand implements SubCommand { + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + MQAdminStartup.main(new String[]{new ConsumerSubCommand().commandName(), "-g", "benchmark_consumer"}); + } + + @Override + public String commandName() { + return "consumer"; + } + + @Override + public String commandDesc() { + return "Query consumer's connection, status, etc."; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "jstack", false, "Run jstack command in the consumer progress"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String group = commandLine.getOptionValue('g').trim(); + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(group); + boolean jstack = commandLine.hasOption('s'); + + if (!commandLine.hasOption('i')) { + + int i = 1; + long now = System.currentTimeMillis(); + final TreeMap criTable = + new TreeMap(); + for (Connection conn : cc.getConnectionSet()) { + try { + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, conn.getClientId(), jstack); + if (consumerRunningInfo != null) { + criTable.put(conn.getClientId(), consumerRunningInfo); + String filePath = now + "/" + conn.getClientId(); + MixAll.string2FileNotSafe(consumerRunningInfo.formatString(), filePath); + System.out.printf("%03d %-40s %-20s %s%n", + i++, + conn.getClientId(), + MQVersion.getVersionDesc(conn.getVersion()), + filePath); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + if (!criTable.isEmpty()) { + boolean subSame = ConsumerRunningInfo.analyzeSubscription(criTable); + boolean rebalanceOK = subSame && ConsumerRunningInfo.analyzeRebalance(criTable); + + if (subSame) { + System.out.printf("%n%nSame subscription in the same group of consumer"); + System.out.printf("%n%nRebalance %s%n", rebalanceOK ? "OK" : "Failed"); + + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String result = + ConsumerRunningInfo.analyzeProcessQueue(next.getKey(), next.getValue()); + if (result.length() > 0) { + System.out.printf(result); + } + } + } else { + System.out.printf("%n%nWARN: Different subscription in the same group of consumer!!!"); + } + } + } else { + String clientId = commandLine.getOptionValue('i').trim(); + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, clientId, jstack); + if (consumerRunningInfo != null) { + System.out.printf(consumerRunningInfo.formatString()); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java new file mode 100644 index 0000000000..712a0d00ec --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import com.alibaba.rocketmq.tools.command.topic.DeleteTopicSubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Set; + + +/** + * @author lansheng.zj + */ +public class DeleteSubscriptionGroupCommand implements SubCommand { + @Override + public String commandName() { + return "deleteSubGroup"; + } + + + @Override + public String commandDesc() { + return "Delete subscription group from broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "delete subscription group from which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "delete subscription group from which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "groupName", true, "subscription group name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); + adminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // groupName + String groupName = commandLine.getOptionValue('g').trim(); + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + adminExt.start(); + + adminExt.deleteSubscriptionGroup(addr, groupName); + System.out.printf("delete subscription group [%s] from broker [%s] success.%n", groupName, + addr); + + return; + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + adminExt.start(); + + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); + for (String master : masterSet) { + adminExt.deleteSubscriptionGroup(master, groupName); + System.out.printf( + "delete subscription group [%s] from broker [%s] in cluster [%s] success.%n", + groupName, master, clusterName); + } + + try { + DeleteTopicSubCommand.deleteTopic(adminExt, clusterName, MixAll.RETRY_GROUP_TOPIC_PREFIX + + groupName); + DeleteTopicSubCommand.deleteTopic(adminExt, clusterName, MixAll.DLQ_GROUP_TOPIC_PREFIX + + groupName); + } catch (Exception e) { + } + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + adminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java new file mode 100644 index 0000000000..8e4bba853f --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import com.alibaba.rocketmq.tools.monitor.DefaultMonitorListener; +import com.alibaba.rocketmq.tools.monitor.MonitorConfig; +import com.alibaba.rocketmq.tools.monitor.MonitorService; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; + + +/** + * @author shijia.wxr + */ +public class StartMonitoringSubCommand implements SubCommand { + private final Logger log = ClientLogger.getLog(); + + + @Override + public String commandName() { + return "startMonitoring"; + } + + + @Override + public String commandDesc() { + return "Start Monitoring"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + try { + MonitorService monitorService = + new MonitorService(new MonitorConfig(), new DefaultMonitorListener(), rpcHook); + + monitorService.start(); + } catch (Exception e) { + e.printStackTrace(); + } finally { + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java new file mode 100644 index 0000000000..683e0fee05 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class UpdateSubGroupSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateSubGroup"; + } + + + @Override + public String commandDesc() { + return "Update or create subscription group"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "create subscription group to which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "create subscription group to which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "groupName", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "consumeEnable", true, "consume enable"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "consumeFromMinEnable", true, "from min offset"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "consumeBroadcastEnable", true, "broadcast"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("q", "retryQueueNums", true, "retry queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("r", "retryMaxTimes", true, "retry max times"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "brokerId", true, "consumer from which broker id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("w", "whichBrokerWhenConsumeSlowly", true, "which broker id when consume slowly"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("a", "notifyConsumerIdsChanged", true, "notify consumerId changed"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setConsumeBroadcastEnable(false); + subscriptionGroupConfig.setConsumeFromMinEnable(false); + + // groupName + subscriptionGroupConfig.setGroupName(commandLine.getOptionValue('g').trim()); + + // consumeEnable + if (commandLine.hasOption('s')) { + subscriptionGroupConfig.setConsumeEnable(Boolean.parseBoolean(commandLine.getOptionValue('s') + .trim())); + } + + // consumeFromMinEnable + if (commandLine.hasOption('m')) { + subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.parseBoolean(commandLine + .getOptionValue('m').trim())); + } + + // consumeBroadcastEnable + if (commandLine.hasOption('d')) { + subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.parseBoolean(commandLine + .getOptionValue('d').trim())); + } + + // retryQueueNums + if (commandLine.hasOption('q')) { + subscriptionGroupConfig.setRetryQueueNums(Integer.parseInt(commandLine.getOptionValue('q') + .trim())); + } + + // retryMaxTimes + if (commandLine.hasOption('r')) { + subscriptionGroupConfig.setRetryMaxTimes(Integer.parseInt(commandLine.getOptionValue('r') + .trim())); + } + + // brokerId + if (commandLine.hasOption('i')) { + subscriptionGroupConfig.setBrokerId(Long.parseLong(commandLine.getOptionValue('i').trim())); + } + + // whichBrokerWhenConsumeSlowly + if (commandLine.hasOption('w')) { + subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(Long.parseLong(commandLine + .getOptionValue('w').trim())); + } + + // notifyConsumerIdsChanged + if (commandLine.hasOption('a')) { + subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(Boolean.parseBoolean(commandLine + .getOptionValue('a').trim())); + } + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig); + System.out.printf("create subscription group to %s success.%n", addr); + System.out.printf("%s", subscriptionGroupConfig); + return; + + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + try { + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig); + System.out.printf("create subscription group to %s success.%n", addr); + } catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000 * 1); + } + } + System.out.printf("%s", subscriptionGroupConfig); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSendRTCommand.java new file mode 100644 index 0000000000..58c2fbfa16 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSendRTCommand.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.List; + +public class CheckMsgSendRTCommand implements SubCommand { + private static String brokerName = ""; + private static int queueId = 0; + + @Override + public String commandName() { + return "checkMsgSendRT"; + } + + @Override + public String commandDesc() { + return "check message send response time"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("a", "amout", true, "message amout | default 100"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "size", true, "message size | default 128 Byte"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQProducer producer = new DefaultMQProducer(rpcHook); + producer.setProducerGroup(Long.toString(System.currentTimeMillis())); + + try { + producer.start(); + long start = 0; + long end = 0; + long timeElapsed = 0; + boolean sendSuccess = false; + String topic = commandLine.getOptionValue('t').trim(); + long amount = !commandLine.hasOption('a') ? 100 : Long.parseLong(commandLine + .getOptionValue('a').trim()); + long msgSize = !commandLine.hasOption('s') ? 128 : Long.parseLong(commandLine + .getOptionValue('s').trim()); + Message msg = new Message(topic, getStringBySize(msgSize).getBytes(MixAll.DEFAULT_CHARSET)); + + System.out.printf("%-32s %-4s %-20s %s%n", + "#Broker Name", + "#QID", + "#Send Result", + "#RT" + ); + for (int i = 0; i < amount; i++) { + start = System.currentTimeMillis(); + try { + producer.send(msg, new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + int queueIndex = (Integer) arg % mqs.size(); + MessageQueue queue = mqs.get(queueIndex); + brokerName = queue.getBrokerName(); + queueId = queue.getQueueId(); + return queue; + } + }, i); + sendSuccess = true; + end = System.currentTimeMillis(); + } catch (Exception e) { + sendSuccess = false; + end = System.currentTimeMillis(); + } + + + if (i != 0) { + timeElapsed += end - start; + } + + System.out.printf("%-32s %-4s %-20s %s%n", + brokerName, + queueId, + sendSuccess, + end - start + ); + } + + double rt = (double) timeElapsed / (amount - 1); + System.out.printf("Avg RT: %s%n", String.format("%.2f", rt)); + } catch (Exception e) { + e.printStackTrace(); + } finally { + producer.shutdown(); + } + } + + public String getStringBySize(long size) { + StringBuilder res = new StringBuilder(); + for (int i = 0; i < size; i++) { + res.append('a'); + } + return res.toString(); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/DecodeMessageIdCommond.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/DecodeMessageIdCommond.java new file mode 100644 index 0000000000..bd5eb7ed5b --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/DecodeMessageIdCommond.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageClientIDSetter; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +public class DecodeMessageIdCommond implements SubCommand { + @Override + public String commandName() { + return "DecodeMessageId"; + } + + @Override + public String commandDesc() { + return "decode unique message ID"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "messageId", true, "unique message ID"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + String messageId = commandLine.getOptionValue('i').trim(); + + try { + System.out.printf("ip=" + MessageClientIDSetter.getIPStrFromID(messageId)); + } catch (Exception e) { + e.printStackTrace(); + } + + try { + String date = UtilAll.formatDate(MessageClientIDSetter.getNearlyTimeFromID(messageId), UtilAll.YYYY_MM_DD_HH_MM_SS_SSS); + System.out.printf("date=" + date); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageByQueueCommand.java new file mode 100644 index 0000000000..956a360ed5 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageByQueueCommand.java @@ -0,0 +1,259 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +public class PrintMessageByQueueCommand implements SubCommand { + + @Override + public String commandName() { + return "printMsgByQueue"; + } + + + @Override + public String commandDesc() { + return "Print Message Detail"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("a", "brokerName ", true, "broker name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "queueId ", true, "queue id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "charsetName ", true, "CharsetName(eg: UTF-8,GBK)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "subExpression ", true, "Subscribe Expression(eg: TagA || TagB)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "beginTimestamp ", true, "Begin timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("e", "endTimestamp ", true, "End timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "print msg", true, "print msg. eg: true | false(default)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "printBody ", true, "print body. eg: true | false(default)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("f", "calculate", true, "calculate by tag. eg: true | false(default)"); + opt.setRequired(false); + options.addOption(opt); + + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook); + + try { + String charsetName = + !commandLine.hasOption('c') ? "UTF-8" : commandLine.getOptionValue('c').trim(); + boolean printMsg = + !commandLine.hasOption('p') ? false : Boolean.parseBoolean(commandLine.getOptionValue('p').trim()); + boolean printBody = + !commandLine.hasOption('d') ? false : Boolean.parseBoolean(commandLine.getOptionValue('d').trim()); + boolean calByTag = + !commandLine.hasOption('f') ? false : Boolean.parseBoolean(commandLine.getOptionValue('f').trim()); + String subExpression = + !commandLine.hasOption('s') ? "*" : commandLine.getOptionValue('s').trim(); + + String topic = commandLine.getOptionValue('t').trim(); + String brokerName = commandLine.getOptionValue('a').trim(); + int queueId = Integer.parseInt(commandLine.getOptionValue('i').trim()); + consumer.start(); + + MessageQueue mq = new MessageQueue(topic, brokerName, queueId); + long minOffset = consumer.minOffset(mq); + long maxOffset = consumer.maxOffset(mq); + + if (commandLine.hasOption('b')) { + String timestampStr = commandLine.getOptionValue('b').trim(); + long timeValue = timestampFormat(timestampStr); + minOffset = consumer.searchOffset(mq, timeValue); + } + + if (commandLine.hasOption('e')) { + String timestampStr = commandLine.getOptionValue('e').trim(); + long timeValue = timestampFormat(timestampStr); + maxOffset = consumer.searchOffset(mq, timeValue); + } + + final Map tagCalmap = new HashMap(); + READQ: + for (long offset = minOffset; offset < maxOffset; ) { + try { + PullResult pullResult = consumer.pull(mq, subExpression, offset, 32); + offset = pullResult.getNextBeginOffset(); + switch (pullResult.getPullStatus()) { + case FOUND: + calculateByTag(pullResult.getMsgFoundList(), tagCalmap, calByTag); + printMessage(pullResult.getMsgFoundList(), charsetName, printMsg, printBody); + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break READQ; + } + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + + printCalculateByTag(tagCalmap, calByTag); + } catch (Exception e) { + e.printStackTrace(); + } finally { + consumer.shutdown(); + } + } + + public static long timestampFormat(final String value) { + long timestamp = 0; + try { + timestamp = Long.parseLong(value); + } catch (NumberFormatException e) { + + timestamp = UtilAll.parseDate(value, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS).getTime(); + } + + return timestamp; + } + + + private static void calculateByTag(final List msgs, final Map tagCalmap, final boolean calByTag) { + if (!calByTag) + return; + + for (MessageExt msg : msgs) { + String tag = msg.getTags(); + if (StringUtils.isNotBlank(tag)) { + AtomicLong count = tagCalmap.get(tag); + if (count == null) { + count = new AtomicLong(); + tagCalmap.put(tag, count); + } + count.incrementAndGet(); + } + } + } + + private static void printCalculateByTag(final Map tagCalmap, final boolean calByTag) { + if (!calByTag) + return; + + List list = new ArrayList(); + for (Map.Entry entry : tagCalmap.entrySet()) { + TagCountBean tagBean = new TagCountBean(entry.getKey(), entry.getValue()); + list.add(tagBean); + } + Collections.sort(list); + + for (TagCountBean tagCountBean : list) { + System.out.printf("Tag: %-30s Count: %s%n", tagCountBean.getTag(), tagCountBean.getCount()); + } + } + + public static void printMessage(final List msgs, final String charsetName, boolean printMsg, boolean printBody) { + if (!printMsg) + return; + + for (MessageExt msg : msgs) { + try { + System.out.printf("MSGID: %s %s BODY: %s%n", msg.getMsgId(), msg.toString(), + printBody ? new String(msg.getBody(), charsetName) : "NOT PRINT BODY"); + } catch (UnsupportedEncodingException e) { + } + } + } + + static class TagCountBean implements Comparable { + private String tag; + private AtomicLong count; + + + public TagCountBean(final String tag, final AtomicLong count) { + this.tag = tag; + this.count = count; + } + + public String getTag() { + return tag; + } + + public void setTag(final String tag) { + this.tag = tag; + } + + public AtomicLong getCount() { + return count; + } + + public void setCount(final AtomicLong count) { + this.count = count; + } + + + @Override + public int compareTo(final TagCountBean o) { + return (int) (o.getCount().get() - this.count.get()); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java new file mode 100644 index 0000000000..ab908d455d --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class PrintMessageSubCommand implements SubCommand { + + @Override + public String commandName() { + return "printMsg"; + } + + + @Override + public String commandDesc() { + return "Print Message Detail"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "charsetName ", true, "CharsetName(eg: UTF-8,GBK)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "subExpression ", true, "Subscribe Expression(eg: TagA || TagB)"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("b", "beginTimestamp ", true, + "Begin timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("e", "endTimestamp ", true, + "End timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("d", "printBody ", true, + "print body"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook); + + try { + String topic = commandLine.getOptionValue('t').trim(); + + String charsetName = // + !commandLine.hasOption('c') ? "UTF-8" : commandLine.getOptionValue('c').trim(); + + String subExpression = // + !commandLine.hasOption('s') ? "*" : commandLine.getOptionValue('s').trim(); + + boolean printBody = // + !commandLine.hasOption('d') ? true : Boolean.parseBoolean(commandLine.getOptionValue('d').trim()); + + consumer.start(); + + Set mqs = consumer.fetchSubscribeMessageQueues(topic); + for (MessageQueue mq : mqs) { + long minOffset = consumer.minOffset(mq); + long maxOffset = consumer.maxOffset(mq); + + if (commandLine.hasOption('b')) { + String timestampStr = commandLine.getOptionValue('b').trim(); + long timeValue = timestampFormat(timestampStr); + minOffset = consumer.searchOffset(mq, timeValue); + } + + if (commandLine.hasOption('e')) { + String timestampStr = commandLine.getOptionValue('e').trim(); + long timeValue = timestampFormat(timestampStr); + maxOffset = consumer.searchOffset(mq, timeValue); + } + + System.out.printf("minOffset=" + minOffset + ", maxOffset=" + maxOffset + ", " + mq); + + READQ: + for (long offset = minOffset; offset < maxOffset; ) { + try { + PullResult pullResult = consumer.pull(mq, subExpression, offset, 32); + offset = pullResult.getNextBeginOffset(); + switch (pullResult.getPullStatus()) { + case FOUND: + printMessage(pullResult.getMsgFoundList(), charsetName, printBody); + break; + case NO_MATCHED_MSG: + System.out.printf(mq + " no matched msg. status=" + pullResult.getPullStatus() + ", offset=" + offset); + break; + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + System.out.printf(mq + " print msg finished. status=" + pullResult.getPullStatus() + ", offset=" + offset); + break READQ; + } + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + consumer.shutdown(); + } + } + + public static long timestampFormat(final String value) { + long timestamp = 0; + try { + timestamp = Long.parseLong(value); + } catch (NumberFormatException e) { + timestamp = UtilAll.parseDate(value, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS).getTime(); + } + + return timestamp; + } + + public static void printMessage(final List msgs, final String charsetName, boolean printBody) { + for (MessageExt msg : msgs) { + try { + System.out.printf("MSGID: %s %s BODY: %s%n", msg.getMsgId(), msg.toString(), + printBody ? new String(msg.getBody(), charsetName) : "NOT PRINT BODY"); + } catch (UnsupportedEncodingException e) { + // + } + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java new file mode 100644 index 0000000000..a6bf0bd43f --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -0,0 +1,294 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageClientExt; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + + +/** + * @author shijia.wxr + */ +public class QueryMsgByIdSubCommand implements SubCommand { + @Override + public String commandName() { + return "queryMsgById"; + } + + @Override + public String commandDesc() { + return "Query Message by Id"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "msgId", true, "Message Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "clientId", true, "The consumer's client id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "sendMessage", true, "resend message"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("u", "unitName", true, "unit name"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + DefaultMQProducer defaultMQProducer = new DefaultMQProducer("ReSendMsgById"); + defaultMQProducer.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + if (commandLine.hasOption('s')) { + if (commandLine.hasOption('u')) { + String unitName = commandLine.getOptionValue('u').trim(); + defaultMQProducer.setUnitName(unitName); + } + defaultMQProducer.start(); + } + + final String msgIds = commandLine.getOptionValue('i').trim(); + final String[] msgIdArr = StringUtils.split(msgIds, ","); + + if (commandLine.hasOption('g') && commandLine.hasOption('d')) { + final String consumerGroup = commandLine.getOptionValue('g').trim(); + final String clientId = commandLine.getOptionValue('d').trim(); + for (String msgId : msgIdArr) { + if (StringUtils.isNotBlank(msgId)) { + pushMsg(defaultMQAdminExt, consumerGroup, clientId, msgId.trim()); + } + } + } else if (commandLine.hasOption('s')) { + boolean resend = Boolean.parseBoolean(commandLine.getOptionValue('s', "false").trim()); + if (resend) { + for (String msgId : msgIdArr) { + if (StringUtils.isNotBlank(msgId)) { + sendMsg(defaultMQAdminExt, defaultMQProducer, msgId.trim()); + } + } + } + } else { + for (String msgId : msgIdArr) { + if (StringUtils.isNotBlank(msgId)) { + queryById(defaultMQAdminExt, msgId.trim()); + } + } + + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQProducer.shutdown(); + defaultMQAdminExt.shutdown(); + } + } + + private void pushMsg(final DefaultMQAdminExt defaultMQAdminExt, final String consumerGroup, final String clientId, final String msgId) { + try { + ConsumeMessageDirectlyResult result = + defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, msgId); + System.out.printf("%s", result); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void sendMsg(final DefaultMQAdminExt defaultMQAdminExt, final DefaultMQProducer defaultMQProducer, final String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + MessageExt msg = defaultMQAdminExt.viewMessage(msgId); + if (msg != null) { + // resend msg by id + System.out.printf("prepare resend msg. originalMsgId=" + msgId); + SendResult result = defaultMQProducer.send(msg); + System.out.printf("%s", result); + } else { + System.out.printf("no message. msgId=" + msgId); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void queryById(final DefaultMQAdminExt admin, final String msgId) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { + MessageExt msg = admin.viewMessage(msgId); + + printMsg(admin, msg); + } + + public static void printMsg(final DefaultMQAdminExt admin, final MessageExt msg) throws IOException { + if (msg == null) { + System.out.printf("%nMessage not found!"); + return; + } + + String bodyTmpFilePath = createBodyFile(msg); + String msgId = msg.getMsgId(); + if (msg instanceof MessageClientExt) { + msgId = ((MessageClientExt) msg).getOffsetMsgId(); + } + + System.out.printf("%-20s %s%n", + "OffsetID:", + msgId + ); + + System.out.printf("%-20s %s%n", + "OffsetID:", + msgId + ); + + System.out.printf("%-20s %s%n", + "Topic:", + msg.getTopic() + ); + + System.out.printf("%-20s %s%n", + "Tags:", + "[" + msg.getTags() + "]" + ); + + System.out.printf("%-20s %s%n", + "Keys:", + "[" + msg.getKeys() + "]" + ); + + System.out.printf("%-20s %d%n", + "Queue ID:", + msg.getQueueId() + ); + + System.out.printf("%-20s %d%n", + "Queue Offset:", + msg.getQueueOffset() + ); + + System.out.printf("%-20s %d%n", + "CommitLog Offset:", + msg.getCommitLogOffset() + ); + + System.out.printf("%-20s %d%n", + "Reconsume Times:", + msg.getReconsumeTimes() + ); + + System.out.printf("%-20s %s%n", + "Born Timestamp:", + UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) + ); + + System.out.printf("%-20s %s%n", + "Store Timestamp:", + UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp()) + ); + + System.out.printf("%-20s %s%n", + "Born Host:", + RemotingHelper.parseSocketAddressAddr(msg.getBornHost()) + ); + + System.out.printf("%-20s %s%n", + "Store Host:", + RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()) + ); + + System.out.printf("%-20s %d%n", + "System Flag:", + msg.getSysFlag() + ); + + System.out.printf("%-20s %s%n", + "Properties:", + msg.getProperties() != null ? msg.getProperties().toString() : "" + ); + + System.out.printf("%-20s %s%n", + "Message Body Path:", + bodyTmpFilePath + ); + + try { + List mtdList = admin.messageTrackDetail(msg); + if (mtdList.isEmpty()) { + System.out.printf("%n%nWARN: No Consumer"); + } else { + System.out.printf("%n%n"); + for (MessageTrack mt : mtdList) { + System.out.printf("%s", mt); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static String createBodyFile(MessageExt msg) throws IOException { + DataOutputStream dos = null; + try { + String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; + File file = new File(bodyTmpFilePath); + if (!file.exists()) { + file.mkdirs(); + } + bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + dos.write(msg.getBody()); + return bodyTmpFilePath; + } finally { + if (dos != null) + dos.close(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java new file mode 100644 index 0000000000..abc1cb1445 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * + * @author shijia.wxr + * + */ +public class QueryMsgByKeySubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgByKey"; + } + + @Override + public String commandDesc() { + return "Query Message by Key"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "msgKey", true, "Message Key"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + final String topic = commandLine.getOptionValue('t').trim(); + final String key = commandLine.getOptionValue('k').trim(); + + this.queryByKey(defaultMQAdminExt, topic, key); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key) + throws MQClientException, InterruptedException { + admin.start(); + + QueryResult queryResult = admin.queryMessage(topic, key, 64, 0, Long.MAX_VALUE); + System.out.printf("%-50s %4s %40s%n", + "#Message ID", + "#QID", + "#Offset"); + for (MessageExt msg : queryResult.getMessageList()) { + System.out.printf("%-50s %4d %40d%n", msg.getMsgId(), msg.getQueueId(), msg.getQueueOffset()); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java new file mode 100644 index 0000000000..8ccf907ad3 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * @author shijia.wxr + */ +public class QueryMsgByOffsetSubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgByOffset"; + } + + @Override + public String commandDesc() { + return "Query Message by offset"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("b", "brokerName", true, "Broker Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "queueId", true, "Queue Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("o", "offset", true, "Queue Offset"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + defaultMQPullConsumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue('t').trim(); + String brokerName = commandLine.getOptionValue('b').trim(); + String queueId = commandLine.getOptionValue('i').trim(); + String offset = commandLine.getOptionValue('o').trim(); + + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(brokerName); + mq.setQueueId(Integer.parseInt(queueId)); + + defaultMQPullConsumer.start(); + defaultMQAdminExt.start(); + + PullResult pullResult = defaultMQPullConsumer.pull(mq, "*", Long.parseLong(offset), 1); + if (pullResult != null) { + switch (pullResult.getPullStatus()) { + case FOUND: + QueryMsgByIdSubCommand.printMsg(defaultMQAdminExt, pullResult.getMsgFoundList().get(0)); + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + default: + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQPullConsumer.shutdown(); + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java new file mode 100644 index 0000000000..faf49c6936 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + +public class QueryMsgByUniqueKeySubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgByUniqueKey"; + } + + @Override + public String commandDesc() { + return "Query Message by Unique key"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "msgId", true, "Message Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "clientId", true, "The consumer's client id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "The topic of msg"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { + MessageExt msg = admin.viewMessage(topic, msgId); + + + String bodyTmpFilePath = createBodyFile(msg); + + System.out.printf("%-20s %s%n", + "Topic:", + msg.getTopic() + ); + + System.out.printf("%-20s %s%n", + "Tags:", + "[" + msg.getTags() + "]" + ); + + System.out.printf("%-20s %s%n", + "Keys:", + "[" + msg.getKeys() + "]" + ); + + System.out.printf("%-20s %d%n", + "Queue ID:", + msg.getQueueId() + ); + + System.out.printf("%-20s %d%n", + "Queue Offset:", + msg.getQueueOffset() + ); + + System.out.printf("%-20s %d%n", + "CommitLog Offset:", + msg.getCommitLogOffset() + ); + + System.out.printf("%-20s %d%n", + "Reconsume Times:", + msg.getReconsumeTimes() + ); + + System.out.printf("%-20s %s%n", + "Born Timestamp:", + UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) + ); + + System.out.printf("%-20s %s%n", + "Store Timestamp:", + UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp()) + ); + + System.out.printf("%-20s %s%n", + "Born Host:", + RemotingHelper.parseSocketAddressAddr(msg.getBornHost()) + ); + + System.out.printf("%-20s %s%n", + "Store Host:", + RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()) + ); + + System.out.printf("%-20s %d%n", + "System Flag:", + msg.getSysFlag() + ); + + System.out.printf("%-20s %s%n", + "Properties:", + msg.getProperties() != null ? msg.getProperties().toString() : "" + ); + + System.out.printf("%-20s %s%n", + "Message Body Path:", + bodyTmpFilePath + ); + + try { + List mtdList = admin.messageTrackDetail(msg); + if (mtdList.isEmpty()) { + System.out.printf("%n%nWARN: No Consumer"); + } else { + System.out.printf("%n%n"); + for (MessageTrack mt : mtdList) { + System.out.printf("%s", mt); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + final String msgId = commandLine.getOptionValue('i').trim(); + final String topic = commandLine.getOptionValue('t').trim(); + if (commandLine.hasOption('g') && commandLine.hasOption('d')) { + final String consumerGroup = commandLine.getOptionValue('g').trim(); + final String clientId = commandLine.getOptionValue('d').trim(); + ConsumeMessageDirectlyResult result = + defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); + System.out.printf("%s", result); + } else { + queryById(defaultMQAdminExt, topic, msgId); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + + private static String createBodyFile(MessageExt msg) throws IOException { + DataOutputStream dos = null; + try { + String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; + File file = new File(bodyTmpFilePath); + if (!file.exists()) { + file.mkdirs(); + } + bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + dos.write(msg.getBody()); + return bodyTmpFilePath; + } finally { + if (dos != null) + dos.close(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java new file mode 100644 index 0000000000..2ce29a06e8 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.message; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.store.ConsumeQueue; +import com.alibaba.rocketmq.store.MappedFile; +import com.alibaba.rocketmq.store.MappedFileQueue; +import com.alibaba.rocketmq.store.SelectMappedBufferResult; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class Store { + public final static int MESSAGE_MAGIC_CODE = 0xAABBCCDD ^ 1880681586 + 8; + private final static int BLANK_MAGIC_CODE = 0xBBCCDDEE ^ 1880681586 + 8; + private MappedFileQueue mapedFileQueue; + private ConcurrentHashMap> consumeQueueTable; + + private String cStorePath; + private int cSize; + private String lStorePath; + private int lSize; + + public Store(String cStorePath, int cSize, String lStorePath, int lSize) { + this.cStorePath = cStorePath; + this.cSize = cSize; + this.lStorePath = lStorePath; + this.lSize = lSize; + mapedFileQueue = new MappedFileQueue(cStorePath, cSize, null); + consumeQueueTable = + new ConcurrentHashMap>(); + } + + public boolean load() { + boolean result = this.mapedFileQueue.load(); + System.out.printf("load commit log " + (result ? "OK" : "Failed")); + if (result) { + result = loadConsumeQueue(); + } + System.out.printf("load logics log " + (result ? "OK" : "Failed")); + return result; + } + + private boolean loadConsumeQueue() { + File dirLogic = new File(StorePathConfigHelper.getStorePathConsumeQueue(lStorePath)); + File[] fileTopicList = dirLogic.listFiles(); + if (fileTopicList != null) { + + for (File fileTopic : fileTopicList) { + String topic = fileTopic.getName(); + + File[] fileQueueIdList = fileTopic.listFiles(); + if (fileQueueIdList != null) { + for (File fileQueueId : fileQueueIdList) { + int queueId = Integer.parseInt(fileQueueId.getName()); + ConsumeQueue logic = new ConsumeQueue( + topic, + queueId, + StorePathConfigHelper.getStorePathConsumeQueue(lStorePath), + lSize, + null); + this.putConsumeQueue(topic, queueId, logic); + if (!logic.load()) { + return false; + } + } + } + } + } + System.out.printf("load logics queue all over, OK"); + return true; + } + + + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueue consumeQueue) { + ConcurrentHashMap map = this.consumeQueueTable.get(topic); + if (null == map) { + map = new ConcurrentHashMap(); + map.put(queueId, consumeQueue); + this.consumeQueueTable.put(topic, map); + } else { + map.put(queueId, consumeQueue); + } + } + + public void traval(boolean openAll) { + boolean success = true; + byte[] bytesContent = new byte[1024]; + List mapedFiles = this.mapedFileQueue.getMappedFiles(); + ALL: + for (MappedFile mapedFile : mapedFiles) { + long startOffset = mapedFile.getFileFromOffset(); + int position = 0; + int msgCount = 0; + int errorCount = 0; + + System.out.printf("start travel " + mapedFile.getFileName()); + long startTime = System.currentTimeMillis(); + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + while (byteBuffer.hasRemaining()) { + // 1 TOTALSIZE + int totalSize = byteBuffer.getInt(); + // 2 MAGICCODE + int magicCode = byteBuffer.getInt(); + if (BLANK_MAGIC_CODE == magicCode) { + position = byteBuffer.limit(); + break; + } + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + + // 5 FLAG + int flag = byteBuffer.getInt(); + flag = flag + 0; + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + bornTimeStamp = bornTimeStamp + 0; + + // 10 BORNHOST(IP+PORT) + byteBuffer.position(byteBuffer.position() + 8); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + + // 12 STOREHOST(IP+PORT) + byteBuffer.position(byteBuffer.position() + 8); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byteBuffer.get(bytesContent, 0, topicLen); + String topic = null; + try { + topic = new String(bytesContent, 0, topicLen, MixAll.DEFAULT_CHARSET); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + + + Date storeTime = new Date(storeTimestamp); + + + long currentPhyOffset = startOffset + position; + if (physicOffset != currentPhyOffset) { + System.out.printf(storeTime + + " [fetal error] physicOffset != currentPhyOffset. position=" + position + + ", msgCount=" + msgCount + ", physicOffset=" + physicOffset + + ", currentPhyOffset=" + currentPhyOffset); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + SelectMappedBufferResult smb = consumeQueue.getIndexBuffer(queueOffset); + try { + long offsetPy = smb.getByteBuffer().getLong(); + int sizePy = smb.getByteBuffer().getInt(); + if (physicOffset != offsetPy) { + System.out.printf(storeTime + " [fetal error] physicOffset != offsetPy. position=" + + position + ", msgCount=" + msgCount + ", physicOffset=" + physicOffset + + ", offsetPy=" + offsetPy); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + if (totalSize != sizePy) { + System.out.printf(storeTime + " [fetal error] totalSize != sizePy. position=" + + position + ", msgCount=" + msgCount + ", totalSize=" + totalSize + + ", sizePy=" + sizePy); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + } finally { + smb.release(); + } + + msgCount++; + position += totalSize; + byteBuffer.position(position); + } + + System.out.printf("end travel " + mapedFile.getFileName() + ", total msg=" + msgCount + + ", error count=" + errorCount + ", cost:" + (System.currentTimeMillis() - startTime)); + } + + System.out.printf("travel " + (success ? "ok" : "fail")); + } + + public ConsumeQueue findConsumeQueue(String topic, int queueId) { + ConcurrentHashMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentHashMap newMap = + new ConcurrentHashMap(128); + ConcurrentHashMap oldMap = consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } else { + map = newMap; + } + } + ConsumeQueue logic = map.get(queueId); + if (null == logic) { + ConsumeQueue newLogic = new ConsumeQueue( + topic, + queueId, + StorePathConfigHelper.getStorePathConsumeQueue(lStorePath), + lSize, + null); + ConsumeQueue oldLogic = map.putIfAbsent(queueId, newLogic); + if (oldLogic != null) { + logic = oldLogic; + } else { + logic = newLogic; + } + } + return logic; + } +} \ No newline at end of file diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java new file mode 100644 index 0000000000..a03baf45d3 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * + * @author manhong.yqd + * + */ +public class DeleteKvConfigCommand implements SubCommand { + @Override + public String commandName() { + return "deleteKvConfig"; + } + + + @Override + public String commandDesc() { + return "Delete KV config."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "namespace", true, "set the namespace"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "key", true, "set the key name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // namespace + String namespace = commandLine.getOptionValue('s').trim(); + // key name + String key = commandLine.getOptionValue('k').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteKvConfig(namespace, key); + System.out.printf("delete kv config from namespace success.%n"); + return; + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetNamesrvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetNamesrvConfigCommand.java new file mode 100644 index 0000000000..0a404846fb --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetNamesrvConfigCommand.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.namesrv; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * @author xigu.lx + */ +public class GetNamesrvConfigCommand implements SubCommand { + + @Override + public String commandName() { + return "getNamesrvConfig"; + } + + @Override + public String commandDesc() { + return "Get configs of name server."; + } + + @Override + public Options buildCommandlineOptions(final Options options) { + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, final RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // servers + String servers = commandLine.getOptionValue('n'); + List serverList = null; + if (servers != null && servers.length() > 0) { + String[] serverArray = servers.trim().split(";"); + + if (serverArray != null && serverArray.length > 0) { + serverList = Arrays.asList(serverArray); + } + } + + defaultMQAdminExt.start(); + + Map nameServerConfigs = defaultMQAdminExt.getNameServerConfig(serverList); + + for (String server : nameServerConfigs.keySet()) { + System.out.printf("============%s============\n", + server); + for (Object key : nameServerConfigs.get(server).keySet()) { + System.out.printf("%-50s= %s\n", key, nameServerConfigs.get(server).get(key)); + } + } + return; + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} \ No newline at end of file diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java new file mode 100644 index 0000000000..21a43e7a20 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * + * @author manhong.yqd + * + */ +public class UpdateKvConfigCommand implements SubCommand { + @Override + public String commandName() { + return "updateKvConfig"; + } + + + @Override + public String commandDesc() { + return "Create or update KV config."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "namespace", true, "set the namespace"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "key", true, "set the key name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "value", true, "set the key value"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // namespace + String namespace = commandLine.getOptionValue('s').trim(); + // key name + String key = commandLine.getOptionValue('k').trim(); + // key name + String value = commandLine.getOptionValue('v').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateKvConfig(namespace, key, value); + System.out.printf("create or update kv config to namespace success.%n"); + return; + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateNamesrvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateNamesrvConfigCommand.java new file mode 100644 index 0000000000..f6ee1f7cc6 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateNamesrvConfigCommand.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.namesrv; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +/** + * @author xigu.lx + */ +public class UpdateNamesrvConfigCommand implements SubCommand { + @Override + public String commandName() { + return "updateNamesrvConfig"; + } + + @Override + public String commandDesc() { + return "Update configs of name server."; + } + + @Override + public Options buildCommandlineOptions(final Options options) { + Option opt = new Option("k", "key", true, "config key"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "value", true, "config value"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, final RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // key name + String key = commandLine.getOptionValue('k').trim(); + // key name + String value = commandLine.getOptionValue('v').trim(); + Properties properties = new Properties(); + properties.put(key, value); + + // servers + String servers = commandLine.getOptionValue('n'); + List serverList = null; + if (servers != null && servers.length() > 0) { + String[] serverArray = servers.trim().split(";"); + + if (serverArray != null && serverArray.length > 0) { + serverList = Arrays.asList(serverArray); + } + } + + defaultMQAdminExt.start(); + + defaultMQAdminExt.updateNameServerConfig(properties, serverList); + + System.out.printf("update name server config success!%s\n%s : %s\n", + serverList == null ? "" : serverList, key, value); + return; + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} \ No newline at end of file diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java new file mode 100644 index 0000000000..053ac7e851 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.List; + +public class WipeWritePermSubCommand implements SubCommand { + + @Override + public String commandName() { + return "wipeWritePerm"; + } + + + @Override + public String commandDesc() { + return "Wipe write perm of broker in all name server"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerName", true, "broker name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String brokerName = commandLine.getOptionValue('b').trim(); + List namesrvList = defaultMQAdminExt.getNameServerAddressList(); + if (namesrvList != null) { + for (String namesrvAddr : namesrvList) { + try { + int wipeTopicCount = defaultMQAdminExt.wipeWritePermOfBroker(namesrvAddr, brokerName); + System.out.printf("wipe write perm of broker[%s] in name server[%s] OK, %d%n", + brokerName, + namesrvAddr, + wipeTopicCount + ); + } catch (Exception e) { + System.out.printf("wipe write perm of broker[%s] in name server[%s] Failed%n", + brokerName, + namesrvAddr + ); + + e.printStackTrace(); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java new file mode 100644 index 0000000000..b72aeaee5c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.offset; + +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Set; + +public class CloneGroupOffsetCommand implements SubCommand { + @Override + public String commandName() { + return "cloneGroupOffset"; + } + + @Override + public String commandDesc() { + return "clone offset from other group."; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "srcGroup", true, "set source consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("d", "destGroup", true, "set destination consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("o", "offline", true, "the group or the topic is offline"); + opt.setRequired(false); + options.addOption(opt); + + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + String srcGroup = commandLine.getOptionValue("s").trim(); + String destGroup = commandLine.getOptionValue("d").trim(); + String topic = commandLine.getOptionValue("t").trim(); + + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName("admin-" + Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(srcGroup); + Set mqs = consumeStats.getOffsetTable().keySet(); + if (mqs != null && !mqs.isEmpty()) { + TopicRouteData topicRoute = defaultMQAdminExt.examineTopicRouteInfo(topic); + for (MessageQueue mq : mqs) { + String addr = null; + for (BrokerData brokerData : topicRoute.getBrokerDatas()) { + if (brokerData.getBrokerName().equals(mq.getBrokerName())) { + addr = brokerData.selectBrokerAddr(); + break; + } + } + long offset = consumeStats.getOffsetTable().get(mq).getBrokerOffset(); + if (offset >= 0) { + defaultMQAdminExt.updateConsumeOffset(addr, destGroup, mq, offset); + } + } + } + System.out.printf("clone group offset success. srcGroup[%s], destGroup=[%s], topic[%s]", + srcGroup, destGroup, topic); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java new file mode 100644 index 0000000000..af79512fad --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.offset; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Map; + +public class GetConsumerStatusCommand implements SubCommand { + @Override + public String commandName() { + return "getConsumerStatus"; + } + + @Override + public String commandDesc() { + return "get consumer status from client."; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "originClientId", true, "set the consumer clientId"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String group = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String originClientId = ""; + if (commandLine.hasOption("i")) { + originClientId = commandLine.getOptionValue("i").trim(); + } + defaultMQAdminExt.start(); + + Map> consumerStatusTable = + defaultMQAdminExt.getConsumeStatus(topic, group, originClientId); + System.out.printf("get consumer status from client. group=%s, topic=%s, originClientId=%s%n", + group, topic, originClientId); + + System.out.printf("%-50s %-15s %-15s %-20s%n", + "#clientId", + "#brokerName", + "#queueId", + "#offset"); + + for (Map.Entry> entry : consumerStatusTable.entrySet()) { + String clientId = entry.getKey(); + Map mqTable = entry.getValue(); + for (Map.Entry entry1 : mqTable.entrySet()) { + MessageQueue mq = entry1.getKey(); + System.out.printf("%-50s %-15s %-15d %-20d%n", + UtilAll.frontStringAtLeast(clientId, 50), + mq.getBrokerName(), + mq.getQueueId(), + mqTable.get(mq)); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java new file mode 100644 index 0000000000..e2bbbffbb7 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.offset; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; + +import java.util.Iterator; +import java.util.Map; + + +/** + * @author manhong.yqd + */ +public class ResetOffsetByTimeCommand implements SubCommand { + public static void main(String[] args) { + ResetOffsetByTimeCommand cmd = new ResetOffsetByTimeCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[]{"-t Jodie_rest_test", "-g CID_Jodie_rest_test", "-s -1", "-f true"}; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, cmd.buildCommandlineOptions(options), new PosixParser()); + cmd.execute(commandLine, options, null); + } + + @Override + public String commandName() { + return "resetOffsetByTime"; + } + + @Override + public String commandDesc() { + return "Reset consumer offset by timestamp(without client restart)."; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "timestamp", true, "set the timestamp[now|currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cplus", false, "reset c++ client offset"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String group = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String timeStampStr = commandLine.getOptionValue("s").trim(); + long timestamp = timeStampStr.equals("now") ? System.currentTimeMillis() : 0; + + try { + if (timestamp == 0) { + timestamp = Long.parseLong(timeStampStr); + } + } catch (NumberFormatException e) { + + timestamp = UtilAll.parseDate(timeStampStr, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS).getTime(); + } + + boolean force = true; + if (commandLine.hasOption('f')) { + force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); + } + + boolean isC = false; + if (commandLine.hasOption('c')) { + isC = true; + } + + defaultMQAdminExt.start(); + Map offsetTable; + try { + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force, isC); + } catch (MQClientException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, group, topic, timestamp, force, timeStampStr); + return; + } + throw e; + } + + System.out.printf("rollback consumer offset by specified group[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]%n", + group, topic, force, timeStampStr, timestamp); + + System.out.printf("%-40s %-40s %-40s%n", + "#brokerName", + "#queueId", + "#offset"); + + Iterator> iterator = offsetTable.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + System.out.printf("%-40s %-40d %-40d%n", + UtilAll.frontStringAtLeast(entry.getKey().getBrokerName(), 32), + entry.getKey().getQueueId(), + entry.getValue()); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java new file mode 100644 index 0000000000..3420795ee5 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.offset; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Date; +import java.util.List; + + +/** + * + * @author manhong.yqd + * + */ +public class ResetOffsetByTimeOldCommand implements SubCommand { + @Override + public String commandName() { + return "resetOffsetByTimeOld"; + } + + @Override + public String commandDesc() { + return "Reset consumer offset by timestamp(execute this command required client restart)."; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "timestamp", true, "set the timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String consumerGroup = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String timeStampStr = commandLine.getOptionValue("s").trim(); + long timestamp = 0; + try { + timestamp = Long.parseLong(timeStampStr); + } catch (NumberFormatException e) { + + Date date = UtilAll.parseDate(timeStampStr, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS); + if (date != null) { + timestamp = UtilAll.parseDate(timeStampStr, UtilAll.YYYY_MM_DD_HH_MM_SS_SSS).getTime(); + } else { + System.out.printf("specified timestamp invalid.%n"); + return; + } + + boolean force = true; + if (commandLine.hasOption('f')) { + force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); + } + + defaultMQAdminExt.start(); + resetOffset(defaultMQAdminExt, consumerGroup, topic, timestamp, force, timeStampStr); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String consumerGroup, String topic, long timestamp, boolean force, + String timeStampStr) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + List rollbackStatsList = defaultMQAdminExt.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + System.out.printf( + "rollback consumer offset by specified consumerGroup[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]%n", + consumerGroup, topic, force, timeStampStr, timestamp); + + System.out.printf("%-20s %-20s %-20s %-20s %-20s %-20s%n", + "#brokerName", + "#queueId", + "#brokerOffset", + "#consumerOffset", + "#timestampOffset", + "#rollbackOffset" + ); + + for (RollbackStats rollbackStats : rollbackStatsList) { + System.out.printf("%-20s %-20d %-20d %-20d %-20d %-20d%n", + UtilAll.frontStringAtLeast(rollbackStats.getBrokerName(), 32), + rollbackStats.getQueueId(), + rollbackStats.getBrokerOffset(), + rollbackStats.getConsumerOffset(), + rollbackStats.getTimestampOffset(), + rollbackStats.getRollbackOffset() + ); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/stats/StatsAllSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/stats/StatsAllSubCommand.java new file mode 100644 index 0000000000..a57f04a84c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/stats/StatsAllSubCommand.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.stats; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.protocol.body.BrokerStatsData; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +public class StatsAllSubCommand implements SubCommand { + @Override + public String commandName() { + return "statsAll"; + } + + @Override + public String commandDesc() { + return "Topic and Consumer tps stats"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("a", "activeTopic", false, "print active topic only"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "print select topic only"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + + System.out.printf("%-32s %-32s %12s %11s %11s %14s %14s%n", + "#Topic", + "#Consumer Group", + "#Accumulation", + "#InTPS", + "#OutTPS", + "#InMsg24Hour", + "#OutMsg24Hour" + ); + + boolean activeTopic = commandLine.hasOption('a'); + String selectTopic = commandLine.getOptionValue('t'); + + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) { + continue; + } + + if (selectTopic != null && selectTopic != "" && !topic.equals(selectTopic)) { + continue; + } + + try { + printTopicDetail(defaultMQAdminExt, topic, activeTopic); + } catch (Exception e) { + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + public static void printTopicDetail(final DefaultMQAdminExt admin, final String topic, final boolean activeTopic) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + TopicRouteData topicRouteData = admin.examineTopicRouteInfo(topic); + + GroupList groupList = admin.queryTopicConsumeByWho(topic); + + double inTPS = 0; + + long inMsgCntToday = 0; + + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID); + if (masterAddr != null) { + try { + BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.TOPIC_PUT_NUMS, topic); + inTPS += bsd.getStatsMinute().getTps(); + inMsgCntToday += compute24HourSum(bsd); + } catch (Exception e) { + } + } + } + + if (groupList != null && !groupList.getGroupList().isEmpty()) { + + for (String group : groupList.getGroupList()) { + double outTPS = 0; + long outMsgCntToday = 0; + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID); + if (masterAddr != null) { + try { + String statsKey = String.format("%s@%s", topic, group); + BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.GROUP_GET_NUMS, statsKey); + outTPS += bsd.getStatsMinute().getTps(); + outMsgCntToday += compute24HourSum(bsd); + } catch (Exception e) { + } + } + } + + long accumulate = 0; + try { + ConsumeStats consumeStats = admin.examineConsumeStats(group, topic); + if (consumeStats != null) { + accumulate = consumeStats.computeTotalDiff(); + if (accumulate < 0) { + accumulate = 0; + } + } + } catch (Exception e) { + } + + if (!activeTopic || (inMsgCntToday > 0) || + (outMsgCntToday > 0)) { + + System.out.printf("%-32s %-32s %12d %11.2f %11.2f %14d %14d%n", + UtilAll.frontStringAtLeast(topic, 32), + UtilAll.frontStringAtLeast(group, 32), + accumulate, + inTPS, + outTPS, + inMsgCntToday, + outMsgCntToday + ); + } + } + } else { + if (!activeTopic || (inMsgCntToday > 0)) { + + System.out.printf("%-32s %-32s %12d %11.2f %11s %14d %14s%n", + UtilAll.frontStringAtLeast(topic, 32), + "", + 0, + inTPS, + "", + inMsgCntToday, + "NO_CONSUMER" + ); + } + } + } + + public static long compute24HourSum(BrokerStatsData bsd) { + if (bsd.getStatsDay().getSum() != 0) { + return bsd.getStatsDay().getSum(); + } + + if (bsd.getStatsHour().getSum() != 0) { + return bsd.getStatsHour().getSum(); + } + + if (bsd.getStatsMinute().getSum() != 0) { + return bsd.getStatsMinute().getSum(); + } + + return 0; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/AllocateMQSubCommand.java new file mode 100644 index 0000000000..35e5d3bf14 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/AllocateMQSubCommand.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + + +public class AllocateMQSubCommand implements SubCommand { + @Override + public String commandName() { + return "allocateMQ"; + } + + + @Override + public String commandDesc() { + return "Allocate MQ"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "ipList", true, "ipList"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); + adminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + adminExt.start(); + + String topic = commandLine.getOptionValue('t').trim(); + String ips = commandLine.getOptionValue('i').trim(); + final String[] split = ips.split(","); + final List ipList = new LinkedList(); + for (String ip : split) { + ipList.add(ip); + } + + final TopicRouteData topicRouteData = adminExt.examineTopicRouteInfo(topic); + final Set mqs = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + + final AllocateMessageQueueAveragely averagely = new AllocateMessageQueueAveragely(); + + + RebalanceResult rr = new RebalanceResult(); + + for (String i : ipList) { + final List mqResult = averagely.allocate("aa", i, new ArrayList(mqs), ipList); + rr.getResult().put(i, mqResult); + } + + final String json = RemotingSerializable.toJson(rr, false); + System.out.printf("%s%n", json); + } catch (Exception e) { + e.printStackTrace(); + } finally { + adminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java new file mode 100644 index 0000000000..1832de67cb --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + + +/** + * + * @author lansheng.zj + * + */ +public class DeleteTopicSubCommand implements SubCommand { + @Override + public String commandName() { + return "deleteTopic"; + } + + + @Override + public String commandDesc() { + return "Delete topic from broker and NameServer."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "delete topic from which cluster"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + public static void deleteTopic(final DefaultMQAdminExt adminExt, + final String clusterName, + final String topic + ) throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); + adminExt.deleteTopicInBroker(masterSet, topic); + System.out.printf("delete topic [%s] from cluster [%s] success.%n", topic, clusterName); + + + Set nameServerSet = null; + if (adminExt.getNamesrvAddr() != null) { + String[] ns = adminExt.getNamesrvAddr().trim().split(";"); + nameServerSet = new HashSet(Arrays.asList(ns)); + } + + + adminExt.deleteTopicInNameServer(nameServerSet, topic); + System.out.printf("delete topic [%s] from NameServer success.%n", topic); + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); + adminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String topic = commandLine.getOptionValue('t').trim(); + + if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + adminExt.start(); + deleteTopic(adminExt, clusterName, topic); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + adminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/RebalanceResult.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/RebalanceResult.java new file mode 100644 index 0000000000..478413e126 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/RebalanceResult.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RebalanceResult { + private Map> result = new HashMap>(); + + public Map> getResult() { + return result; + } + + public void setResult(final Map> result) { + this.result = result; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicClusterSubCommand.java new file mode 100644 index 0000000000..9954305eb1 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicClusterSubCommand.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Set; + + +/** + * + * @author zhouli + * + */ +public class TopicClusterSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicClusterList"; + } + + + @Override + public String commandDesc() { + return "get cluster info for topic"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + String topic = commandLine.getOptionValue('t').trim(); + try { + defaultMQAdminExt.start(); + Set clusters = defaultMQAdminExt.getTopicClusterList(topic); + for (String value : clusters) { + System.out.printf("%s%n", value); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java new file mode 100644 index 0000000000..9224d65c29 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; + + +/** + * + * @author shijia.wxr + * + */ +public class TopicListSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicList"; + } + + @Override + public String commandDesc() { + return "Fetch all topic list from name server"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("c", "clusterModel", false, "clusterModel"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + if (commandLine.hasOption('c')) { + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-20s %-48s %-48s%n", + "#Cluster Name", + "#Topic", + "#Consumer Group" + ); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) { + continue; + } + + String clusterName = ""; + GroupList groupList = new GroupList(); + + try { + clusterName = + this.findTopicBelongToWhichCluster(topic, clusterInfo, defaultMQAdminExt); + groupList = defaultMQAdminExt.queryTopicConsumeByWho(topic); + } catch (Exception e) { + } + + if (null == groupList || groupList.getGroupList().isEmpty()) { + groupList = new GroupList(); + groupList.getGroupList().add(""); + } + + for (String group : groupList.getGroupList()) { + System.out.printf("%-20s %-48s %-48s%n", + UtilAll.frontStringAtLeast(clusterName, 20), + UtilAll.frontStringAtLeast(topic, 48), + UtilAll.frontStringAtLeast(group, 48) + ); + } + } + } else { + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + System.out.printf("%s%n", topic); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + private String findTopicBelongToWhichCluster(final String topic, final ClusterInfo clusterInfo, + final DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, MQClientException, + InterruptedException { + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + + BrokerData brokerData = topicRouteData.getBrokerDatas().get(0); + + String brokerName = brokerData.getBrokerName(); + + Iterator>> it = clusterInfo.getClusterAddrTable().entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + if (next.getValue().contains(brokerName)) { + return next.getKey(); + } + } + return null; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java new file mode 100644 index 0000000000..d1d6d28f4a --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * + * @author shijia.wxr + * + */ +public class TopicRouteSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicRoute"; + } + + + @Override + public String commandDesc() { + return "Examine topic route info"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String topic = commandLine.getOptionValue('t').trim(); + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + String json = topicRouteData.toJson(true); + System.out.printf("%s%n", json); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java new file mode 100644 index 0000000000..685dbea488 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.TopicOffset; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + + +/** + * + * @author shijia.wxr + * + */ +public class TopicStatusSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicStatus"; + } + + + @Override + public String commandDesc() { + return "Examine topic Status info"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String topic = commandLine.getOptionValue('t').trim(); + TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + + List mqList = new LinkedList(); + mqList.addAll(topicStatsTable.getOffsetTable().keySet()); + Collections.sort(mqList); + + System.out.printf("%-32s %-4s %-20s %-20s %s%n", + "#Broker Name", + "#QID", + "#Min Offset", + "#Max Offset", + "#Last Updated" + ); + + for (MessageQueue mq : mqList) { + TopicOffset topicOffset = topicStatsTable.getOffsetTable().get(mq); + + String humanTimestamp = ""; + if (topicOffset.getLastUpdateTimestamp() > 0) { + humanTimestamp = UtilAll.timeMillisToHumanString2(topicOffset.getLastUpdateTimestamp()); + } + + System.out.printf("%-32s %-4d %-20d %-20d %s%n", + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32), + mq.getQueueId(), + topicOffset.getMinOffset(), + topicOffset.getMaxOffset(), + humanTimestamp + ); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java new file mode 100644 index 0000000000..164579f910 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + + +/** + * + * @author manhong.yqd + * + */ +public class UpdateOrderConfCommand implements SubCommand { + + @Override + public String commandName() { + return "updateOrderConf"; + } + + + @Override + public String commandDesc() { + return "Create or update or delete order conf"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "orderConf", true, "set order conf [eg. brokerName1:num;brokerName2:num]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "method", true, "option type [eg. put|get|delete"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue('t').trim(); + String type = commandLine.getOptionValue('m').trim(); + + if ("get".equals(type)) { + + defaultMQAdminExt.start(); + String orderConf = + defaultMQAdminExt.getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, topic); + System.out.printf("get orderConf success. topic=[%s], orderConf=[%s] ", topic, orderConf); + + return; + } else if ("put".equals(type)) { + + defaultMQAdminExt.start(); + String orderConf = ""; + if (commandLine.hasOption('v')) { + orderConf = commandLine.getOptionValue('v').trim(); + } + if (UtilAll.isBlank(orderConf)) { + throw new Exception("please set orderConf with option -v."); + } + + defaultMQAdminExt.createOrUpdateOrderConf(topic, orderConf, true); + System.out.printf("update orderConf success. topic=[%s], orderConf=[%s]", topic, + orderConf.toString()); + return; + } else if ("delete".equals(type)) { + + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteKvConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, topic); + System.out.printf("delete orderConf success. topic=[%s]", topic); + + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java new file mode 100644 index 0000000000..1938934c65 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.List; +import java.util.Set; + + +public class UpdateTopicPermSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateTopicPerm"; + } + + + @Override + public String commandDesc() { + return "Update topic perm"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "create topic to which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "create topic to which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("p", "perm", true, "set topic's permission(2|4|6), intro[2:R; 4:W; 6:RW]"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + defaultMQAdminExt.start(); + TopicConfig topicConfig = new TopicConfig(); + + String topic = commandLine.getOptionValue('t').trim(); + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + assert topicRouteData != null; + List queueDatas = topicRouteData.getQueueDatas(); + assert queueDatas != null && queueDatas.size() > 0; + + QueueData queueData = queueDatas.get(0); + topicConfig.setTopicName(topic); + topicConfig.setWriteQueueNums(queueData.getWriteQueueNums()); + topicConfig.setReadQueueNums(queueData.getReadQueueNums()); + topicConfig.setPerm(queueData.getPerm()); + topicConfig.setTopicSysFlag(queueData.getTopicSynFlag()); + + //new perm + int perm = Integer.parseInt(commandLine.getOptionValue('p').trim()); + int oldPerm = topicConfig.getPerm(); + if (perm == oldPerm) { + System.out.printf("new perm equals to the old one!%n"); + return; + } + topicConfig.setPerm(perm); + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + System.out.printf("update topic perm from %s to %s in %s success.%n", oldPerm, perm, addr); + System.out.printf("%s%n", topicConfig); + return; + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + System.out.printf("update topic perm from %s to %s in %s success.%n", oldPerm, perm, addr); + } + return; + } + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java new file mode 100644 index 0000000000..c33018f289 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class UpdateTopicSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateTopic"; + } + + + @Override + public String commandDesc() { + return "Update or create topic"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "create topic to which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "create topic to which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("r", "readQueueNums", true, "set read queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("w", "writeQueueNums", true, "set write queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "perm", true, "set topic's permission(2|4|6), intro[2:W 4:R; 6:RW]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("o", "order", true, "set topic's order(true|false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("u", "unit", true, "is unit topic (true|false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "hasUnitSub", true, "has unit sub (true|false"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setReadQueueNums(8); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(commandLine.getOptionValue('t').trim()); + + // readQueueNums + if (commandLine.hasOption('r')) { + topicConfig.setReadQueueNums(Integer.parseInt(commandLine.getOptionValue('r').trim())); + } + + // writeQueueNums + if (commandLine.hasOption('w')) { + topicConfig.setWriteQueueNums(Integer.parseInt(commandLine.getOptionValue('w').trim())); + } + + // perm + if (commandLine.hasOption('p')) { + topicConfig.setPerm(Integer.parseInt(commandLine.getOptionValue('p').trim())); + } + + boolean isUnit = false; + if (commandLine.hasOption('u')) { + isUnit = Boolean.parseBoolean(commandLine.getOptionValue('u').trim()); + } + + boolean isCenterSync = false; + if (commandLine.hasOption('s')) { + isCenterSync = Boolean.parseBoolean(commandLine.getOptionValue('s').trim()); + } + + int topicCenterSync = TopicSysFlag.buildSysFlag(isUnit, isCenterSync); + topicConfig.setTopicSysFlag(topicCenterSync); + + boolean isOrder = false; + if (commandLine.hasOption('o')) { + isOrder = Boolean.parseBoolean(commandLine.getOptionValue('o').trim()); + } + topicConfig.setOrder(isOrder); + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + + if (isOrder) { + String brokerName = CommandUtil.fetchBrokerNameByAddr(defaultMQAdminExt, addr); + String orderConf = brokerName + ":" + topicConfig.getWriteQueueNums(); + defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(), orderConf, false); + System.out.printf(String.format("set broker orderConf. isOrder=%s, orderConf=[%s]", + isOrder, orderConf.toString())); + } + System.out.printf("create topic to %s success.%n", addr); + System.out.printf("%s", topicConfig); + return; + + } else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + System.out.printf("create topic to %s success.%n", addr); + } + + if (isOrder) { + Set brokerNameSet = + CommandUtil.fetchBrokerNameByClusterName(defaultMQAdminExt, clusterName); + StringBuilder orderConf = new StringBuilder(); + String splitor = ""; + for (String s : brokerNameSet) { + orderConf.append(splitor).append(s).append(":") + .append(topicConfig.getWriteQueueNums()); + splitor = ";"; + } + defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(), + orderConf.toString(), true); + System.out.printf(String.format("set cluster orderConf. isOrder=%s, orderConf=[%s]", + isOrder, orderConf.toString())); + } + + System.out.printf("%s", topicConfig); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } catch (Exception e) { + e.printStackTrace(); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java new file mode 100644 index 0000000000..b8176d4ef5 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.slf4j.Logger; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + + +public class DefaultMonitorListener implements MonitorListener { + private final static String LOG_PREFIX = "[MONITOR] "; + private final static String LOG_NOTIFY = LOG_PREFIX + " [NOTIFY] "; + private final Logger log = ClientLogger.getLog(); + + + public DefaultMonitorListener() { + } + + + @Override + public void beginRound() { + log.info(LOG_PREFIX + "=========================================beginRound"); + } + + + @Override + public void reportUndoneMsgs(UndoneMsgs undoneMsgs) { + log.info(String.format(LOG_PREFIX + "reportUndoneMsgs: %s", undoneMsgs)); + } + + + @Override + public void reportFailedMsgs(FailedMsgs failedMsgs) { + log.info(String.format(LOG_PREFIX + "reportFailedMsgs: %s", failedMsgs)); + } + + + @Override + public void reportDeleteMsgsEvent(DeleteMsgsEvent deleteMsgsEvent) { + log.info(String.format(LOG_PREFIX + "reportDeleteMsgsEvent: %s", deleteMsgsEvent)); + } + + + @Override + public void reportConsumerRunningInfo(TreeMap criTable) { + + { + boolean result = ConsumerRunningInfo.analyzeSubscription(criTable); + if (!result) { + log.info(String.format(LOG_NOTIFY + + "reportConsumerRunningInfo: ConsumerGroup: %s, Subscription different", criTable + .firstEntry().getValue().getProperties().getProperty("consumerGroup"))); + } + } + + + { + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String result = ConsumerRunningInfo.analyzeProcessQueue(next.getKey(), next.getValue()); + if (!result.isEmpty()) { + log.info(String.format(LOG_NOTIFY + + "reportConsumerRunningInfo: ConsumerGroup: %s, ClientId: %s, %s", + criTable.firstEntry().getValue().getProperties().getProperty("consumerGroup"), + next.getKey(), + result)); + } + } + } + } + + + @Override + public void endRound() { + log.info(LOG_PREFIX + "=========================================endRound"); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java new file mode 100644 index 0000000000..25ac420e23 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; + + +public class DeleteMsgsEvent { + private OffsetMovedEvent offsetMovedEvent; + private long eventTimestamp; + + + public OffsetMovedEvent getOffsetMovedEvent() { + return offsetMovedEvent; + } + + + public void setOffsetMovedEvent(OffsetMovedEvent offsetMovedEvent) { + this.offsetMovedEvent = offsetMovedEvent; + } + + + public long getEventTimestamp() { + return eventTimestamp; + } + + + public void setEventTimestamp(long eventTimestamp) { + this.eventTimestamp = eventTimestamp; + } + + + @Override + public String toString() { + return "DeleteMsgsEvent [offsetMovedEvent=" + offsetMovedEvent + ", eventTimestamp=" + eventTimestamp + + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java new file mode 100644 index 0000000000..3ae5c2f23c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +public class FailedMsgs { + private String consumerGroup; + private String topic; + private long failedMsgsTotalRecently; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public long getFailedMsgsTotalRecently() { + return failedMsgsTotalRecently; + } + + + public void setFailedMsgsTotalRecently(long failedMsgsTotalRecently) { + this.failedMsgsTotalRecently = failedMsgsTotalRecently; + } + + + @Override + public String toString() { + return "FailedMsgs [consumerGroup=" + consumerGroup + ", topic=" + topic + + ", failedMsgsTotalRecently=" + failedMsgsTotalRecently + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java new file mode 100644 index 0000000000..7ef45133b0 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.common.MixAll; + + +public class MonitorConfig { + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + + private int roundInterval = 1000 * 60; + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public int getRoundInterval() { + return roundInterval; + } + + + public void setRoundInterval(int roundInterval) { + this.roundInterval = roundInterval; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java new file mode 100644 index 0000000000..1586ef96c8 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; + +import java.util.TreeMap; + +public interface MonitorListener { + void beginRound(); + + void reportUndoneMsgs(UndoneMsgs undoneMsgs); + + void reportFailedMsgs(FailedMsgs failedMsgs); + + void reportDeleteMsgsEvent(DeleteMsgsEvent deleteMsgsEvent); + + void reportConsumerRunningInfo(TreeMap criTable); + + void endRound(); +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java new file mode 100644 index 0000000000..2b50862a7f --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java @@ -0,0 +1,325 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import org.slf4j.Logger; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class MonitorService { + private final Logger log = ClientLogger.getLog(); + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorService")); + + private final MonitorConfig monitorConfig; + + private final MonitorListener monitorListener; + + private final DefaultMQAdminExt defaultMQAdminExt; + private final DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer( + MixAll.TOOLS_CONSUMER_GROUP); + private final DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer( + MixAll.MONITOR_CONSUMER_GROUP); + + + public MonitorService(MonitorConfig monitorConfig, MonitorListener monitorListener, RPCHook rpcHook) { + this.monitorConfig = monitorConfig; + this.monitorListener = monitorListener; + + this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + this.defaultMQAdminExt.setInstanceName(instanceName()); + this.defaultMQAdminExt.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + + this.defaultMQPullConsumer.setInstanceName(instanceName()); + this.defaultMQPullConsumer.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + + this.defaultMQPushConsumer.setInstanceName(instanceName()); + this.defaultMQPushConsumer.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + try { + this.defaultMQPushConsumer.setConsumeThreadMin(1); + this.defaultMQPushConsumer.setConsumeThreadMax(1); + this.defaultMQPushConsumer.subscribe(MixAll.OFFSET_MOVED_EVENT, "*"); + this.defaultMQPushConsumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + try { + OffsetMovedEvent ome = + OffsetMovedEvent.decode(msgs.get(0).getBody(), OffsetMovedEvent.class); + + DeleteMsgsEvent deleteMsgsEvent = new DeleteMsgsEvent(); + deleteMsgsEvent.setOffsetMovedEvent(ome); + deleteMsgsEvent.setEventTimestamp(msgs.get(0).getStoreTimestamp()); + + MonitorService.this.monitorListener.reportDeleteMsgsEvent(deleteMsgsEvent); + } catch (Exception e) { + } + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + } catch (MQClientException e) { + } + } + + + private String instanceName() { + String name = + System.currentTimeMillis() + new Random().nextInt() + this.monitorConfig.getNamesrvAddr(); + + return "MonitorService_" + name.hashCode(); + } + + public static void main(String[] args) throws MQClientException { + main0(args, null); + } + + public static void main0(String[] args, RPCHook rpcHook) throws MQClientException { + final MonitorService monitorService = + new MonitorService(new MonitorConfig(), new DefaultMonitorListener(), rpcHook); + monitorService.start(); + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + + + @Override + public void run() { + synchronized (this) { + if (!this.hasShutdown) { + this.hasShutdown = true; + monitorService.shutdown(); + } + } + } + }, "ShutdownHook")); + } + + public void start() throws MQClientException { + this.defaultMQPullConsumer.start(); + this.defaultMQAdminExt.start(); + this.defaultMQPushConsumer.start(); + this.startScheduleTask(); + } + + public void shutdown() { + this.defaultMQPullConsumer.shutdown(); + this.defaultMQAdminExt.shutdown(); + this.defaultMQPushConsumer.shutdown(); + } + + private void startScheduleTask() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + MonitorService.this.doMonitorWork(); + } catch (Exception e) { + log.error("doMonitorWork Exception", e); + } + } + }, 1000 * 20, this.monitorConfig.getRoundInterval(), TimeUnit.MILLISECONDS); + } + + public void doMonitorWork() throws RemotingException, MQClientException, InterruptedException { + long beginTime = System.currentTimeMillis(); + this.monitorListener.beginRound(); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String consumerGroup = topic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + + try { + this.reportUndoneMsgs(consumerGroup); + } catch (Exception e) { + // log.error("reportUndoneMsgs Exception", e); + } + + + try { + this.reportConsumerRunningInfo(consumerGroup); + } catch (Exception e) { + // log.error("reportConsumerRunningInfo Exception", e); + } + } + } + this.monitorListener.endRound(); + long spentTimeMills = System.currentTimeMillis() - beginTime; + log.info("Execute one round monitor work, spent timemills: {}", spentTimeMills); + } + + private void reportUndoneMsgs(final String consumerGroup) { + ConsumeStats cs = null; + try { + cs = defaultMQAdminExt.examineConsumeStats(consumerGroup); + } catch (Exception e) { + return; + } + + ConsumerConnection cc = null; + try { + cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + } catch (Exception e) { + return; + } + + if (cs != null) { + + HashMap csByTopic = new HashMap(); + { + Iterator> it = cs.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + OffsetWrapper ow = next.getValue(); + ConsumeStats csTmp = csByTopic.get(mq.getTopic()); + if (null == csTmp) { + csTmp = new ConsumeStats(); + csByTopic.put(mq.getTopic(), csTmp); + } + + csTmp.getOffsetTable().put(mq, ow); + } + } + + + { + Iterator> it = csByTopic.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + UndoneMsgs undoneMsgs = new UndoneMsgs(); + undoneMsgs.setConsumerGroup(consumerGroup); + undoneMsgs.setTopic(next.getKey()); + this.computeUndoneMsgs(undoneMsgs, next.getValue()); + this.monitorListener.reportUndoneMsgs(undoneMsgs); + this.reportFailedMsgs(consumerGroup, next.getKey()); + } + } + } + } + + public void reportConsumerRunningInfo(final String consumerGroup) throws InterruptedException, + MQBrokerException, RemotingException, MQClientException { + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + TreeMap infoMap = new TreeMap(); + for (Connection c : cc.getConnectionSet()) { + String clientId = c.getClientId(); + + if (c.getVersion() < MQVersion.Version.V3_1_8_SNAPSHOT.ordinal()) { + continue; + } + + try { + ConsumerRunningInfo info = + defaultMQAdminExt.getConsumerRunningInfo(consumerGroup, clientId, false); + infoMap.put(clientId, info); + } catch (Exception e) { + } + } + + if (!infoMap.isEmpty()) { + this.monitorListener.reportConsumerRunningInfo(infoMap); + } + } + + private void computeUndoneMsgs(final UndoneMsgs undoneMsgs, final ConsumeStats consumeStats) { + long total = 0; + long singleMax = 0; + long delayMax = 0; + Iterator> it = consumeStats.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + OffsetWrapper ow = next.getValue(); + long diff = ow.getBrokerOffset() - ow.getConsumerOffset(); + + if (diff > singleMax) { + singleMax = diff; + } + + if (diff > 0) { + total += diff; + } + + // Delay + if (ow.getLastTimestamp() > 0) { + try { + long maxOffset = this.defaultMQPullConsumer.maxOffset(mq); + if (maxOffset > 0) { + PullResult pull = this.defaultMQPullConsumer.pull(mq, "*", maxOffset - 1, 1); + switch (pull.getPullStatus()) { + case FOUND: + long delay = + pull.getMsgFoundList().get(0).getStoreTimestamp() - ow.getLastTimestamp(); + if (delay > delayMax) { + delayMax = delay; + } + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break; + default: + break; + } + } + } catch (Exception e) { + } + } + } + + undoneMsgs.setUndoneMsgsTotal(total); + undoneMsgs.setUndoneMsgsSingleMQ(singleMax); + undoneMsgs.setUndoneMsgsDelayTimeMills(delayMax); + } + + private void reportFailedMsgs(final String consumerGroup, final String topic) { + + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java new file mode 100644 index 0000000000..1638d14b5e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.rocketmq.tools.monitor; + +public class UndoneMsgs { + private String consumerGroup; + private String topic; + + private long undoneMsgsTotal; + + private long undoneMsgsSingleMQ; + + private long undoneMsgsDelayTimeMills; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public long getUndoneMsgsTotal() { + return undoneMsgsTotal; + } + + + public void setUndoneMsgsTotal(long undoneMsgsTotal) { + this.undoneMsgsTotal = undoneMsgsTotal; + } + + + public long getUndoneMsgsSingleMQ() { + return undoneMsgsSingleMQ; + } + + + public void setUndoneMsgsSingleMQ(long undoneMsgsSingleMQ) { + this.undoneMsgsSingleMQ = undoneMsgsSingleMQ; + } + + + public long getUndoneMsgsDelayTimeMills() { + return undoneMsgsDelayTimeMills; + } + + + public void setUndoneMsgsDelayTimeMills(long undoneMsgsDelayTimeMills) { + this.undoneMsgsDelayTimeMills = undoneMsgsDelayTimeMills; + } + + + @Override + public String toString() { + return "UndoneMsgs [consumerGroup=" + consumerGroup + ", topic=" + topic + ", undoneMsgsTotal=" + + undoneMsgsTotal + ", undoneMsgsSingleMQ=" + undoneMsgsSingleMQ + + ", undoneMsgsDelayTimeMills=" + undoneMsgsDelayTimeMills + "]"; + } +}