From 9a9414452bd8a4d1345b06a49d25dda3dec26bec Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 31 Oct 2023 20:39:57 +0800 Subject: [PATCH] feat(monitor,region,host,cloudmon): support VictoriaMetrics as TSDB backend --- cmd/climc/shell/misc/version.go | 1 + go.mod | 4 + go.sum | 11 + pkg/apis/const.go | 10 +- pkg/apis/monitor/alert.go | 7 +- pkg/apis/monitor/datasource.go | 8 + pkg/apis/monitor/types.go | 4 - pkg/baremetal/cronjobs.go | 11 +- pkg/cloudcommon/tsdb/doc.go | 1 + pkg/cloudcommon/tsdb/tsdb.go | 55 + pkg/cloudmon/misc/alerts.go | 4 +- pkg/cloudmon/misc/pinger.go | 4 +- pkg/cloudmon/misc/system.go | 3 +- pkg/cloudmon/misc/usageservice.go | 6 +- pkg/cloudmon/providerdriver/base.go | 4 +- pkg/cloudmon/resources/resources.go | 4 +- pkg/compute/guestdrivers/base.go | 7 +- pkg/compute/models/hosts.go | 2 +- pkg/compute/models/loadbalanceragents.go | 4 +- pkg/compute/service/influxdb.go | 6 +- pkg/devtool/models/actions.go | 14 +- pkg/devtool/utils/influxdb_url.go | 25 +- pkg/devtool/utils/service_url.go | 5 +- pkg/hostman/hostinfo/hostinfo.go | 11 +- pkg/hostman/hostinfo/hostpinger.go | 2 +- pkg/hostman/metadata/metadatahandler.go | 8 +- pkg/hostman/system_service/telegraf.go | 14 +- pkg/lbagent/haproxy.go | 3 +- pkg/mcclient/modules/monitor/helper.go | 14 + pkg/mcclient/modules/monitor/helper_test.go | 17 - pkg/mcclient/options/monitor/ds.go | 4 + pkg/mcclient/session.go | 2 +- .../alerting/conditions/metricquery.go | 30 +- pkg/monitor/alerting/conditions/query.go | 28 +- pkg/monitor/alerting/engine.go | 2 +- pkg/monitor/controller/balancer/balancer.go | 4 +- pkg/monitor/controller/balancer/utils.go | 11 +- pkg/monitor/datasource/datasource.go | 156 + pkg/monitor/datasource/doc.go | 1 + pkg/monitor/models/alert.go | 19 +- pkg/monitor/models/datasource.go | 202 +- pkg/monitor/models/meteralert.go | 32 +- pkg/monitor/models/metric.go | 3 +- pkg/monitor/models/nodealert.go | 9 +- pkg/monitor/models/unifiedmonitor.go | 97 +- pkg/monitor/service/service.go | 10 +- pkg/monitor/subscriptionmodel/subcription.go | 4 +- .../tsdb/driver/influxdb/helper_test.go | 43 + pkg/monitor/tsdb/driver/influxdb/influxdb.go | 67 +- .../tsdb/driver/influxdb/response_parser.go | 21 +- .../tsdb/driver/victoriametrics/client.go | 153 + .../driver/victoriametrics/client_test.go | 20 + .../tsdb/driver/victoriametrics/doc.go | 1 + .../driver/victoriametrics/result_parser.go | 154 + .../victoriametrics/result_parser_test.go | 123 + pkg/monitor/tsdb/driver/victoriametrics/vm.go | 264 + .../tsdb/driver/victoriametrics/vm_test.go | 70 + pkg/monitor/tsdb/models.go | 71 +- pkg/monitor/tsdb/query_endpoint.go | 37 +- pkg/monitor/tsdb/request.go | 2 +- vendor/github.com/cespare/xxhash/LICENSE.txt | 22 + vendor/github.com/cespare/xxhash/README.md | 50 + vendor/github.com/cespare/xxhash/rotate.go | 14 + vendor/github.com/cespare/xxhash/rotate19.go | 14 + vendor/github.com/cespare/xxhash/xxhash.go | 168 + .../github.com/cespare/xxhash/xxhash_amd64.go | 12 + .../github.com/cespare/xxhash/xxhash_amd64.s | 233 + .../github.com/cespare/xxhash/xxhash_other.go | 75 + .../github.com/cespare/xxhash/xxhash_safe.go | 10 + .../cespare/xxhash/xxhash_unsafe.go | 30 + .../github.com/influxdata/influxql/.gitignore | 14 + .../influxdata/influxql/Jenkinsfile | 35 + vendor/github.com/influxdata/influxql/LICENSE | 20 + .../github.com/influxdata/influxql/README.md | 1170 ++++ vendor/github.com/influxdata/influxql/ast.go | 5814 +++++++++++++++++ vendor/github.com/influxdata/influxql/doc.go | 12 + .../influxdata/influxql/influxql.go | 3 + .../influxql/internal/internal.pb.go | 120 + .../influxql/internal/internal.proto | 14 + .../github.com/influxdata/influxql/params.go | 180 + .../influxdata/influxql/parse_tree.go | 236 + .../github.com/influxdata/influxql/parser.go | 3163 +++++++++ .../influxdata/influxql/sanitize.go | 47 + .../github.com/influxdata/influxql/scanner.go | 617 ++ .../github.com/influxdata/influxql/token.go | 332 + .../github.com/influxdata/promql/v2/LICENSE | 201 + .../github.com/influxdata/promql/v2/README.md | 40 + vendor/github.com/influxdata/promql/v2/ast.go | 319 + .../influxdata/promql/v2/functions.go | 374 ++ vendor/github.com/influxdata/promql/v2/lex.go | 917 +++ .../github.com/influxdata/promql/v2/parse.go | 1089 +++ .../influxdata/promql/v2/pkg/labels/labels.go | 368 ++ .../promql/v2/pkg/labels/matcher.go | 88 + .../influxdata/promql/v2/pkg/value/value.go | 34 + .../influxdata/promql/v2/printer.go | 198 + .../promql/v2/util/strutil/quote.go | 223 + .../promql/v2/util/strutil/strconv.go | 44 + .../github.com/influxdata/promql/v2/value.go | 199 + .../influxql-to-promql/converter/converter.go | 56 + .../converter/translator/promql.go | 521 ++ .../converter/translator/tranlator.go | 8 + vendor/modules.txt | 17 + 102 files changed, 18574 insertions(+), 436 deletions(-) create mode 100644 pkg/apis/monitor/datasource.go create mode 100644 pkg/cloudcommon/tsdb/doc.go create mode 100644 pkg/cloudcommon/tsdb/tsdb.go create mode 100644 pkg/monitor/datasource/datasource.go create mode 100644 pkg/monitor/datasource/doc.go create mode 100644 pkg/monitor/tsdb/driver/influxdb/helper_test.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/client.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/client_test.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/doc.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/result_parser.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/result_parser_test.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/vm.go create mode 100644 pkg/monitor/tsdb/driver/victoriametrics/vm_test.go create mode 100644 vendor/github.com/cespare/xxhash/LICENSE.txt create mode 100644 vendor/github.com/cespare/xxhash/README.md create mode 100644 vendor/github.com/cespare/xxhash/rotate.go create mode 100644 vendor/github.com/cespare/xxhash/rotate19.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.s create mode 100644 vendor/github.com/cespare/xxhash/xxhash_other.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_safe.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_unsafe.go create mode 100644 vendor/github.com/influxdata/influxql/.gitignore create mode 100644 vendor/github.com/influxdata/influxql/Jenkinsfile create mode 100644 vendor/github.com/influxdata/influxql/LICENSE create mode 100644 vendor/github.com/influxdata/influxql/README.md create mode 100644 vendor/github.com/influxdata/influxql/ast.go create mode 100644 vendor/github.com/influxdata/influxql/doc.go create mode 100644 vendor/github.com/influxdata/influxql/influxql.go create mode 100644 vendor/github.com/influxdata/influxql/internal/internal.pb.go create mode 100644 vendor/github.com/influxdata/influxql/internal/internal.proto create mode 100644 vendor/github.com/influxdata/influxql/params.go create mode 100644 vendor/github.com/influxdata/influxql/parse_tree.go create mode 100644 vendor/github.com/influxdata/influxql/parser.go create mode 100644 vendor/github.com/influxdata/influxql/sanitize.go create mode 100644 vendor/github.com/influxdata/influxql/scanner.go create mode 100644 vendor/github.com/influxdata/influxql/token.go create mode 100644 vendor/github.com/influxdata/promql/v2/LICENSE create mode 100644 vendor/github.com/influxdata/promql/v2/README.md create mode 100644 vendor/github.com/influxdata/promql/v2/ast.go create mode 100644 vendor/github.com/influxdata/promql/v2/functions.go create mode 100644 vendor/github.com/influxdata/promql/v2/lex.go create mode 100644 vendor/github.com/influxdata/promql/v2/parse.go create mode 100644 vendor/github.com/influxdata/promql/v2/pkg/labels/labels.go create mode 100644 vendor/github.com/influxdata/promql/v2/pkg/labels/matcher.go create mode 100644 vendor/github.com/influxdata/promql/v2/pkg/value/value.go create mode 100644 vendor/github.com/influxdata/promql/v2/printer.go create mode 100644 vendor/github.com/influxdata/promql/v2/util/strutil/quote.go create mode 100644 vendor/github.com/influxdata/promql/v2/util/strutil/strconv.go create mode 100644 vendor/github.com/influxdata/promql/v2/value.go create mode 100644 vendor/github.com/zexi/influxql-to-promql/converter/converter.go create mode 100644 vendor/github.com/zexi/influxql-to-promql/converter/translator/promql.go create mode 100644 vendor/github.com/zexi/influxql-to-promql/converter/translator/tranlator.go diff --git a/cmd/climc/shell/misc/version.go b/cmd/climc/shell/misc/version.go index 623bb8a09a..6af3deaf9d 100644 --- a/cmd/climc/shell/misc/version.go +++ b/cmd/climc/shell/misc/version.go @@ -96,6 +96,7 @@ func init() { apis.SERVICE_TYPE_OFFLINE_CLOUDMETA, apis.SERVICE_TYPE_CLOUDMETA, apis.SERVICE_TYPE_INFLUXDB, + apis.SERVICE_TYPE_VICTORIA_METRICS, apis.SERVICE_TYPE_ETCD, "torrent-tracker", }) { diff --git a/go.mod b/go.mod index ba4fa61894..c18a1389f7 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,8 @@ require ( github.com/gosuri/uitable v0.0.0-20160404203958-36ee7e946282 github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead github.com/hugozhu/godingtalk v1.0.6 + github.com/influxdata/influxql v1.1.0 + github.com/influxdata/promql/v2 v2.12.0 github.com/jaypipes/ghw v0.11.0 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c github.com/lestrrat-go/jwx v1.0.2 @@ -62,6 +64,7 @@ require ( github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/xuri/excelize/v2 v2.7.1 + github.com/zexi/influxql-to-promql v0.0.5 go.etcd.io/etcd/api/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 golang.org/x/crypto v0.8.0 @@ -132,6 +135,7 @@ require ( github.com/boltdb/bolt v1.3.1 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/coreos/go-semver v0.3.0 // indirect diff --git a/go.sum b/go.sum index a0569d24d4..da9ccaa72a 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/LeeEirc/terminalparser v0.0.0-20220328021224-de16b7643ea4/go.mod h1:t github.com/Microsoft/azure-vhd-utils v0.0.0-20181115010904-44cbada2ece3 h1:gImoAO1xAcC1oDlYmD/X7dggsodGf2DFJOVE5m0ssms= github.com/Microsoft/azure-vhd-utils v0.0.0-20181115010904-44cbada2ece3/go.mod h1:u0H9gMieFLxkUy8RS0X8VbFWyPs2815qQAaitRbj6x0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= @@ -173,6 +175,8 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w github.com/c-bata/go-prompt v0.2.4 h1:7pKUJ3CUgzdu1HJeWhNRkpVyY/NnlJhM/7d6YgHNOao= github.com/c-bata/go-prompt v0.2.4/go.mod h1:PqlttLXp0E7bZcoDW+dmzyKqFbmQTFoNzGSuW/AQRmo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -421,6 +425,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/influxdata/influxql v1.1.0 h1:sPsaumLFRPMwR5QtD3Up54HXpNND8Eu7G1vQFmi3quQ= +github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= +github.com/influxdata/promql/v2 v2.12.0 h1:kXn3p0D7zPw16rOtfDR+wo6aaiH8tSMfhPwONTxrlEc= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/ipandtcp/godingtalk v0.0.0-20180410032244-ca3d6ac197fb/go.mod h1:3umLciE1jBnh2LgrL15R2zbEe+HVPRZXnlSjjb/dYiw= github.com/jaypipes/ghw v0.11.0 h1:i0pKvAM7eZk0KvLm9vzpcpDKTRnfR6AQ5pFkPVnYJXU= github.com/jaypipes/ghw v0.11.0/go.mod h1:jeJGbkRB2lL3/gxYzNYzEDETV1ZJ56OKr+CSeSEym+g= @@ -599,6 +607,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= @@ -734,6 +743,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zexi/influxql-to-promql v0.0.5 h1:4gRTqrof31O2Ku443RoTh20AadJG4sXbREn8i892WQo= +github.com/zexi/influxql-to-promql v0.0.5/go.mod h1:a+S9SfEZ/aVyBM5abA7UCQmp7vD8Q9o+GfkQhk5mM58= go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= diff --git a/pkg/apis/const.go b/pkg/apis/const.go index 3282d2f597..46d58367e3 100644 --- a/pkg/apis/const.go +++ b/pkg/apis/const.go @@ -36,8 +36,10 @@ const ( SERVICE_TYPE_CLOUDMON = "cloudmon" SERVICE_TYPE_VPCAGENT = "vpcagent" - SERVICE_TYPE_ETCD = "etcd" - SERVICE_TYPE_INFLUXDB = "influxdb" + SERVICE_TYPE_ETCD = "etcd" + SERVICE_TYPE_INFLUXDB = "influxdb" + SERVICE_TYPE_NTP = "ntp" + SERVICE_TYPE_VICTORIA_METRICS = "victoria-metrics" SERVICE_TYPE_SCHEDULEDTASK = "scheduledtask" @@ -101,9 +103,9 @@ var ( "apigateway", "yunionapi", "billing", - "etcd", + SERVICE_TYPE_ETCD, "itsm", - "ntp", + SERVICE_TYPE_NTP, } ) diff --git a/pkg/apis/monitor/alert.go b/pkg/apis/monitor/alert.go index 502bdc3c3f..7bc0e7664e 100644 --- a/pkg/apis/monitor/alert.go +++ b/pkg/apis/monitor/alert.go @@ -93,10 +93,9 @@ type AlertCondition struct { } type AlertQuery struct { - Model MetricQuery `json:"model"` - DataSourceId string `json:"data_source_id"` - From string `json:"from"` - To string `json:"to"` + Model MetricQuery `json:"model"` + From string `json:"from"` + To string `json:"to"` } type AlertCreateInput struct { diff --git a/pkg/apis/monitor/datasource.go b/pkg/apis/monitor/datasource.go new file mode 100644 index 0000000000..f36b3428ec --- /dev/null +++ b/pkg/apis/monitor/datasource.go @@ -0,0 +1,8 @@ +package monitor + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + DataSourceTypeInfluxdb = apis.SERVICE_TYPE_INFLUXDB + DataSourceTypeVictoriaMetrics = apis.SERVICE_TYPE_VICTORIA_METRICS +) diff --git a/pkg/apis/monitor/types.go b/pkg/apis/monitor/types.go index 3608cac530..1d19583b33 100644 --- a/pkg/apis/monitor/types.go +++ b/pkg/apis/monitor/types.go @@ -14,10 +14,6 @@ package monitor -const ( - DataSourceTypeInfluxdb = "influxdb" -) - type DataSourceConfig struct { Id string Name string diff --git a/pkg/baremetal/cronjobs.go b/pkg/baremetal/cronjobs.go index c08695c631..cd9ab8bf84 100644 --- a/pkg/baremetal/cronjobs.go +++ b/pkg/baremetal/cronjobs.go @@ -26,6 +26,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" o "yunion.io/x/onecloud/pkg/baremetal/options" "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" modules "yunion.io/x/onecloud/pkg/mcclient/modules/logger" @@ -221,11 +222,11 @@ func (job *SSendMetricsJob) Name() string { func (job *SSendMetricsJob) Do(ctx context.Context, now time.Time) error { s := auth.GetAdminSession(ctx, consts.GetRegion()) - urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType) + src, err := tsdb.GetDefaultServiceSource(s, o.Options.SessionEndpointType) if err != nil { - return errors.Wrap(err, "s.GetServiceURLs") + return errors.Wrap(err, "tsdb.GetDefaultServiceSource") } - if len(urls) == 0 { + if len(src.URLs) == 0 { return nil } if !job.baremetal.isRedfishCapable() { @@ -253,9 +254,9 @@ func (job *SSendMetricsJob) Do(ctx context.Context, now time.Time) error { Timestamp: now, }) } - err = influxdb.SendMetrics(urls, "telegraf", metrics, false) + err = influxdb.SendMetrics(src.URLs, "telegraf", metrics, false) if err != nil { - return errors.Wrap(err, "influxdb.SendMetrics") + return errors.Wrapf(err, "tsdb.SendMetrics %q", src.Type) } job.lastTime = now return nil diff --git a/pkg/cloudcommon/tsdb/doc.go b/pkg/cloudcommon/tsdb/doc.go new file mode 100644 index 0000000000..c3746f4421 --- /dev/null +++ b/pkg/cloudcommon/tsdb/doc.go @@ -0,0 +1 @@ +package tsdb // import "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" diff --git a/pkg/cloudcommon/tsdb/tsdb.go b/pkg/cloudcommon/tsdb/tsdb.go new file mode 100644 index 0000000000..41bf2d1370 --- /dev/null +++ b/pkg/cloudcommon/tsdb/tsdb.go @@ -0,0 +1,55 @@ +package tsdb + +import ( + "math/rand" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type TSDBServiceSource struct { + Type string + URLs []string +} + +func NewTSDBServiceSource(t string, urls []string) *TSDBServiceSource { + return &TSDBServiceSource{ + Type: t, + URLs: urls, + } +} + +func GetDefaultServiceSource(s *mcclient.ClientSession, endpointType string) (*TSDBServiceSource, error) { + errs := []error{} + for _, sType := range []string{apis.SERVICE_TYPE_INFLUXDB, apis.SERVICE_TYPE_VICTORIA_METRICS} { + urls, err := s.GetServiceURLs(sType, endpointType) + if err != nil { + errs = append(errs, errors.Wrapf(err, "get %s service type %q", endpointType, sType)) + } + if len(urls) != 0 { + return NewTSDBServiceSource(sType, urls), nil + } + } + return nil, errors.NewAggregate(errs) +} + +func GetDefaultServiceSourceURLs(s *mcclient.ClientSession, endpointType string) ([]string, error) { + src, err := GetDefaultServiceSource(s, endpointType) + if err != nil { + return nil, errors.Wrap(err, "GetDefaultServiceSource") + } + if len(src.URLs) == 0 { + return nil, errors.Errorf("tsdb source %q URLs are empty", src.Type) + } + return src.URLs, nil +} + +func GetDefaultServiceSourceURL(s *mcclient.ClientSession, endpointType string) (string, error) { + urls, err := GetDefaultServiceSourceURLs(s, endpointType) + if err != nil { + return "", err + } + return urls[rand.Intn(len(urls))], nil +} diff --git a/pkg/cloudmon/misc/alerts.go b/pkg/cloudmon/misc/alerts.go index de70f99edc..824d712854 100644 --- a/pkg/cloudmon/misc/alerts.go +++ b/pkg/cloudmon/misc/alerts.go @@ -23,8 +23,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloutpost/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -101,7 +101,7 @@ func AlertHistoryReport(ctx context.Context, userCred mcclient.TokenCredential, } metrics = append(metrics, metric) } - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { return errors.Wrap(err, "GetServiceURLs") } diff --git a/pkg/cloudmon/misc/pinger.go b/pkg/cloudmon/misc/pinger.go index 529efe060c..edaba82a0b 100644 --- a/pkg/cloudmon/misc/pinger.go +++ b/pkg/cloudmon/misc/pinger.go @@ -25,8 +25,8 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/rbacscope" - "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -78,7 +78,7 @@ func PingProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart b } metrics = append(metrics, m...) } - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { return errors.Wrap(err, "GetServiceURLs") } diff --git a/pkg/cloudmon/misc/system.go b/pkg/cloudmon/misc/system.go index b2dbb406a4..2412595fdf 100644 --- a/pkg/cloudmon/misc/system.go +++ b/pkg/cloudmon/misc/system.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -100,7 +101,7 @@ func CollectServiceMetrics(ctx context.Context, userCred mcclient.TokenCredentia } metrics = append(metrics, part...) } - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { return errors.Wrap(err, "GetServiceURLs") } diff --git a/pkg/cloudmon/misc/usageservice.go b/pkg/cloudmon/misc/usageservice.go index 196655e811..6fd188e573 100644 --- a/pkg/cloudmon/misc/usageservice.go +++ b/pkg/cloudmon/misc/usageservice.go @@ -24,7 +24,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -89,8 +89,8 @@ func UsegReport(ctx context.Context, userCred mcclient.TokenCredential, isStart return errors.Wrap(err, "getDomainAndProjectServerUsage err") } dataList = append(dataList, data...) - //写入influDb - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + // 写入 influxdb 或者 VictoriaMetrics + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { return errors.Wrap(err, "GetServiceURLs") } diff --git a/pkg/cloudmon/providerdriver/base.go b/pkg/cloudmon/providerdriver/base.go index bb7d918251..4ba8c3f1f2 100644 --- a/pkg/cloudmon/providerdriver/base.go +++ b/pkg/cloudmon/providerdriver/base.go @@ -24,8 +24,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/util/influxdb" @@ -94,7 +94,7 @@ func (self *SBaseCollectDriver) CollectStorageMetrics(ctx context.Context, manag func (self *SBaseCollectDriver) sendMetrics(ctx context.Context, manager api.CloudproviderDetails, resName string, resCnt int, metrics []influxdb.SMetricData) error { s := auth.GetAdminSession(ctx, options.Options.Region) - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { return errors.Wrap(err, "GetServiceURLs") } diff --git a/pkg/cloudmon/resources/resources.go b/pkg/cloudmon/resources/resources.go index b4c7a91d43..746bea14d0 100644 --- a/pkg/cloudmon/resources/resources.go +++ b/pkg/cloudmon/resources/resources.go @@ -26,9 +26,9 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/cloudmon/providerdriver" "yunion.io/x/onecloud/pkg/mcclient" @@ -907,7 +907,7 @@ func (self *SResources) CollectMetrics(ctx context.Context, userCred mcclient.To } metrics = append(metrics, metric) } - urls, err := s.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.SessionEndpointType) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { log.Errorf("Get influxdb %s service url: %v", options.Options.SessionEndpointType, err) return diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index db0aac282a..b8371822a2 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -28,11 +28,11 @@ import ( "yunion.io/x/pkg/util/billing" "yunion.io/x/pkg/util/osprofile" - "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" guestdriver_types "yunion.io/x/onecloud/pkg/compute/guestdrivers/types" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" @@ -526,11 +526,12 @@ func (drv *SBaseGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcc func (drv *SBaseGuestDriver) FetchMonitorUrl(ctx context.Context, guest *models.SGuest) string { s := auth.GetAdminSessionWithPublic(ctx, consts.GetRegion()) - influxdbUrl, err := s.GetServiceURL(apis.SERVICE_TYPE_INFLUXDB, options.Options.MonitorEndpointType) + tsdbURL, err := tsdb.GetDefaultServiceSourceURL(s, options.Options.MonitorEndpointType) if err != nil { log.Errorf("FetchMonitorUrl fail %s", err) + return "" } - return influxdbUrl + return tsdbURL } func (drv *SBaseGuestDriver) RequestResetNicTrafficLimit(ctx context.Context, task taskman.ITask, host *models.SHost, guest *models.SGuest, input []api.ServerNicTrafficLimit) error { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index c7895c8974..1bedf77969 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -4226,7 +4226,7 @@ func (hh *SHost) PerformPing(ctx context.Context, userCred mcclient.TokenCredent } result := jsonutils.NewDict() result.Set("name", jsonutils.NewString(hh.GetName())) - dependSvcs := []string{"ntpd", "kafka", "influxdb", "elasticsearch"} + dependSvcs := []string{"ntpd", "kafka", apis.SERVICE_TYPE_INFLUXDB, apis.SERVICE_TYPE_VICTORIA_METRICS, "elasticsearch"} catalog := auth.GetCatalogData(dependSvcs, options.Options.Region) if catalog == nil { return nil, fmt.Errorf("Get catalog error") diff --git a/pkg/compute/models/loadbalanceragents.go b/pkg/compute/models/loadbalanceragents.go index 43a6b04083..f708037846 100644 --- a/pkg/compute/models/loadbalanceragents.go +++ b/pkg/compute/models/loadbalanceragents.go @@ -34,6 +34,7 @@ import ( identity_apis "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" @@ -279,8 +280,7 @@ func (p *SLoadbalancerAgentParamsTelegraf) updateBy(pp *SLoadbalancerAgentParams func (p *SLoadbalancerAgentParamsTelegraf) initDefault(data *jsonutils.JSONDict) { if p.InfluxDbOutputUrl == "" { baseOpts := &options.Options - u, _ := auth.GetServiceURL("influxdb", baseOpts.Region, "", - identity_apis.EndpointInterfacePublic) + u, _ := tsdb.GetDefaultServiceSourceURL(auth.GetAdminSession(context.Background(), baseOpts.Region), identity_apis.EndpointInterfacePublic) p.InfluxDbOutputUrl = u p.InfluxDbOutputUnsafeSsl = true } diff --git a/pkg/compute/service/influxdb.go b/pkg/compute/service/influxdb.go index 2dafc6b0fa..878d3b00f9 100644 --- a/pkg/compute/service/influxdb.go +++ b/pkg/compute/service/influxdb.go @@ -15,12 +15,13 @@ package service import ( + "context" "fmt" "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -32,7 +33,8 @@ type sInfluxdbEndpointListener struct { } func (listener *sInfluxdbEndpointListener) OnServiceCatalogChange(catalog mcclient.IServiceCatalog) { - urls, err := auth.GetServiceURLs(apis.SERVICE_TYPE_INFLUXDB, options.Options.Region, "", "") + s := auth.GetAdminSession(context.Background(), options.Options.Region) + urls, err := tsdb.GetDefaultServiceSourceURLs(s, options.Options.SessionEndpointType) if err != nil { log.Debugf("sInfluxdbEndpointListener: no influxdb endpoints found, retry later...") return diff --git a/pkg/devtool/models/actions.go b/pkg/devtool/models/actions.go index 189861ad9e..d0fa360e8e 100644 --- a/pkg/devtool/models/actions.go +++ b/pkg/devtool/models/actions.go @@ -20,9 +20,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" apis "yunion.io/x/onecloud/pkg/apis/ansible" apiidentity "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" ansible_modules "yunion.io/x/onecloud/pkg/mcclient/modules/ansible" @@ -59,27 +61,25 @@ func getServerAttrs(ID string, s *mcclient.ClientSession) (map[string]string, er return params, err } -func getInfluxdbURL() (string, error) { +func getTSDBURL() (string, error) { s := auth.GetAdminSessionWithPublic(nil, "") - url, err := s.GetServiceURL("influxdb", apiidentity.EndpointInterfacePublic) - + url, err := tsdb.GetDefaultServiceSourceURL(s, apiidentity.EndpointInterfacePublic) if err != nil { - log.Errorf("get influxdb Endpoint error %s", err) - return "", err + return "", errors.Wrap(err, "get influxdb Endpoint error") } return url, nil } func renderExtraVars(vars map[string]string) { - InfluxdbURL, err := getInfluxdbURL() + tsdbURL, err := getTSDBURL() if err != nil { log.Errorf("template binding: get influxdb url error: %s", err) return } for key, value := range vars { if key == "influxdb" && value == "INFLUXDB" { - vars[key] = InfluxdbURL + vars[key] = tsdbURL } } } diff --git a/pkg/devtool/utils/influxdb_url.go b/pkg/devtool/utils/influxdb_url.go index f013fd19b7..f9fa4e3b32 100644 --- a/pkg/devtool/utils/influxdb_url.go +++ b/pkg/devtool/utils/influxdb_url.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/sets" + "yunion.io/x/onecloud/pkg/apis" ansible_api "yunion.io/x/onecloud/pkg/apis/ansible" comapi "yunion.io/x/onecloud/pkg/apis/compute" ) @@ -55,16 +56,26 @@ func GetArgs(ctx context.Context, serverId, proxyEndpointId string, others inter if err != nil { return nil, errors.Wrapf(err, "unable to get serverInfo of server %s", serverId) } - influxdbUrl := info.serverDetails.MonitorUrl - log.Infof("influxdbUrl: %s", influxdbUrl) - influxdbUrl, err = FindValidServiceUrl(ctx, Service{"influxdb", influxdbUrl}, proxyEndpointId, info, host) - if err != nil { - return nil, errors.Wrapf(err, "unable to convertInfluxdbUrl %s", influxdbUrl) + monitorUrl := info.serverDetails.MonitorUrl + log.Infof("TSDB monitor Url: %s", monitorUrl) + foundSvc := false + errs := []error{} + for _, svcName := range []string{apis.SERVICE_TYPE_INFLUXDB, apis.SERVICE_TYPE_VICTORIA_METRICS} { + if tsdbUrl, err := FindValidServiceUrl(ctx, Service{svcName, monitorUrl}, proxyEndpointId, info, host); err != nil { + errs = append(errs, errors.Wrapf(err, "unable to convertInfluxdbUrl %s", monitorUrl)) + } else { + monitorUrl = tsdbUrl + foundSvc = true + break + } } - if len(influxdbUrl) == 0 { + if !foundSvc { + return nil, errors.Wrapf(errors.NewAggregate(errs), "convert TSDB service URL") + } + if len(monitorUrl) == 0 { return nil, errors.Wrap(ErrCannotReachInfluxbd, "please create usable Proxy Endpoint for server and try again") } - return getArgs(&info, influxdbUrl), nil + return getArgs(&info, monitorUrl), nil } func getArgs(info *sServerInfo, influxdbUrl string) map[string]interface{} { tags := map[string]string{ diff --git a/pkg/devtool/utils/service_url.go b/pkg/devtool/utils/service_url.go index b0b51ce099..83efc045c3 100644 --- a/pkg/devtool/utils/service_url.go +++ b/pkg/devtool/utils/service_url.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/pkg/util/httputils" "yunion.io/x/pkg/util/sets" + "yunion.io/x/onecloud/pkg/apis" ansible_api "yunion.io/x/onecloud/pkg/apis/ansible" proxy_api "yunion.io/x/onecloud/pkg/apis/cloudproxy" comapi "yunion.io/x/onecloud/pkg/apis/compute" @@ -46,7 +47,7 @@ type Service struct { func serviceComplete(serviceName, address string, port int) (url, checkUrl string, expectedCode int) { switch serviceName { - case "influxdb": + case apis.SERVICE_TYPE_INFLUXDB, apis.SERVICE_TYPE_VICTORIA_METRICS: return fmt.Sprintf("https://%s:%d", address, port), fmt.Sprintf("https://%s:%d/ping", address, port), 204 case "repo": return fmt.Sprintf("http://%s:%d", address, port), fmt.Sprintf("http://%s:%d", address, port), 200 @@ -57,7 +58,7 @@ func serviceComplete(serviceName, address string, port int) (url, checkUrl strin func serviceComplete2(service Service) (completeUrl string, expectedCode int) { switch service.Name { - case "influxdb": + case apis.SERVICE_TYPE_INFLUXDB, apis.SERVICE_TYPE_VICTORIA_METRICS: return fmt.Sprintf("%s/ping", service.Url), 204 case "repo": return service.Url, 200 diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 7a0a443291..68772f4101 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -44,6 +44,7 @@ import ( napi "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" @@ -2284,9 +2285,13 @@ func (h *SHostInfo) OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3) if len(urls) > 0 { conf["kafka"] = map[string]interface{}{"brokers": urls, "topic": "telegraf"} } - urls, _ = s.GetServiceURLs("influxdb", defaultEndpointType) - if len(urls) > 0 { - conf["influxdb"] = map[string]interface{}{"url": urls, "database": "telegraf"} + tsdb, _ := tsdb.GetDefaultServiceSource(s, defaultEndpointType) + if tsdb != nil && len(tsdb.URLs) > 0 { + conf[apis.SERVICE_TYPE_INFLUXDB] = map[string]interface{}{ + "url": tsdb.URLs, + "database": "telegraf", + "tsdb_type": tsdb.Type, + } } if !reflect.DeepEqual(telegraf.GetConf(), conf) || (!strings.Contains(svcs, "telegraf") && !telegraf.IsActive()) { log.Debugf("telegraf config: %s", conf) diff --git a/pkg/hostman/hostinfo/hostpinger.go b/pkg/hostman/hostinfo/hostpinger.go index b69adef821..5febe744bc 100644 --- a/pkg/hostman/hostinfo/hostpinger.go +++ b/pkg/hostman/hostinfo/hostpinger.go @@ -143,7 +143,7 @@ func (p *SHostPingTask) ping(div int, hostId string) error { Instance().OnCatalogChanged(cl) } else { - log.Errorln(err) + log.Errorf("get catalog from res %s: %v", res.String(), err) } } return nil diff --git a/pkg/hostman/metadata/metadatahandler.go b/pkg/hostman/metadata/metadatahandler.go index 8eeba25dce..69ec5e6c62 100644 --- a/pkg/hostman/metadata/metadatahandler.go +++ b/pkg/hostman/metadata/metadatahandler.go @@ -31,6 +31,7 @@ import ( identity_api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -312,7 +313,12 @@ func (s *Service) monitorReverseEndpoint() *proxy.SEndpointFactory { if guestDesc == nil { return "", httperrors.NewNotFoundError("vm not found") } - return auth.GetServiceURL("influxdb", options.HostOptions.Region, guestDesc.Zone, identity_api.EndpointInterfaceInternal) + s := auth.GetAdminSession(ctx, options.HostOptions.Region) + srcURL, err := tsdb.GetDefaultServiceSourceURL(s, identity_api.EndpointInterfaceInternal) + if err != nil { + return "", errors.Wrap(err, "monitorReverseEndpoint get tsdb url") + } + return srcURL, nil } return proxy.NewEndpointFactory(f, "monitorService") } diff --git a/pkg/hostman/system_service/telegraf.go b/pkg/hostman/system_service/telegraf.go index 094e01d4d6..3255da2d52 100644 --- a/pkg/hostman/system_service/telegraf.go +++ b/pkg/hostman/system_service/telegraf.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -76,11 +77,17 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string { conf += fmt.Sprintf(" hostname = \"%s\"\n", hostname) conf += " omit_hostname = false\n" conf += "\n" - if ifluxb, ok := kwargs["influxdb"]; ok { - influxdb, _ := ifluxb.(map[string]interface{}) + if influx, ok := kwargs[apis.SERVICE_TYPE_INFLUXDB]; ok { + influxdb, _ := influx.(map[string]interface{}) inUrls, _ := influxdb["url"] tUrls, _ := inUrls.([]string) inDatabase, _ := influxdb["database"] + isVM := false + if tsdbType, ok := influxdb["tsdb_type"]; ok { + if tsdbType.(string) == apis.SERVICE_TYPE_VICTORIA_METRICS { + isVM = true + } + } tdb, _ := inDatabase.(string) urls := []string{} for _, u := range tUrls { @@ -90,6 +97,9 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string { conf += fmt.Sprintf(" urls = [%s]\n", strings.Join(urls, ", ")) conf += fmt.Sprintf(" database = \"%s\"\n", tdb) conf += " insecure_skip_verify = true\n" + if isVM { + conf += " skip_database_creation = true\n" + } conf += " timeout = \"30s\"\n" conf += "\n" } diff --git a/pkg/lbagent/haproxy.go b/pkg/lbagent/haproxy.go index 785d1804a5..e1778d1a96 100644 --- a/pkg/lbagent/haproxy.go +++ b/pkg/lbagent/haproxy.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts" "yunion.io/x/onecloud/pkg/hostman/system_service" agentmodels "yunion.io/x/onecloud/pkg/lbagent/models" @@ -445,7 +446,7 @@ func (h *HaproxyHelper) remoteReloadTelegraf(ctx context.Context, agentParams *a } conf["nics"] = h.getNicsTelegrafConf() if len(agentParams.AgentModel.Params.Telegraf.InfluxDbOutputUrl) > 0 { - conf["influxdb"] = map[string]interface{}{ + conf[apis.SERVICE_TYPE_INFLUXDB] = map[string]interface{}{ "url": []string{ agentParams.AgentModel.Params.Telegraf.InfluxDbOutputUrl, }, diff --git a/pkg/mcclient/modules/monitor/helper.go b/pkg/mcclient/modules/monitor/helper.go index d30f54e854..00ef634d8c 100644 --- a/pkg/mcclient/modules/monitor/helper.go +++ b/pkg/mcclient/modules/monitor/helper.go @@ -456,6 +456,10 @@ func (s *AlertQuerySelect) MAX() *AlertQuerySelect { return s.addFunc("max") } +func (s *AlertQuerySelect) LAST() *AlertQuerySelect { + return s.addFunc("last") +} + // AS is alias method func (s *AlertQuerySelect) AS(alias string) *AlertQuerySelect { s.MetricQuerySelect = append(s.MetricQuerySelect, monitor.MetricQueryPart{ @@ -538,6 +542,16 @@ func (w *AlertQueryWhere) IN(key string, vals []string) *AlertQueryWhere { return w.REGEX(key, valStr) } +func (w *AlertQueryWhere) AddTag(tag *monitor.MetricQueryTag) *AlertQueryWhere { + if tag == nil { + return w + } + if tag.Condition != "" { + w.cond = tag.Condition + } + return w.filter(tag.Operator, tag.Key, tag.Value) +} + func (w *AlertQueryWhere) newTag(op string, key string, value string) monitor.MetricQueryTag { return monitor.MetricQueryTag{ Key: key, diff --git a/pkg/mcclient/modules/monitor/helper_test.go b/pkg/mcclient/modules/monitor/helper_test.go index e03efaf6a5..96f7072c64 100644 --- a/pkg/mcclient/modules/monitor/helper_test.go +++ b/pkg/mcclient/modules/monitor/helper_test.go @@ -20,7 +20,6 @@ import ( . "github.com/smartystreets/goconvey/convey" "yunion.io/x/onecloud/pkg/apis/monitor" - "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb" ) func TestHelperWhere(t *testing.T) { @@ -86,22 +85,6 @@ func TestHelperSelects(t *testing.T) { }) } -func TestAlertQuery(t *testing.T) { - Convey("Alert query test", t, func() { - parser := new(influxdb.InfluxdbQueryParser) - q := NewAlertQuery("telegraf", "diskio").From("5m").To("now") - q.Selects().Select("await").MEAN() - q.Where().Equal("hostname", "host1").Equal("provider", "kvm") - q.GroupBy().TAG("*").FILL_NULL() - qCtx := q.ToTsdbQuery() - influxdbQ, err := parser.Parse(qCtx.Queries[0], nil) - So(err, ShouldBeNil) - rawQuery, err := influxdbQ.Build(qCtx) - So(err, ShouldBeNil) - So(rawQuery, ShouldEqual, `SELECT mean("await") FROM "diskio" WHERE ("hostname" = 'host1' AND "provider" = 'kvm') AND time > now() - 5m GROUP BY * fill(null)`) - }) -} - func TestAlertConfig(t *testing.T) { Convey("Alert config test", t, func() { enabled := true diff --git a/pkg/mcclient/options/monitor/ds.go b/pkg/mcclient/options/monitor/ds.go index 9d978c7771..c8ed45c2c1 100644 --- a/pkg/mcclient/options/monitor/ds.go +++ b/pkg/mcclient/options/monitor/ds.go @@ -29,6 +29,10 @@ type DataSourceListOptions struct { options.BaseListOptions } +func (d DataSourceListOptions) Params() (jsonutils.JSONObject, error) { + return d.BaseListOptions.Params() +} + type DataSourceDeleteOptions struct { ID string `json:"-"` } diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index 0231b6a287..286b4dab7e 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -212,7 +212,7 @@ func (this *ClientSession) getServiceVersionURLs(service, region, zone, endpoint return []string{this.client.authUrl}, nil } urls, err := catalog.getServiceURLs(service, region, zone, endpointType) - // HACK! in case of fail to get kestone url or schema of keystone changed, always trust authUrl + // HACK! in case of fail to get keystone url or schema of keystone changed, always trust authUrl if service == api.SERVICE_TYPE && (err != nil || len(urls) == 0 || (len(this.client.authUrl) != 0 && this.client.authUrl[:5] != urls[0][:5])) { var msg string if err != nil { diff --git a/pkg/monitor/alerting/conditions/metricquery.go b/pkg/monitor/alerting/conditions/metricquery.go index 64436dd432..88b38e3b3c 100644 --- a/pkg/monitor/alerting/conditions/metricquery.go +++ b/pkg/monitor/alerting/conditions/metricquery.go @@ -31,8 +31,8 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/monitor/alerting" + "yunion.io/x/onecloud/pkg/monitor/datasource" mq "yunion.io/x/onecloud/pkg/monitor/metricquery" - "yunion.io/x/onecloud/pkg/monitor/models" "yunion.io/x/onecloud/pkg/monitor/tsdb" "yunion.io/x/onecloud/pkg/monitor/validators" ) @@ -67,7 +67,6 @@ func NewMetricQueryCondition(models []*monitor.AlertCondition) (*MetricQueryCond return nil, errors.Wrapf(err, "to value %q", qc.Query.To) } qc.setResType() - qc.Query.DataSourceId = q.DataSourceId cond.QueryCons = append(cond.QueryCons, *qc) } @@ -92,7 +91,7 @@ func (query *MetricQueryCondition) ExecuteQuery(userCred mcclient.TokenCredentia startTime := time.Now() queryResult, err := query.executeQuery(&evalContext, timeRange) if err != nil { - return nil, err + return nil, errors.Wrap(err, "query.executeQuery") } log.Debugf("query metrics from influxdb elapsed: %s", time.Since(startTime)) return queryResult, nil @@ -126,6 +125,10 @@ func (query *MetricQueryCondition) ExecuteQuery(userCred mcclient.TokenCredentia ) go func() { startTime := time.Now() + defer func() { + qRegionCh <- true + log.Debugf("get resources from region elapsed: %s", time.Since(startTime)) + }() s := auth.GetSession(ctx, userCred, "") ress, regionErr = firstCond.getOnecloudResources(s, false) @@ -142,13 +145,15 @@ func (query *MetricQueryCondition) ExecuteQuery(userCred mcclient.TokenCredentia tmpRes := res resMap[id] = tmpRes } - log.Debugf("get resources from region elapsed: %s", time.Since(startTime)) - qRegionCh <- true }() <-qInfluxdbCh <-qRegionCh + if err != nil || regionErr != nil { + return nil, errors.Errorf("tsdb error: %v, region error: %v", err, regionErr) + } + startTime := time.Now() //for _, serie := range queryResult.series { // isLatestOfSerie, resource := firstCond.serieIsLatestResource(resMap, serie) @@ -213,9 +218,9 @@ func (query *MetricQueryCondition) noCheckSeries(skipCheckSeries bool) bool { } func (c *MetricQueryCondition) executeQuery(context *alerting.EvalContext, timeRange *tsdb.TimeRange) (*queryResult, error) { - ds, err := models.DataSourceManager.GetSource(c.QueryCons[0].Query.DataSourceId) + ds, err := datasource.GetDefaultSource("") if err != nil { - return nil, errors.Wrapf(err, "Cound not find datasource %v", c.QueryCons[0].Query.DataSourceId) + return nil, errors.Wrapf(err, "Can't find default datasource") } req := c.getRequestQuery(ds, timeRange, context.IsDebug) @@ -226,13 +231,12 @@ func (c *MetricQueryCondition) executeQuery(context *alerting.EvalContext, timeR setContextLog(context, req) } - resp, err := c.HandleRequest(context.Ctx, ds.ToTSDBDataSource(""), req) + resp, err := c.HandleRequest(context.Ctx, ds, req) if err != nil { if err == gocontext.DeadlineExceeded { return nil, errors.Error("Alert execution exceeded the timeout") } - log.Errorf("metricQuery HandleRequest error:%v", err) - return nil, err + return nil, errors.Wrap(err, "metricQuery HandleRequest") } for _, v := range resp.Results { if v.Error != nil { @@ -300,13 +304,15 @@ func setContextLog(context *alerting.EvalContext, req *tsdb.TsdbQuery) { }) } -func (query *MetricQueryCondition) getRequestQuery(ds *models.SDataSource, timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery { +func (query *MetricQueryCondition) getRequestQuery(ds *tsdb.DataSource, timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery { querys := make([]*tsdb.Query, 0) for _, qc := range query.QueryCons { + nDs := *ds + nDs.Database = qc.Query.Model.Database querys = append(querys, &tsdb.Query{ RefId: strconv.FormatInt(int64(qc.Index), 10), MetricQuery: qc.Query.Model, - DataSource: *ds.ToTSDBDataSource(qc.Query.Model.Database), + DataSource: nDs, }) } req := &tsdb.TsdbQuery{ diff --git a/pkg/monitor/alerting/conditions/query.go b/pkg/monitor/alerting/conditions/query.go index f18ef3dd71..12af8ae29f 100644 --- a/pkg/monitor/alerting/conditions/query.go +++ b/pkg/monitor/alerting/conditions/query.go @@ -31,6 +31,7 @@ import ( mc_mds "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" "yunion.io/x/onecloud/pkg/monitor/alerting" + "yunion.io/x/onecloud/pkg/monitor/datasource" "yunion.io/x/onecloud/pkg/monitor/models" "yunion.io/x/onecloud/pkg/monitor/options" "yunion.io/x/onecloud/pkg/monitor/tsdb" @@ -59,10 +60,9 @@ type QueryCondition struct { // AlertQuery contains information about what datasource a query // should be send to and the query object. type AlertQuery struct { - Model monitor.MetricQuery - DataSourceId string - From string - To string + Model monitor.MetricQuery + From string + To string } type FormatCond struct { @@ -369,12 +369,7 @@ type queryResult struct { } func (c *QueryCondition) executeQuery(evalCtx *alerting.EvalContext, timeRange *tsdb.TimeRange) (*queryResult, error) { - ds, err := models.DataSourceManager.GetSource(c.Query.DataSourceId) - if err != nil { - return nil, errors.Wrapf(err, "Cound not find datasource %v", c.Query.DataSourceId) - } - - req := c.getRequestForAlertRule(ds, timeRange, evalCtx.IsDebug) + req := c.getRequestForAlertRule(timeRange, evalCtx.IsDebug) result := make(tsdb.TimeSeriesSlice, 0) metas := make([]tsdb.QueryResultMeta, 0) @@ -412,7 +407,12 @@ func (c *QueryCondition) executeQuery(evalCtx *alerting.EvalContext, timeRange * }) } - resp, err := c.HandleRequest(evalCtx.Ctx, ds.ToTSDBDataSource(c.Query.Model.Database), req) + ds, err := datasource.GetDefaultSource(c.Query.Model.Database) + if err != nil { + return nil, errors.Wrap(err, "GetDefaultDataSource") + } + + resp, err := c.HandleRequest(evalCtx.Ctx, ds, req) if err != nil { if err == context.DeadlineExceeded { return nil, errors.Error("Alert execution exceeded the timeout") @@ -452,14 +452,15 @@ func (c *QueryCondition) executeQuery(evalCtx *alerting.EvalContext, timeRange * }, nil } -func (c *QueryCondition) getRequestForAlertRule(ds *models.SDataSource, timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery { +func (c *QueryCondition) getRequestForAlertRule(timeRange *tsdb.TimeRange, debug bool) *tsdb.TsdbQuery { + ds, _ := datasource.GetDefaultSource(c.Query.Model.Database) req := &tsdb.TsdbQuery{ TimeRange: timeRange, Queries: []*tsdb.Query{ { RefId: "A", MetricQuery: c.Query.Model, - DataSource: *ds.ToTSDBDataSource(c.Query.Model.Database), + DataSource: *ds, }, }, Debug: debug, @@ -485,7 +486,6 @@ func newQueryCondition(model *monitor.AlertCondition, index int) (*QueryConditio return nil, errors.Wrapf(err, "to value %q", cond.Query.To) } - cond.Query.DataSourceId = q.DataSourceId //reducer := model.Reducer //cond.Reducer = newSimpleReducer(reducer.Type) reducer, err := NewAlertReducer(&model.Reducer) diff --git a/pkg/monitor/alerting/engine.go b/pkg/monitor/alerting/engine.go index 3b33e1a832..d4d0342952 100644 --- a/pkg/monitor/alerting/engine.go +++ b/pkg/monitor/alerting/engine.go @@ -46,7 +46,7 @@ func init() { registry.RegisterService(&AlertEngine{}) } -// IsDisabled returns true if the alerting service is disable for this instance. +// IsDisabled returns true if the alerting service is disabled for this instance. func (e *AlertEngine) IsDisabled() bool { // TODO: read from config options return false diff --git a/pkg/monitor/controller/balancer/balancer.go b/pkg/monitor/controller/balancer/balancer.go index b99c849910..f22c9deb17 100644 --- a/pkg/monitor/controller/balancer/balancer.go +++ b/pkg/monitor/controller/balancer/balancer.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules/compute" compute_options "yunion.io/x/onecloud/pkg/mcclient/options/compute" "yunion.io/x/onecloud/pkg/monitor/alerting" + "yunion.io/x/onecloud/pkg/monitor/datasource" "yunion.io/x/onecloud/pkg/monitor/models" "yunion.io/x/onecloud/pkg/monitor/tsdb" ) @@ -145,11 +146,10 @@ func NewRules(_ *alerting.EvalContext, m *monitor.EvalMatch, alert *models.SMigr allHosts = append(allHosts, oh) } - dsObj, err := models.DataSourceManager.GetDefaultSource() + ds, err := datasource.GetDefaultSource("telegraf") if err != nil { return nil, errors.Wrapf(err, "Get default DataSource") } - ds := dsObj.ToTSDBDataSource("") // find guests to filtered by source setting of source host alerted cds, err := findGuestsOfHost(drv, srcHost, ds, msettings) if err != nil { diff --git a/pkg/monitor/controller/balancer/utils.go b/pkg/monitor/controller/balancer/utils.go index 42fa5c1c5d..a7718da28b 100644 --- a/pkg/monitor/controller/balancer/utils.go +++ b/pkg/monitor/controller/balancer/utils.go @@ -23,7 +23,6 @@ import ( api "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" "yunion.io/x/onecloud/pkg/monitor/tsdb" - "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb" ) type HostMetric struct { @@ -78,14 +77,12 @@ func InfluxdbQuery( q.Where().IN(idKey, ids) q.GroupBy().TAG(idKey).FILL_NULL() qCtx := q.ToTsdbQuery() - endpoint, err := influxdb.NewInfluxdbExecutor(nil) + + resp, err := tsdb.HandleRequest(context.Background(), ds, qCtx) if err != nil { - return nil, errors.Wrap(err, "influxdb.NewInfluxdbExecutor") - } - resp, err := endpoint.Query(context.TODO(), ds, qCtx) - if err != nil { - return nil, errors.Wrap(err, "influxdb endpoint Query") + return nil, errors.Wrap(err, "TSDB endpoint Query") } + ss := resp.Results[""].Series ms := make([]*HostMetric, len(ss)) for i, s := range ss { diff --git a/pkg/monitor/datasource/datasource.go b/pkg/monitor/datasource/datasource.go new file mode 100644 index 0000000000..fda82d59e8 --- /dev/null +++ b/pkg/monitor/datasource/datasource.go @@ -0,0 +1,156 @@ +package datasource + +import ( + "context" + "time" + + "golang.org/x/sync/errgroup" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/stringutils" + "yunion.io/x/pkg/util/wait" + + commontsdb "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/monitor/options" + "yunion.io/x/onecloud/pkg/monitor/registry" + "yunion.io/x/onecloud/pkg/monitor/tsdb" +) + +func init() { + registry.RegisterService(GetManager()) +} + +var ( + dsMan *dataSourceManager +) + +type DataSourceManager interface { + registry.Service + + GetDefaultSource(db string) *tsdb.DataSource +} + +func GetManager() DataSourceManager { + if dsMan == nil { + dsMan = newDataSourceManager() + } + return dsMan +} + +func GetDefaultSource(db string) (*tsdb.DataSource, error) { + src := GetManager().GetDefaultSource(db) + if src == nil { + return nil, errors.Errorf("default data source is not initialization") + } + return src, nil +} + +func GetDefaultQueryEndpoint() (tsdb.TsdbQueryEndpoint, error) { + ds, err := GetDefaultSource("") + if err != nil { + return nil, errors.Wrap(err, "get default datasource") + } + return tsdb.GetTsdbQueryEndpointFor(ds) +} + +func newDataSourceManager() *dataSourceManager { + return &dataSourceManager{ + defaultDS: nil, + } +} + +type dataSourceManager struct { + defaultDS *tsdb.DataSource +} + +func (m *dataSourceManager) Init() error { + return m.initDefaultDataSource(context.Background()) +} +func (m *dataSourceManager) Run(ctx context.Context) error { + errgrp, ctx := errgroup.WithContext(ctx) + errgrp.Go(func() error { + m.run(ctx) + return nil + }) + return errgrp.Wait() +} + +func (m *dataSourceManager) GetDefaultSource(db string) *tsdb.DataSource { + ds := m.defaultDS + return &tsdb.DataSource{ + Id: ds.Id, + Name: ds.Name, + Type: ds.Type, + Url: ds.Url, + User: ds.User, + Password: ds.Password, + Database: db, + BasicAuth: ds.BasicAuth, + BasicAuthUser: ds.BasicAuthUser, + BasicAuthPassword: ds.BasicAuthPassword, + TimeInterval: ds.TimeInterval, + Updated: ds.Updated, + } +} + +func (man *dataSourceManager) shouldChangeDataSource(dsType string, url string) bool { + if man.defaultDS == nil { + return true + } + if man.defaultDS.Type != dsType { + return true + } + if man.defaultDS.Url != url { + return true + } + return false +} + +func (man *dataSourceManager) setDataSource(dsType string, url string) { + if !man.shouldChangeDataSource(dsType, url) { + return + } + + log.Infof("set TSDB data source %q: %q", dsType, url) + man.defaultDS = &tsdb.DataSource{ + Id: stringutils.UUID4(), + Name: dsType, + Type: dsType, + Url: url, + Updated: time.Now(), + } +} + +func (man *dataSourceManager) initDefaultDataSource(ctx context.Context) error { + region := options.Options.Region + epType := options.Options.SessionEndpointType + s := auth.GetAdminSession(ctx, region) + //dsSvc := options.Options.MonitorDataSource + if s == nil { + return errors.Errorf("get empty public session for region %s", region) + } + source, err := commontsdb.GetDefaultServiceSource(s, epType) + if err != nil { + return errors.Wrap(err, "get default TSDB source") + } + dsSvc := source.Type + if err := tsdb.IsValidDataSource(dsSvc); err != nil { + return errors.Wrapf(err, "invalid type %q", dsSvc) + } + url, err := s.GetServiceURL(dsSvc, epType) + if err != nil { + return errors.Errorf("get %q public url: %v", dsSvc, err) + } + man.setDataSource(dsSvc, url) + return nil +} + +func (m *dataSourceManager) run(ctx context.Context) { + wait.Forever(func() { + if err := m.initDefaultDataSource(ctx); err != nil { + log.Errorf("init default source") + } + }, 30*time.Second) +} diff --git a/pkg/monitor/datasource/doc.go b/pkg/monitor/datasource/doc.go new file mode 100644 index 0000000000..6514f6dca9 --- /dev/null +++ b/pkg/monitor/datasource/doc.go @@ -0,0 +1 @@ +package datasource // import "yunion.io/x/onecloud/pkg/monitor/datasource" diff --git a/pkg/monitor/models/alert.go b/pkg/monitor/models/alert.go index 4be831f506..c603b1ee54 100644 --- a/pkg/monitor/models/alert.go +++ b/pkg/monitor/models/alert.go @@ -201,16 +201,16 @@ type AlertRuleTag struct { Value string } -func setAlertDefaultSetting(setting *monitor.AlertSetting, dsId string) *monitor.AlertSetting { +func setAlertDefaultSetting(setting *monitor.AlertSetting) *monitor.AlertSetting { for idx, cond := range setting.Conditions { - cond = setAlertDefaultCondition(cond, dsId) + cond = setAlertDefaultCondition(cond) setting.Conditions[idx] = cond } return setting } -func setAlertDefaultCreateData(data monitor.AlertCreateInput, dsId string) monitor.AlertCreateInput { - setting := setAlertDefaultSetting(&data.Settings, dsId) +func setAlertDefaultCreateData(data monitor.AlertCreateInput) monitor.AlertCreateInput { + setting := setAlertDefaultSetting(&data.Settings) data.Settings = *setting enable := true if data.Enabled == nil { @@ -219,7 +219,7 @@ func setAlertDefaultCreateData(data monitor.AlertCreateInput, dsId string) monit return data } -func setAlertDefaultCondition(cond monitor.AlertCondition, dsId string) monitor.AlertCondition { +func setAlertDefaultCondition(cond monitor.AlertCondition) monitor.AlertCondition { if cond.Type == "" { cond.Type = "query" } @@ -229,18 +229,11 @@ func setAlertDefaultCondition(cond monitor.AlertCondition, dsId string) monitor. if cond.Operator == "" { cond.Operator = "and" } - if cond.Query.DataSourceId == "" { - cond.Query.DataSourceId = dsId - } return cond } func (man *SAlertManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, _ jsonutils.JSONObject, data monitor.AlertCreateInput) (monitor.AlertCreateInput, error) { - ds, err := DataSourceManager.GetDefaultSource() - if err != nil { - return data, errors.Wrap(err, "get default data source") - } - data = setAlertDefaultCreateData(data, ds.GetId()) + data = setAlertDefaultCreateData(data) if err := validators.ValidateAlertCreateInput(data); err != nil { return data, err } diff --git a/pkg/monitor/models/datasource.go b/pkg/monitor/models/datasource.go index 8ff58f81f6..da24f3c7ab 100644 --- a/pkg/monitor/models/datasource.go +++ b/pkg/monitor/models/datasource.go @@ -17,7 +17,6 @@ package models import ( "bytes" "context" - "database/sql" "fmt" "math" "regexp" @@ -31,7 +30,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/util/wait" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/monitor" @@ -40,10 +38,8 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/monitor/datasource" merrors "yunion.io/x/onecloud/pkg/monitor/errors" - "yunion.io/x/onecloud/pkg/monitor/options" - "yunion.io/x/onecloud/pkg/monitor/registry" - "yunion.io/x/onecloud/pkg/monitor/tsdb" "yunion.io/x/onecloud/pkg/monitor/validators" "yunion.io/x/onecloud/pkg/util/influxdb" "yunion.io/x/onecloud/pkg/util/stringutils2" @@ -72,80 +68,12 @@ func init() { ), } DataSourceManager.SetVirtualObject(DataSourceManager) - registry.RegisterService(DataSourceManager) } type SDataSourceManager struct { db.SStandaloneResourceBaseManager } -func (_ *SDataSourceManager) IsDisabled() bool { - return false -} - -func (_ *SDataSourceManager) Init() error { - return nil -} - -func (man *SDataSourceManager) Run(ctx context.Context) error { - errgrp, ctx := errgroup.WithContext(ctx) - errgrp.Go(func() error { return man.initDefaultDataSource(ctx) }) - return errgrp.Wait() -} - -func (man *SDataSourceManager) initDefaultDataSource(ctx context.Context) error { - region := options.Options.Region - epType := options.Options.SessionEndpointType - initF := func() { - ds, err := man.GetDefaultSource() - if err != nil && err != ErrDataSourceDefaultNotFound { - log.Errorf("Get default datasource: %v", err) - return - } - s := auth.GetAdminSession(ctx, region) - if s == nil { - log.Errorf("get empty public session for region %s", region) - return - } - url, err := s.GetServiceURL("influxdb", epType) - if err != nil { - log.Errorf("get influxdb public url: %v", err) - return - } - if ds != nil { - if _, err := db.Update(ds, func() error { - ds.Url = url - return nil - }); err != nil { - log.Errorf("update datasource url error: %v", err) - } - return - } - ds = &SDataSource{ - Type: monitor.DataSourceTypeInfluxdb, - Url: url, - } - ds.Name = DefaultDataSource - if err := man.TableSpec().Insert(ctx, ds); err != nil { - log.Errorf("insert default influxdb: %v", err) - } - } - wait.Forever(initF, 30*time.Second) - return nil -} - -func (man *SDataSourceManager) GetDefaultSource() (*SDataSource, error) { - obj, err := man.FetchByName(nil, DefaultDataSource) - if err != nil { - if err == sql.ErrNoRows { - return nil, ErrDataSourceDefaultNotFound - } else { - return nil, err - } - } - return obj.(*SDataSource), nil -} - type SDataSource struct { db.SStandaloneResourceBase @@ -171,29 +99,9 @@ func (m *SDataSourceManager) GetSource(id string) (*SDataSource, error) { return ret.(*SDataSource), nil } -func (ds *SDataSource) ToTSDBDataSource(db string) *tsdb.DataSource { - if db == "" { - db = ds.Database - } - return &tsdb.DataSource{ - Id: ds.GetId(), - Name: ds.GetName(), - Type: ds.Type, - Url: ds.Url, - User: ds.User, - Password: ds.Password, - Database: db, - Updated: ds.UpdatedAt, - /*BasicAuth: ds.BasicAuth, - BasicAuthUser: ds.BasicAuthUser, - BasicAuthPassword: ds.BasicAuthPassword, - TimeInterval: ds.TimeInterval,*/ - } -} - func (self *SDataSourceManager) GetDatabases() (jsonutils.JSONObject, error) { ret := jsonutils.NewDict() - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return jsonutils.JSONNull, errors.Wrap(err, "s.GetDefaultSource") } @@ -225,7 +133,7 @@ func (self *SDataSourceManager) getMeasurementQueryInfluxdb(query jsonutils.JSON if database == "" { return rtnMeasurements, merrors.NewArgIsEmptyErr("database") } - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return rtnMeasurements, errors.Wrap(err, "s.GetDefaultSource") } @@ -262,15 +170,14 @@ func (self *SDataSourceManager) getMeasurementQueryInfluxdb(query jsonutils.JSON return } -func (self *SDataSourceManager) GetMeasurementsWithDescriptionInfos(query jsonutils.JSONObject, measurementFilter, - tagFilter string) (jsonutils.JSONObject, error) { +func (self *SDataSourceManager) GetMeasurementsWithDescriptionInfos(query jsonutils.JSONObject, measurementFilter string, tagFilter *monitor.MetricQueryTag) (jsonutils.JSONObject, error) { ret := jsonutils.NewDict() rtnMeasurements := make([]monitor.InfluxMeasurement, 0) measurements, err := MetricMeasurementManager.getInfluxdbMeasurements() if err != nil { - return jsonutils.JSONNull, err + return jsonutils.JSONNull, errors.Wrap(err, "getInfluxdbMeasurements") } - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return jsonutils.JSONNull, errors.Wrap(err, "s.GetDefaultSource") } @@ -318,7 +225,7 @@ func (self *SDataSourceManager) GetMeasurementsWithOutTimeFilter(query jsonutils if database == "" { return jsonutils.JSONNull, httperrors.NewInputParameterError("not support database") } - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return jsonutils.JSONNull, errors.Wrap(err, "s.GetDefaultSource") } @@ -419,13 +326,12 @@ type influxdbQueryChan struct { } func (self *SDataSourceManager) filterMeasurementsByTime(db influxdb.SInfluxdb, - measurements []monitor.InfluxMeasurement, query jsonutils.JSONObject, tagFilter string) ([]monitor.InfluxMeasurement, - error) { + measurements []monitor.InfluxMeasurement, query jsonutils.JSONObject, tagFilter *monitor.MetricQueryTag) ([]monitor.InfluxMeasurement, error) { timeF, err := self.getFromAndToFromParam(query) if err != nil { return nil, err } - filterMeasurements, err := self.getFilterMeasurementsAsyn(timeF.From, timeF.To, measurements, db, tagFilter) + filterMeasurements, err := self.getFilterMeasurementsAsync(timeF.From, timeF.To, measurements, db, tagFilter) if err != nil { return nil, err } @@ -460,8 +366,8 @@ func (self *SDataSourceManager) getFromAndToFromParam(query jsonutils.JSONObject return timeF, nil } -func (self *SDataSourceManager) getFilterMeasurementsAsyn(from, to string, - measurements []monitor.InfluxMeasurement, db influxdb.SInfluxdb, tagFilter string) ([]monitor.InfluxMeasurement, error) { +func (self *SDataSourceManager) getFilterMeasurementsAsync(from, to string, + measurements []monitor.InfluxMeasurement, db influxdb.SInfluxdb, tagFilter *monitor.MetricQueryTag) ([]monitor.InfluxMeasurement, error) { filterMeasurements := make([]monitor.InfluxMeasurement, 0) queryChan := new(influxdbQueryChan) queryChan.queryRtnChan = make(chan monitor.InfluxMeasurement, len(measurements)) @@ -493,59 +399,17 @@ func (self *SDataSourceManager) getFilterMeasurementsAsyn(from, to string, return filterMeasurements, err } -func (self *SDataSourceManager) getFilterMeasurement(queryChan *influxdbQueryChan, from, to string, - measurement monitor.InfluxMeasurement, db influxdb.SInfluxdb, tagFilter string) error { - rtnMeasurement := new(monitor.InfluxMeasurement) - var buffer bytes.Buffer - buffer.WriteString(fmt.Sprintf(`SELECT last(*) FROM %s WHERE %s`, measurement.Measurement, - renderTimeFilter(from, to))) - if len(tagFilter) != 0 { - buffer.WriteString(" AND ") - buffer.WriteString(fmt.Sprintf(" %s", tagFilter)) - } - log.Errorln(buffer.String()) - (&db).SetDatabase(measurement.Database) - rtn, err := db.Query(buffer.String()) +func (self *SDataSourceManager) getFilterMeasurement(queryChan *influxdbQueryChan, from, to string, measurement monitor.InfluxMeasurement, db influxdb.SInfluxdb, tagFilter *monitor.MetricQueryTag) error { + dds, _ := datasource.GetDefaultSource("") + ep, err := datasource.GetDefaultQueryEndpoint() if err != nil { - return errors.Wrap(err, "getFilterMeasurement error") + return errors.Wrap(err, "GetDefaultQueryEndpoint") } - - rtnFields := make([]string, 0) - if len(rtn) != 0 && len(rtn[0]) != 0 { - for rtnIndex, _ := range rtn { - for serieIndex, _ := range rtn[rtnIndex] { - meanFieldArr := rtn[rtnIndex][serieIndex].Columns - for i, _ := range meanFieldArr { - if !strings.Contains(meanFieldArr[i], "last") { - continue - } - - containsVal := false - for _, value := range rtn[rtnIndex][serieIndex].Values { - if value[i] == nil { - continue - } - _, err := value[i].Float() - if err != nil { - continue - } - containsVal = true - break - } - if containsVal { - rtnFields = append(rtnFields, strings.Replace(meanFieldArr[i], "last_", "", 1)) - } - } - } - } + retMs, err := ep.FilterMeasurement(context.Background(), dds, from, to, &measurement, tagFilter) + if err != nil { + return errors.Wrap(err, "Get endpoint filtered measurement") } - rtnMeasurement.FieldKey = rtnFields - if len(rtnMeasurement.FieldKey) != 0 { - rtnMeasurement.Measurement = measurement.Measurement - rtnMeasurement.Database = measurement.Database - rtnMeasurement.ResType = measurement.ResType - } - queryChan.queryRtnChan <- *rtnMeasurement + queryChan.queryRtnChan <- *retMs return nil } @@ -565,7 +429,7 @@ func renderTimeFilter(from, to string) string { } -func (self *SDataSourceManager) GetMetricMeasurement(userCred mcclient.TokenCredential, query jsonutils.JSONObject, tagFilter string) (jsonutils.JSONObject, error) { +func (self *SDataSourceManager) GetMetricMeasurement(userCred mcclient.TokenCredential, query jsonutils.JSONObject, tagFilter *monitor.MetricQueryTag) (jsonutils.JSONObject, error) { database, _ := query.GetString("database") if database == "" { return jsonutils.JSONNull, merrors.NewArgIsEmptyErr("database") @@ -582,7 +446,7 @@ func (self *SDataSourceManager) GetMetricMeasurement(userCred mcclient.TokenCred if len(from) == 0 { return jsonutils.JSONNull, merrors.NewArgIsEmptyErr("from") } - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return jsonutils.JSONNull, errors.Wrap(err, "s.GetDefaultSource") } @@ -639,7 +503,7 @@ func (self *SDataSourceManager) GetMetricMeasurement(userCred mcclient.TokenCred // if err != nil { // return jsonutils.JSONNull, errors.Wrap(err, "getTagValue error") //** } - if err := getTagValues(userCred, output, timeF, dataSource.GetId(), tagFilter, skipCheckSeries); err != nil { + if err := getTagValues(userCred, output, timeF, tagFilter, skipCheckSeries); err != nil { return jsonutils.JSONNull, errors.Wrap(err, "getTagValues error") } @@ -755,7 +619,7 @@ func (self *SDataSourceManager) AddSubscription(subscription InfluxdbSubscriptio jsonutils.NewString(subscription.Rc).String(), strings.ReplaceAll(jsonutils.NewString(subscription.Url).String(), "\"", "'"), ) - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return errors.Wrap(err, "s.GetDefaultSource") } @@ -781,7 +645,7 @@ func (self *SDataSourceManager) DropSubscription(subscription InfluxdbSubscripti jsonutils.NewString(subscription.DataBase).String(), jsonutils.NewString(subscription.Rc).String(), ) - dataSource, err := self.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource("") if err != nil { return errors.Wrap(err, "s.GetDefaultSource") } @@ -828,7 +692,7 @@ func getAttributesOnMeasurement(database, tp string, output *monitor.InfluxMeasu return nil } -func getTagValues(userCred mcclient.TokenCredential, output *monitor.InfluxMeasurement, timeF timeFilter, dsId string, tagFilter string, skipCheckSeries bool) error { +func getTagValues(userCred mcclient.TokenCredential, output *monitor.InfluxMeasurement, timeF timeFilter, tagFilter *monitor.MetricQueryTag, skipCheckSeries bool) error { mq := monitor.MetricQuery{ Database: output.Database, Measurement: output.Measurement, @@ -850,22 +714,20 @@ func getTagValues(userCred mcclient.TokenCredential, output *monitor.InfluxMeasu }, }, } - if tagFilter != "" { - parts := strings.Split(tagFilter, " ") + if tagFilter != nil { mq.Tags = []monitor.MetricQueryTag{ { - Key: parts[0], - Operator: parts[1], - Value: parts[2], + Key: tagFilter.Key, + Operator: tagFilter.Operator, + Value: tagFilter.Value, }, } } aq := &monitor.AlertQuery{ - Model: mq, - From: timeF.From, - To: timeF.To, - DataSourceId: dsId, + Model: mq, + From: timeF.From, + To: timeF.To, } q := monitor.MetricInputQuery{ diff --git a/pkg/monitor/models/meteralert.go b/pkg/monitor/models/meteralert.go index d65e270fbe..2f0a1cce10 100644 --- a/pkg/monitor/models/meteralert.go +++ b/pkg/monitor/models/meteralert.go @@ -51,7 +51,7 @@ type IMeterAlertDriver interface { GetType() string GetName() string GetFor() time.Duration - ToAlertCreateInput(input monitor.MeterAlertCreateInput, dsId string, allAccountIds []string, level string) monitor.AlertCreateInput + ToAlertCreateInput(input monitor.MeterAlertCreateInput, allAccountIds []string, level string) monitor.AlertCreateInput } type SMeterAlertManager struct { @@ -172,12 +172,7 @@ func (man *SMeterAlertManager) ValidateCreateData( return nil, merrors.NewArgIsEmptyErr("recipients") } - ds, err := DataSourceManager.GetDefaultSource() - if err != nil { - return nil, err - } - - alertInput := drv.ToAlertCreateInput(data, ds.GetId(), allAccountIds, data.Level) + alertInput := drv.ToAlertCreateInput(data, allAccountIds, data.Level) alertInput, err = AlertManager.ValidateCreateData(ctx, userCred, ownerId, query, alertInput) if err != nil { return nil, err @@ -203,7 +198,6 @@ func (_ *sMeterDailyFee) GetFor() time.Duration { func (f *sMeterDailyFee) ToAlertCreateInput( input monitor.MeterAlertCreateInput, - dsId string, allAccountIds []string, level string, ) monitor.AlertCreateInput { @@ -213,7 +207,6 @@ func (f *sMeterDailyFee) ToAlertCreateInput( Level: level, Frequency: int64(freq / time.Second), Settings: GetMeterAlertSetting(input, - dsId, "account_daily_resfee", "meter_db", allAccountIds, "sumDate"), } @@ -236,7 +229,6 @@ func (_ *sMeterMonthFee) GetFor() time.Duration { func (f *sMeterMonthFee) ToAlertCreateInput( input monitor.MeterAlertCreateInput, - dsId string, allAccountIds []string, level string, ) monitor.AlertCreateInput { @@ -246,7 +238,6 @@ func (f *sMeterMonthFee) ToAlertCreateInput( Level: level, Frequency: int64(freq / time.Second), Settings: GetMeterAlertSetting(input, - dsId, "account_month_resfee", "meter_db", allAccountIds, "sumMonth"), } @@ -255,7 +246,6 @@ func (f *sMeterMonthFee) ToAlertCreateInput( func GetMeterAlertSetting( input monitor.MeterAlertCreateInput, - dsId string, measurement string, db string, accountIds []string, @@ -268,10 +258,9 @@ func GetMeterAlertSetting( Type: "query", Operator: "and", Query: monitor.AlertQuery{ - Model: q, - From: input.Period, - To: "now", - DataSourceId: dsId, + Model: q, + From: input.Period, + To: "now", }, Reducer: reducer, Evaluator: eval, @@ -605,11 +594,6 @@ func (alert *SMeterAlert) ValidateUpdateData( details.Comparator = *input.Comparator } - ds, err := DataSourceManager.GetDefaultSource() - if err != nil { - return input, errors.Wrap(err, "get default data source") - } - // hack: update notification here if err := alert.UpdateNotification(AlertNotificationUsedByMeterAlert, input.Channel, input.Recipients); err != nil { return input, errors.Wrap(err, "update notification") @@ -621,7 +605,7 @@ func (alert *SMeterAlert) ValidateUpdateData( return input, err } } - tmpS := alert.getUpdateSetting(details, ds.GetId(), allAccountIds) + tmpS := alert.getUpdateSetting(details, allAccountIds) input.Settings = &tmpS input.V1AlertUpdateInput, err = alert.SV1Alert.ValidateUpdateData(ctx, userCred, query, input.V1AlertUpdateInput) @@ -632,7 +616,7 @@ func (alert *SMeterAlert) ValidateUpdateData( return input, nil } -func (alert *SMeterAlert) getUpdateSetting(details monitor.MeterAlertDetails, dsId string, accountIds []string) monitor.AlertSetting { +func (alert *SMeterAlert) getUpdateSetting(details monitor.MeterAlertDetails, accountIds []string) monitor.AlertSetting { drv := MeterAlertManager.GetDriver(alert.getType()) input := monitor.MeterAlertCreateInput{ ResourceAlertV1CreateInput: monitor.ResourceAlertV1CreateInput{ @@ -649,7 +633,7 @@ func (alert *SMeterAlert) getUpdateSetting(details monitor.MeterAlertDetails, ds AccountId: details.AccountId, } input.Level = details.Level - out := drv.ToAlertCreateInput(input, dsId, accountIds, details.Level) + out := drv.ToAlertCreateInput(input, accountIds, details.Level) return out.Settings } diff --git a/pkg/monitor/models/metric.go b/pkg/monitor/models/metric.go index f72f956d78..968c9e6ace 100644 --- a/pkg/monitor/models/metric.go +++ b/pkg/monitor/models/metric.go @@ -374,8 +374,7 @@ func (manager *SMetricMeasurementManager) getMeasurement(query *sqlchemy.SQuery) return measurements, nil } -func (manager *SMetricMeasurementManager) getInfluxdbMeasurements() (influxdbMeasurements []monitor. - InfluxMeasurement, err error) { +func (manager *SMetricMeasurementManager) getInfluxdbMeasurements() (influxdbMeasurements []monitor.InfluxMeasurement, err error) { metric, err := manager.getMeasurement(manager.Query()) if err != nil { return diff --git a/pkg/monitor/models/nodealert.go b/pkg/monitor/models/nodealert.go index d282fde0e1..cc773cc66b 100644 --- a/pkg/monitor/models/nodealert.go +++ b/pkg/monitor/models/nodealert.go @@ -828,11 +828,7 @@ func (alert *SNodeAlert) GetCommonAlertUpdateData(ctx context.Context, userCred input.Name = name } - ds, err := DataSourceManager.GetDefaultSource() - if err != nil { - return nil, errors.Wrap(err, "get default data source") - } - tmpS := alert.getUpdateInput(name, details, ds.GetId()) + tmpS := alert.getUpdateInput(name, details) input.Settings = &tmpS.Settings uData, err := alert.SCommonAlert.ValidateUpdateData(ctx, userCred, nil, tmpS.JSON(tmpS)) @@ -855,7 +851,6 @@ func (alert *SNodeAlert) ValidateUpdateData( func (alert *SNodeAlert) getUpdateInput( name string, details monitor.NodeAlertDetails, - dsId string, ) monitor.CommonAlertCreateInput { data := monitor.NodeAlertCreateInput{ ResourceAlertV1CreateInput: monitor.ResourceAlertV1CreateInput{ @@ -872,7 +867,7 @@ func (alert *SNodeAlert) getUpdateInput( } data.Level = details.Level out := data.ToCommonAlertCreateInput(name, details.Field, details.Measurement, details.DB) - out.Settings = *setAlertDefaultSetting(&out.Settings, dsId) + out.Settings = *setAlertDefaultSetting(&out.Settings) return out } diff --git a/pkg/monitor/models/unifiedmonitor.go b/pkg/monitor/models/unifiedmonitor.go index f6be8eb9ce..43679cb3e1 100644 --- a/pkg/monitor/models/unifiedmonitor.go +++ b/pkg/monitor/models/unifiedmonitor.go @@ -34,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/monitor/datasource" merrors "yunion.io/x/onecloud/pkg/monitor/errors" mq "yunion.io/x/onecloud/pkg/monitor/metricquery" "yunion.io/x/onecloud/pkg/monitor/options" @@ -79,25 +80,24 @@ func (self *SUnifiedMonitorManager) GetPropertyMeasurements(ctx context.Context, filter, err := getTagFilterByRequestQuery(ctx, userCred, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getTagFilterByRequestQuery") } return DataSourceManager.GetMeasurementsWithDescriptionInfos(query, "", filter) } -func getTagFilterByRequestQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (filter string, err error) { +func getTagFilterByRequestQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*monitor.MetricQueryTag, error) { scope, _ := query.GetString("scope") - filter, err = filterByScope(ctx, userCred, scope, query) - return + return filterByScope(ctx, userCred, scope, query) } -func filterByScope(ctx context.Context, userCred mcclient.TokenCredential, scope string, data jsonutils.JSONObject) (string, error) { +func filterByScope(ctx context.Context, userCred mcclient.TokenCredential, scope string, data jsonutils.JSONObject) (*monitor.MetricQueryTag, error) { domainId := jsonutils.GetAnyString(data, db.DomainFetchKeys) projectId := jsonutils.GetAnyString(data, db.ProjectFetchKeys) if projectId != "" { project, err := db.DefaultProjectFetcher(ctx, projectId, domainId) if err != nil { - return "", err + return nil, errors.Wrap(err, "db.DefaultProjectFetcher") } projectId = project.GetProjectId() domainId = project.GetProjectDomainId() @@ -105,14 +105,14 @@ func filterByScope(ctx context.Context, userCred mcclient.TokenCredential, scope if domainId != "" { domain, err := db.DefaultDomainFetcher(ctx, domainId) if err != nil { - return "", err + return nil, errors.Wrap(err, "db.DefaultDomainFetcher") } domainId = domain.GetProjectDomainId() domain.GetProjectId() } switch scope { case "system": - return "", nil + return nil, nil case "domain": if domainId == "" { domainId = userCred.GetProjectDomainId() @@ -126,9 +126,9 @@ func filterByScope(ctx context.Context, userCred mcclient.TokenCredential, scope } } -func getTenantIdStr(role string, userCred mcclient.TokenCredential) (string, error) { +func getTenantIdStr(role string, userCred mcclient.TokenCredential) (*monitor.MetricQueryTag, error) { if role == "admin" { - return "", nil + return nil, nil } if role == "domainadmin" { domainId := userCred.GetDomainId() @@ -138,10 +138,10 @@ func getTenantIdStr(role string, userCred mcclient.TokenCredential) (string, err tenantId := userCred.GetProjectId() return getProjectIdFilterByProject(tenantId) } - return "", errors.ErrNotFound + return nil, errors.Wrapf(errors.ErrNotFound, "not supported role %q", role) } -func getProjectIdsFilterByDomain(domainId string) (string, error) { +func getProjectIdsFilterByDomain(domainId string) (*monitor.MetricQueryTag, error) { //s := auth.GetAdminSession(context.Background(), "", "") //params := jsonutils.Marshal(map[string]string{"domain_id": domainId}) //tenants, err := modules.Projects.List(s, params) @@ -160,11 +160,21 @@ func getProjectIdsFilterByDomain(domainId string) (string, error) { //} //buffer.WriteString(" )") //return buffer.String(), nil - return fmt.Sprintf(`%s =~ /%s/`, "domain_id", domainId), nil + return &monitor.MetricQueryTag{ + Key: "domain_id", + Operator: "=~", + Value: fmt.Sprintf("/%s/", domainId), + }, nil + //return fmt.Sprintf(`%s =~ /%s/`, "domain_id", domainId), nil } -func getProjectIdFilterByProject(projectId string) (string, error) { - return fmt.Sprintf(`%s =~ /%s/`, "tenant_id", projectId), nil +func getProjectIdFilterByProject(projectId string) (*monitor.MetricQueryTag, error) { + //return fmt.Sprintf(`%s =~ /%s/`, "tenant_id", projectId), nil + return &monitor.MetricQueryTag{ + Key: "tenant_id", + Operator: "=~", + Value: fmt.Sprintf("/%s/", projectId), + }, nil } func (self *SUnifiedMonitorManager) GetPropertyMetricMeasurement(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { @@ -257,7 +267,6 @@ func (self *SUnifiedMonitorManager) PerformQuery(ctx context.Context, userCred m } } - setSerieRowName(&rtn.Series, groupByTag) fillSerieTags(&rtn.Series) return jsonutils.Marshal(rtn), nil } @@ -317,11 +326,11 @@ func doQuery(userCred mcclient.TokenCredential, query monitor.MetricInputQuery) factory := mq.GetQueryFactories()["metricquery"] metricQ, err := factory(conditions) if err != nil { - return nil, err + return nil, errors.Wrap(err, "factory") } metrics, err := metricQ.ExecuteQuery(userCred, query.SkipCheckSeries) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ExecuteQuery") } // drop metas contains raw_query if !query.ShowMeta { @@ -349,7 +358,6 @@ func (self *SUnifiedMonitorManager) ValidateInputQuery(query *monitor.AlertQuery func setDefaultValue(query *monitor.AlertQuery, inputQuery *monitor.MetricInputQuery, scope string, ownerId mcclient.IIdentityProvider) { - setDataSourceId(query) query.From = inputQuery.From query.To = inputQuery.To query.Model.Interval = inputQuery.Interval @@ -449,11 +457,6 @@ func setDefaultValue(query *monitor.AlertQuery, inputQuery *monitor.MetricInputQ } } -func setDataSourceId(query *monitor.AlertQuery) { - datasource, _ := DataSourceManager.GetDefaultSource() - query.DataSourceId = datasource.Id -} - func checkQueryGroupBy(query *monitor.AlertQuery, inputQuery *monitor.MetricInputQuery) { if len(query.Model.GroupBy) != 0 { return @@ -476,50 +479,6 @@ func checkQueryGroupBy(query *monitor.AlertQuery, inputQuery *monitor.MetricInpu }) } -func setSerieRowName(series *tsdb.TimeSeriesSlice, groupTag []string) { - // Add rowname，The front end displays the curve according to rowname - var index, unknownIndex = 1, 1 - for i, serie := range *series { - // setRowName by groupTag - if len(groupTag) != 0 { - for key, val := range serie.Tags { - if strings.Contains(strings.Join(groupTag, ","), key) { - serie.RawName = fmt.Sprintf("%s", val) - (*series)[i] = serie - break - } - } - continue - } - measurement := strings.Split(serie.Name, ".")[0] - // sep measurement set RowName by spe param - measurements, _ := MetricMeasurementManager.getMeasurementByName(measurement) - if len(measurements) != 0 { - if key, ok := monitor.MEASUREMENT_TAG_KEYWORD[measurements[0].ResType]; ok { - serie.RawName = fmt.Sprintf("%d: %s", index, serie.Tags[key]) - (*series)[i] = serie - index++ - continue - } - } - // other condition set RowName - for key, val := range serie.Tags { - if strings.Contains(key, "id") { - serie.RawName = fmt.Sprintf("%d: %s", index, val) - (*series)[i] = serie - index++ - break - } - } - if serie.RawName == "" { - serie.RawName = fmt.Sprintf("unknown-%d", unknownIndex) - (*series)[i] = serie - unknownIndex++ - } - } - -} - func fillSerieTags(series *tsdb.TimeSeriesSlice) { for i, serie := range *series { for _, tag := range []string{"brand", "platform", "hypervisor"} { @@ -577,7 +536,7 @@ func (self *SUnifiedMonitorManager) GetPropertySimpleQuery(ctx context.Context, conditions = append(conditions, fmt.Sprintf("time <= '%s'", et)) sqlstr += strings.Join(conditions, " and ") sqlstr += " limit 2000" - dataSource, err := DataSourceManager.GetDefaultSource() + dataSource, err := datasource.GetDefaultSource(input.Database) if err != nil { return nil, errors.Wrap(err, "s.GetDefaultSource") } diff --git a/pkg/monitor/service/service.go b/pkg/monitor/service/service.go index 2f99f31b13..cbbdee0cfc 100644 --- a/pkg/monitor/service/service.go +++ b/pkg/monitor/service/service.go @@ -44,6 +44,7 @@ import ( "yunion.io/x/onecloud/pkg/monitor/registry" "yunion.io/x/onecloud/pkg/monitor/subscriptionmodel" _ "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb" + _ "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/victoriametrics" "yunion.io/x/onecloud/pkg/monitor/worker" ) @@ -85,16 +86,12 @@ func StartService() { cron.Start() defer cron.Stop() - subscriptionmodel.SubscriptionManager.AddSubscription() - models.CommonAlertManager.SetSubscriptionManager(subscriptionmodel.SubscriptionManager) - worker, err := worker.NewWorker(opts) if err != nil { log.Fatalf("new worker failed: %v", err) } go worker.Start(app.GetContext(), app, "") - //common_app.ServeForever(app, baseOpts) InitInfluxDBSubscriptionHandlers(app, baseOpts) // start migration recover routine @@ -115,10 +112,13 @@ func startServices() { log.Infof("Initializing " + svc.Name) if err := svc.Instance.Init(); err != nil { - log.Fatalf("Service %s init failed", svc.Name) + log.Fatalf("Service %s init failed: %v", svc.Name, err) } } + subscriptionmodel.SubscriptionManager.AddSubscription() + models.CommonAlertManager.SetSubscriptionManager(subscriptionmodel.SubscriptionManager) + childRoutines, ctx := errgroup.WithContext(context.Background()) // Start background services for _, svc := range services { diff --git a/pkg/monitor/subscriptionmodel/subcription.go b/pkg/monitor/subscriptionmodel/subcription.go index bac103ccaa..55d5c8ecd6 100644 --- a/pkg/monitor/subscriptionmodel/subcription.go +++ b/pkg/monitor/subscriptionmodel/subcription.go @@ -81,7 +81,7 @@ func (self *SSubscriptionManager) AddSubscription() { return } log.Infof("drop success") - err = models.DataSourceManager.AddSubscription(sub) + /*err = models.DataSourceManager.AddSubscription(sub) if err != nil { log.Errorln("add subscription err:", err) return @@ -90,7 +90,7 @@ func (self *SSubscriptionManager) AddSubscription() { if err := self.LoadSystemAlerts(); err != nil { log.Errorf("load system alerts error: %v", err) return - } + }*/ } func (self *SSubscriptionManager) LoadSystemAlerts() error { diff --git a/pkg/monitor/tsdb/driver/influxdb/helper_test.go b/pkg/monitor/tsdb/driver/influxdb/helper_test.go new file mode 100644 index 0000000000..94d388a1dd --- /dev/null +++ b/pkg/monitor/tsdb/driver/influxdb/helper_test.go @@ -0,0 +1,43 @@ +package influxdb + +import ( + "testing" + + "github.com/smartystreets/goconvey/convey" + + monitor2 "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" +) + +func TestAlertQuery(t *testing.T) { + convey.Convey("Alert query test", t, func() { + parser := new(InfluxdbQueryParser) + q := monitor.NewAlertQuery("telegraf", "diskio").From("5m").To("now") + q.Selects().Select("await").MEAN() + q.Where().Equal("hostname", "host1").Equal("provider", "kvm") + q.GroupBy().TAG("*").FILL_NULL() + qCtx := q.ToTsdbQuery() + influxdbQ, err := parser.Parse(qCtx.Queries[0], nil) + convey.So(err, convey.ShouldBeNil) + rawQuery, err := influxdbQ.Build(qCtx) + convey.So(err, convey.ShouldBeNil) + convey.So(rawQuery, convey.ShouldEqual, `SELECT mean("await") FROM "diskio" WHERE ("hostname" = 'host1' AND "provider" = 'kvm') AND time > now() - 5m GROUP BY * fill(null)`) + }) + + convey.Convey("Alert last query", t, func() { + q := monitor.NewAlertQuery("telegraf", "diskio").From("5m").To("now") + q.Selects().Select("*").LAST() + q.Where().AddTag(&monitor2.MetricQueryTag{ + Operator: "=~", + Key: "project_id", + Value: "/xxx/", + }) + tq := q.ToTsdbQuery() + parser := new(InfluxdbQueryParser) + influxQ, err := parser.Parse(tq.Queries[0], nil) + convey.So(err, convey.ShouldBeNil) + rawQ, err := influxQ.Build(tq) + convey.So(err, convey.ShouldBeNil) + convey.So(rawQ, convey.ShouldEqual, `SELECT last(*) FROM "diskio" WHERE "project_id" =~ /xxx/ AND time > now() - 5m`) + }) +} diff --git a/pkg/monitor/tsdb/driver/influxdb/influxdb.go b/pkg/monitor/tsdb/driver/influxdb/influxdb.go index d94e7b6a12..e945fc5dc4 100644 --- a/pkg/monitor/tsdb/driver/influxdb/influxdb.go +++ b/pkg/monitor/tsdb/driver/influxdb/influxdb.go @@ -29,6 +29,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis/monitor" + mod "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" "yunion.io/x/onecloud/pkg/monitor/tsdb" ) @@ -37,7 +39,7 @@ const ( ) func init() { - tsdb.RegisterTsdbQueryEndpoint("influxdb", NewInfluxdbExecutor) + tsdb.RegisterTsdbQueryEndpoint(monitor.DataSourceTypeInfluxdb, NewInfluxdbExecutor) } type InfluxdbExecutor struct { @@ -52,8 +54,7 @@ func NewInfluxdbExecutor(datasource *tsdb.DataSource) (tsdb.TsdbQueryEndpoint, e }, nil } -func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { - result := &tsdb.Response{} +func (e *InfluxdbExecutor) GetRawQuery(dsInfo *tsdb.DataSource, tsdbQuery *tsdb.TsdbQuery) (string, []*Query, error) { querys := make([]*tsdb.Query, len(tsdbQuery.Queries)+1) influxQ := make([]*Query, 0) copy(querys, tsdbQuery.Queries) @@ -61,14 +62,14 @@ func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, t var rawQuery string for i := 0; i < len(querys); i++ { query, err := e.getQuery(dsInfo, querys, tsdbQuery) - influxQ = append(influxQ, query) if err != nil { - return nil, err + return "", nil, errors.Wrap(err, "getQuery") } + influxQ = append(influxQ, query) rawQuery, err := query.Build(tsdbQuery) if err != nil { - return nil, err + return "", nil, errors.Wrap(err, "query.Build") } buffer.WriteString(rawQuery + ";") if len(querys) > 0 { @@ -87,6 +88,14 @@ func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, t //if err != nil { // return nil, err //} + return rawQuery, influxQ, nil +} + +func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) { + rawQuery, influxQ, err := e.GetRawQuery(dsInfo, tsdbQuery) + if err != nil { + return nil, errors.Wrap(err, "GetRawQuery") + } log.Debugf("sql: %s", rawQuery) db := dsInfo.Database @@ -127,7 +136,9 @@ func (e *InfluxdbExecutor) Query(ctx context.Context, dsInfo *tsdb.DataSource, t return nil, response.Err } - result.Results = make(map[string]*tsdb.QueryResult) + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } for i, query := range tsdbQuery.Queries { ret := e.ResponseParser.Parse(&response, influxQ[i]) ret.Meta = tsdb.QueryResultMeta{ @@ -193,3 +204,45 @@ func (e *InfluxdbExecutor) createRequest(dsInfo *tsdb.DataSource, query string) log.Debugf("Influxdb raw query: %q from db %s, curl: %s", query, dsInfo.Database, curlCmd) return req, nil } + +func (e *InfluxdbExecutor) FilterMeasurement( + ctx context.Context, + ds *tsdb.DataSource, + from, to string, + ms *monitor.InfluxMeasurement, + tagFilter *monitor.MetricQueryTag, +) (*monitor.InfluxMeasurement, error) { + retMs := new(monitor.InfluxMeasurement) + q := mod.NewAlertQuery(ms.Database, ms.Measurement).From(from).To(to) + q.Selects().Select("*").LAST() + if tagFilter != nil { + q.Where().AddTag(tagFilter) + } + tq := q.ToTsdbQuery() + resp, err := e.Query(ctx, ds, tq) + if err != nil { + return nil, errors.Wrap(err, "influxdb.Query") + } + ss := resp.Results[""].Series + //log.Infof("=====get ss: %s", jsonutils.Marshal(ss).PrettyString()) + + // parse fields + retFields := make([]string, 0) + for _, s := range ss { + cols := s.Columns + for _, col := range cols { + if !strings.Contains(col, "last") { + continue + } + retFields = append(retFields, strings.Replace(col, "last_", "", 1)) + } + } + retMs.FieldKey = retFields + if len(retMs.FieldKey) != 0 { + retMs.Measurement = ms.Measurement + retMs.Database = ms.Database + retMs.ResType = ms.ResType + } + + return retMs, nil +} diff --git a/pkg/monitor/tsdb/driver/influxdb/response_parser.go b/pkg/monitor/tsdb/driver/influxdb/response_parser.go index d5a68dec56..2c2b556adb 100644 --- a/pkg/monitor/tsdb/driver/influxdb/response_parser.go +++ b/pkg/monitor/tsdb/driver/influxdb/response_parser.go @@ -75,7 +75,7 @@ func (rp *ResponseParser) transformRows(rows []Row, queryResult *tsdb.QueryResul func (rp *ResponseParser) transformRowsV2(rows []Row, queryResult *tsdb.QueryResult, query *Query) tsdb.TimeSeriesSlice { var result tsdb.TimeSeriesSlice - for _, row := range rows { + for idx, row := range rows { col := "" columns := make([]string, 0) for _, column := range row.Columns { @@ -104,17 +104,24 @@ func (rp *ResponseParser) transformRowsV2(rows []Row, queryResult *tsdb.QueryRes val_ := strings.ReplaceAll(val, "+", " ") tags[key] = val_ } - result = append(result, &tsdb.TimeSeries{ - Name: rp.formatSerieName(row, col, query), - Columns: columns, - Points: points, - Tags: tags, - }) + name := rp.formatSerieName(row, col, query) + ts := tsdb.NewTimeSeries(name, formatRawName(idx, name, query, tags), columns, points, tags) + result = append(result, ts) } return result } +func formatRawName(idx int, name string, query *Query, tags map[string]string) string { + groupByTags := []string{} + for _, group := range query.GroupBy { + if group.Type == "tag" { + groupByTags = append(groupByTags, group.Params[0]) + } + } + return tsdb.FormatRawName(idx, name, groupByTags, tags) +} + func (rp *ResponseParser) transformRowToTable(row Row, table *tsdb.Table) *tsdb.Table { for _, col := range row.Columns { table.Columns = append(table.Columns, tsdb.TableColumn{ diff --git a/pkg/monitor/tsdb/driver/victoriametrics/client.go b/pkg/monitor/tsdb/driver/victoriametrics/client.go new file mode 100644 index 0000000000..1abd9c1a3d --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/client.go @@ -0,0 +1,153 @@ +package victoriametrics + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "time" + + "github.com/influxdata/influxql" + "golang.org/x/net/context/ctxhttp" + "moul.io/http2curl/v2" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +const ( + ErrVMInvalidResponse = errors.Error("VictoriaMetrics invalid response") +) + +type TimeRange struct { + Start int64 + End int64 +} + +func NewTimeRange(start, end int64) *TimeRange { + return &TimeRange{ + Start: start, + End: end, + } +} + +func NewTimeRangeByInfluxTimeRange(tr *influxql.TimeRange) *TimeRange { + // format should be: https://docs.victoriametrics.com/#timestamp-formats + nTr := &TimeRange{} + if !tr.MinTime().IsZero() { + nTr.Start = tr.MinTime().Unix() + } + if !tr.MaxTime().IsZero() { + nTr.End = tr.MaxTime().Unix() + } + return nTr +} + +type Client interface { + QueryRange(ctx context.Context, httpCli *http.Client, query string, step time.Duration, timeRange *TimeRange, disableCache bool) (*Response, error) +} + +type client struct { + endpoint string + endpointURL url.URL +} + +func (c *client) getAPIURL(reqPath string) string { + apiPrefix := "/api/v1" + reqPath = fmt.Sprintf("%s/%s", apiPrefix, reqPath) + reqURL := c.endpointURL + reqURL.Path = path.Join(reqURL.Path, reqPath) + return reqURL.String() +} + +// ResponseDataResultValue likes: [ 1652169600, "1" ] +type ResponseDataResultValue []interface{} + +type ResponseDataResult struct { + Metric map[string]string `json:"metric"` + Values []ResponseDataResultValue `json:"values"` +} + +type ResponseData struct { + ResultType string `json:"resultType"` + Result []ResponseDataResult `json:"result"` +} + +type ResponseStats struct { + // SeriesFetched is like integer type: {seriesFetched: "2"} + SeriesFetched string `json:"seriesFetched"` +} + +type Response struct { + Status string `json:"status"` + Data ResponseData + Stats ResponseStats +} + +// QueryRange implements Client. +func (c *client) QueryRange(ctx context.Context, httpCli *http.Client, query string, step time.Duration, tr *TimeRange, disableCache bool) (*Response, error) { + req, err := c.createQueryRangeReq(query, step, tr, disableCache) + if err != nil { + return nil, errors.Wrap(err, "get request") + } + + resp, err := ctxhttp.Do(ctx, httpCli, req) + if err != nil { + return nil, errors.Wrap(err, "Do request") + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, errors.Wrapf(ErrVMInvalidResponse, "status code: %d", resp.StatusCode) + } + + var response Response + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + if err := dec.Decode(&response); err != nil { + return nil, errors.Wrap(err, "decode json response") + } + return &response, nil +} + +func (c *client) createQueryRangeReq(query string, step time.Duration, tr *TimeRange, disableCache bool) (*http.Request, error) { + reqURL := c.getAPIURL("/query_range") + req, err := http.NewRequest(http.MethodGet, reqURL, nil) + if err != nil { + return nil, errors.Wrapf(err, "new HTTP request of: %s", reqURL) + } + req.Header.Set("User-Agent", "Cloudpods Monitor Service") + params := req.URL.Query() + params.Set("query", query) + if step != 0 { + params.Set("step", step.String()) + } + if tr != nil { + if tr.Start != 0 { + params.Set("start", fmt.Sprintf("%d", tr.Start)) + } + if tr.End != 0 { + params.Set("end", fmt.Sprintf("%d", tr.End)) + } + } + if disableCache { + params.Set("nocache", "1") + } + req.URL.RawQuery = params.Encode() + curlCmd, _ := http2curl.GetCurlCommand(req) + log.Infof("VictoriaMetrics curl cmd: %s", curlCmd) + return req, nil +} + +func NewClient(endpoint string) (Client, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, errors.Wrapf(err, "invalid url: %q", endpoint) + } + cli := &client{ + endpoint: endpoint, + endpointURL: *u, + } + return cli, nil +} diff --git a/pkg/monitor/tsdb/driver/victoriametrics/client_test.go b/pkg/monitor/tsdb/driver/victoriametrics/client_test.go new file mode 100644 index 0000000000..5fa3210cb1 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/client_test.go @@ -0,0 +1,20 @@ +package victoriametrics + +func newTestClient(ep string) (Client, error) { + return NewClient(ep) +} + +/*func Test_client_QueryRange(t *testing.T) { + cli, err := newTestClient("http://192.168.222.171:34795/") + if err != nil { + t.Fatalf("newTestClient") + } + resp, err := cli.QueryRange(context.Background(), http.DefaultClient, `avg by(host_id) (avg_over_time(cpu_usage_active{res_type="host"}))`, time.Second*50, &TimeRange{ + Start: 1698724482.213, + End: 1698746082.213, + }, true) + if err != nil { + t.Fatalf("query err: %v", err) + } + log.Infof("get resp: %s", jsonutils.Marshal(resp).PrettyString()) +}*/ diff --git a/pkg/monitor/tsdb/driver/victoriametrics/doc.go b/pkg/monitor/tsdb/driver/victoriametrics/doc.go new file mode 100644 index 0000000000..48d12494d0 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/doc.go @@ -0,0 +1 @@ +package victoriametrics // import "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/victoriametrics" diff --git a/pkg/monitor/tsdb/driver/victoriametrics/result_parser.go b/pkg/monitor/tsdb/driver/victoriametrics/result_parser.go new file mode 100644 index 0000000000..95a9eb1db0 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/result_parser.go @@ -0,0 +1,154 @@ +package victoriametrics + +import ( + "fmt" + "reflect" + "sort" + "strings" + + "github.com/influxdata/promql/v2/pkg/labels" + "github.com/zexi/influxql-to-promql/converter/translator" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/monitor/tsdb" +) + +func newMapId(input map[string]string, ignoreKeys ...string) string { + keys := make([]string, 0) + ignoreKS := sets.NewString(ignoreKeys...) + for key := range input { + if ignoreKS.Has(key) { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + pairs := make([]string, len(keys)) + for i, key := range keys { + pair := fmt.Sprintf("%s->%s", key, input[key]) + pairs[i] = pair + } + return strings.Join(pairs, ",") +} + +type points struct { + id string + columns []string + values []ResponseDataResultValue + tags map[string]string +} + +func (p *points) add(op *points) error { + if len(op.columns) != 1 { + return errors.Errorf("input points' columns are %#v, which length isn't equal 1", op.columns) + } + p.columns = append(p.columns, op.columns[0]) + // merge values + for i, val := range p.values { + oVal := op.values[i] + valTime := val[0] + oValTime := oVal[0] + if valTime != oValTime { + return errors.Errorf("value time %v != other value time %v", valTime, oValTime) + } + if len(oVal) != 2 { + return errors.Errorf("input values' are %#v, which length isn't equal 2", oVal) + } + val = append(val, oVal[1]) + } + return nil +} + +func (p *points) isEqual(op *points) bool { + if p.id != op.id { + return false + } + return reflect.DeepEqual(p.columns, op.columns) && reflect.DeepEqual(p.tags, op.tags) && reflect.DeepEqual(p.values, op.values) +} + +func newPointsByResult(result ResponseDataResult) (*points, error) { + tags := result.Metric + column, ok := tags[translator.UNION_RESULT_NAME] + if !ok { + return nil, errors.Errorf("result tags %#v don't contain key %s", tags, translator.UNION_RESULT_NAME) + } + for _, ignoreKey := range []string{ + translator.UNION_RESULT_NAME, + labels.MetricName, + } { + delete(tags, ignoreKey) + } + values := result.Values + id := newMapId(tags) + return &points{ + id: id, + columns: []string{column}, + values: values, + tags: tags, + }, nil +} + +func newPointsByResults(results []ResponseDataResult) ([]*points, error) { + uniq := make(map[string]*points, 0) + ret := make([]*points, 0) + + for _, result := range results { + p, err := newPointsByResult(result) + if err != nil { + return nil, errors.Wrapf(err, "new points by result: %#v", result) + } + if ep, ok := uniq[p.id]; ok { + if err := ep.add(p); err != nil { + return nil, errors.Wrapf(err, "add point %#v", p) + } + } else { + uniq[p.id] = p + ret = append(ret, p) + } + } + return ret, nil +} + +func transPointsToSeries(points []*points, query *tsdb.Query) tsdb.TimeSeriesSlice { + var result tsdb.TimeSeriesSlice + for _, point := range points { + result = append(result, transPointToSeries(point, query)...) + } + return result +} + +func transValuesToTSDBPoints(vals []ResponseDataResultValue) tsdb.TimeSeriesPoints { + var points tsdb.TimeSeriesPoints + for _, val := range vals { + point, err := parseTimepoint(val) + if err != nil { + log.Errorf("parseTimepoint: %#v", val) + } else { + points = append(points, point) + } + } + return points +} + +func reviseTags(tags map[string]string) map[string]string { + ret := make(map[string]string) + for key, val := range tags { + val_ := strings.ReplaceAll(val, "+", " ") + ret[key] = val_ + } + return ret +} + +func transPointToSeries(p *points, query *tsdb.Query) tsdb.TimeSeriesSlice { + var result tsdb.TimeSeriesSlice + + points := transValuesToTSDBPoints(p.values) + tags := reviseTags(p.tags) + metricName := strings.Join(p.columns, ",") + ts := tsdb.NewTimeSeries(metricName, formatRawName(0, metricName, query, tags), append(p.columns, "time"), points, tags) + result = append(result, ts) + return result +} diff --git a/pkg/monitor/tsdb/driver/victoriametrics/result_parser_test.go b/pkg/monitor/tsdb/driver/victoriametrics/result_parser_test.go new file mode 100644 index 0000000000..662bbe5221 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/result_parser_test.go @@ -0,0 +1,123 @@ +package victoriametrics + +import ( + "testing" +) + +func Test_newPointsByResults(t *testing.T) { + tests := []struct { + name string + results []ResponseDataResult + want []*points + wantErr bool + }{ + { + name: "merge ok", + results: []ResponseDataResult{ + { + Metric: map[string]string{ + "__name__": "disk_free", + "__union_result__": "disk_free", + "fstype": "fuse.s3fs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "9223372036854775807"}, + []interface{}{1699416000, "9223372036854775807"}, + }, + }, + { + Metric: map[string]string{ + "__name__": "disk_free", + "__union_result__": "disk_free", + "fstype": "xfs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "277637316608"}, + []interface{}{1699416000, "277599338496"}, + }, + }, + { + Metric: map[string]string{ + "__name__": "disk_total", + "__union_result__": "disk_total", + "fstype": "fuse.s3fs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "9223372036854775807"}, + []interface{}{1699416000, "9223372036854775807"}, + }, + }, + { + Metric: map[string]string{ + "__name__": "disk_total", + "__union_result__": "disk_total", + "fstype": "xfs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "536608768000"}, + []interface{}{1699416000, "536608768000"}, + }, + }, + { + Metric: map[string]string{ + "__name__": "disk_used", + "__union_result__": "disk_used", + "fstype": "fuse.s3fs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "0"}, + []interface{}{1699416000, "0"}, + }, + }, + { + Metric: map[string]string{ + "__name__": "disk_used", + "__union_result__": "disk_used", + "fstype": "xfs", + }, + Values: []ResponseDataResultValue{ + []interface{}{1699414400, "258971451392"}, + []interface{}{1699416000, "259009429504"}, + }, + }, + }, + want: []*points{ + { + id: "fstype->fuse.s3fs", + columns: []string{"disk_free", "disk_total", "disk_used"}, + values: []ResponseDataResultValue{ + []interface{}{1699414400, "9223372036854775807", "9223372036854775807", "0"}, + []interface{}{1699416000, "9223372036854775807", "9223372036854775807", "0"}, + }, + tags: map[string]string{"fstype": "fuse.s3fs"}, + }, + { + id: "fstype->xfs", + columns: []string{"disk_free", "disk_total", "disk_used"}, + values: []ResponseDataResultValue{ + []interface{}{1699414400, "277637316608", "536608768000", "258971451392"}, + []interface{}{1699416000, "277599338496", "536608768000", "259009429504"}, + }, + tags: map[string]string{"fstype": "xfs"}, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newPointsByResults(tt.results) + if (err != nil) != tt.wantErr { + t.Errorf("newPointsByResults() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + gotP := got[i] + wantP := tt.want[i] + if gotP.isEqual(wantP) { + t.Errorf("newPointsByResults() got = %v, want %v", gotP, wantP) + } + } + }) + } +} diff --git a/pkg/monitor/tsdb/driver/victoriametrics/vm.go b/pkg/monitor/tsdb/driver/victoriametrics/vm.go new file mode 100644 index 0000000000..7ca7ca1d90 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/vm.go @@ -0,0 +1,264 @@ +package victoriametrics + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/influxdata/influxql" + "github.com/influxdata/promql/v2/pkg/labels" + "github.com/zexi/influxql-to-promql/converter" + "github.com/zexi/influxql-to-promql/converter/translator" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/apis/monitor" + mod "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" + "yunion.io/x/onecloud/pkg/monitor/tsdb" + "yunion.io/x/onecloud/pkg/monitor/tsdb/driver/influxdb" +) + +func init() { + tsdb.RegisterTsdbQueryEndpoint(monitor.DataSourceTypeVictoriaMetrics, NewVMAdapter) +} + +type vmAdapter struct { + datasource *tsdb.DataSource + influxdbExecutor *influxdb.InfluxdbExecutor +} + +func NewVMAdapter(datasource *tsdb.DataSource) (tsdb.TsdbQueryEndpoint, error) { + ie, _ := influxdb.NewInfluxdbExecutor(nil) + return &vmAdapter{ + datasource: datasource, + influxdbExecutor: ie.(*influxdb.InfluxdbExecutor), + }, nil +} + +// Query implements tsdb.TsdbQueryEndpoint. +func (vm *vmAdapter) Query(ctx context.Context, ds *tsdb.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) { + rawQuery, influxQs, err := vm.influxdbExecutor.GetRawQuery(ds, query) + if err != nil { + return nil, errors.Wrapf(err, "get influxdb raw query: %#v", influxQs) + } + + return queryByRaw(ctx, ds, rawQuery, query) +} + +func queryByRaw(ctx context.Context, ds *tsdb.DataSource, rawQuery string, query *tsdb.TsdbQuery) (*tsdb.Response, error) { + promQL, tr, err := convertInfluxQL(rawQuery) + if err != nil { + return nil, errors.Wrap(err, "convert influxQL to promQL") + } + log.Infof("influxQL: %s, promQL: %s", rawQuery, promQL) + resp, err := queryRange(ctx, ds, tr, promQL) + if err != nil { + return nil, errors.Wrapf(err, "query VM range by: %s", promQL) + } + + //log.Infof("get vm resp: %s", jsonutils.Marshal(resp).PrettyString()) + + tsdbRet, err := convertVMResponse(rawQuery, query, resp) + if err != nil { + return nil, errors.Wrapf(err, "convert to tsdb.Response") + } + + return tsdbRet, nil +} + +func queryRange(ctx context.Context, ds *tsdb.DataSource, tr *influxql.TimeRange, promQL string) (*Response, error) { + cli, err := NewClient(ds.Url) + if err != nil { + return nil, errors.Wrap(err, "New VM client") + } + httpCli, err := ds.GetHttpClient() + if err != nil { + return nil, errors.Wrap(err, "GetHttpClient of data source") + } + vmTr := NewTimeRangeByInfluxTimeRange(tr) + return cli.QueryRange(ctx, httpCli, promQL, 0, vmTr, false) +} + +func convertInfluxQL(influxQL string) (string, *influxql.TimeRange, error) { + promQL, timeRange, err := converter.TranslateWithTimeRange(influxQL) + if err != nil { + return "", nil, errors.Wrapf(err, "TranslateWithTimeRange: %s", influxQL) + } + return promQL, timeRange, nil +} + +func convertVMResponse(rawQuery string, tsdbQuery *tsdb.TsdbQuery, resp *Response) (*tsdb.Response, error) { + result := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + for _, query := range tsdbQuery.Queries { + ret, err := translateResponse(resp, query) + if err != nil { + return nil, errors.Wrap(err, "translate response") + } + ret.Meta = tsdb.QueryResultMeta{ + RawQuery: rawQuery, + } + result.Results[query.RefId] = ret + } + + return result, nil +} + +func translateResponse(resp *Response, query *tsdb.Query) (*tsdb.QueryResult, error) { + queryRes := tsdb.NewQueryResult() + + isUnionResult := false + results := resp.Data.Result + if len(results) > 0 { + _, isUnionResult = results[0].Metric[translator.UNION_RESULT_NAME] + } + if !isUnionResult { + for _, result := range results { + ss := transformSeries(result, query) + queryRes.Series = append(queryRes.Series, ss...) + } + } else { + // process union multiple fields response + points, err := newPointsByResults(results) + if err != nil { + return nil, errors.Wrap(err, "process multi fields") + } + ss := transPointsToSeries(points, query) + queryRes.Series = ss + } + + return queryRes, nil +} + +// Check VictoriaMetrics response at: https://docs.victoriametrics.com/keyConcepts.html#range-query +func transformSeries(vmResult ResponseDataResult, query *tsdb.Query) tsdb.TimeSeriesSlice { + var result tsdb.TimeSeriesSlice + metric := vmResult.Metric + + points := transValuesToTSDBPoints(vmResult.Values) + + tags := reviseTags(metric) + + aliasName := "" + if len(query.Selects) > 0 { + lastSel := query.Selects[len(query.Selects)-1] + lastSelPart := lastSel[len(lastSel)-1] + if lastSelPart.Type == "alias" && len(lastSelPart.Params) > 0 { + aliasName = lastSelPart.Params[0] + } + } + metricName := metric[labels.MetricName] + if metricName == "" { + metricName = "value" + } + if aliasName != "" { + metricName = aliasName + } + ts := tsdb.NewTimeSeries(metricName, formatRawName(0, metricName, query, tags), []string{metricName, "time"}, points, tags) + result = append(result, ts) + return result +} + +func formatRawName(idx int, name string, query *tsdb.Query, tags map[string]string) string { + groupByTags := []string{} + if query != nil { + for _, group := range query.GroupBy { + if group.Type == "tag" { + groupByTags = append(groupByTags, group.Params[0]) + } + } + } + return tsdb.FormatRawName(idx, name, groupByTags, tags) +} + +func parseTimepoint(val ResponseDataResultValue) (tsdb.TimePoint, error) { + timepoint := make(tsdb.TimePoint, 0) + // parse timestamp + timestampNumber, _ := val[0].(json.Number) + timestamp, err := timestampNumber.Float64() + if err != nil { + return tsdb.TimePoint{}, errors.Wrapf(err, "parse timestampNumber") + } + // to influxdb timestamp format, millisecond ? + timestamp *= 1000 + + // parse value + for i := 1; i < len(val); i++ { + valStr := val[i] + pVal := parsePointValue(valStr) + timepoint = append(timepoint, pVal) + timepoint = append(timepoint, timestamp) + } + return timepoint, nil +} + +func parsePointValue(value interface{}) interface{} { + number, ok := value.(json.Number) + if !ok { + // try parse string + valStr, ok := value.(string) + if ok { + valF, err := strconv.ParseFloat(valStr, 64) + if err == nil { + return &valF + } + return value + } + return value + } + + fvalue, err := number.Float64() + if err == nil { + return &fvalue + } + + ivalue, err := number.Int64() + if err == nil { + ret := float64(ivalue) + return &ret + } + return number.String() +} + +func (vm *vmAdapter) FilterMeasurement(ctx context.Context, ds *tsdb.DataSource, from, to string, ms *monitor.InfluxMeasurement, tagFilter *monitor.MetricQueryTag) (*monitor.InfluxMeasurement, error) { + retMs := new(monitor.InfluxMeasurement) + q := mod.NewAlertQuery(ms.Database, ms.Measurement).From(from).To(to) + q.Selects().Select("*").LAST() + if tagFilter != nil { + q.Where().AddTag(tagFilter) + } + q.GroupBy().TAG(labels.MetricName) + tq := q.ToTsdbQuery() + resp, err := vm.Query(ctx, ds, tq) + if err != nil { + return nil, errors.Wrap(err, "VictoriaMetrics.Query") + } + ss := resp.Results[""].Series + //log.Infof("=====get ss: %s", jsonutils.Marshal(ss).PrettyString()) + + // parse fields + retFields := sets.NewString() + msPrefix := fmt.Sprintf("%s_", ms.Measurement) + for _, s := range ss { + cols := s.Columns + for _, col := range cols { + if !strings.HasPrefix(col, msPrefix) { + continue + } + field := strings.TrimPrefix(col, msPrefix) + retFields.Insert(field) + } + } + retMs.FieldKey = retFields.List() + if len(retMs.FieldKey) != 0 { + retMs.Measurement = ms.Measurement + retMs.Database = ms.Database + retMs.ResType = ms.ResType + } + return retMs, nil +} diff --git a/pkg/monitor/tsdb/driver/victoriametrics/vm_test.go b/pkg/monitor/tsdb/driver/victoriametrics/vm_test.go new file mode 100644 index 0000000000..f6a07c3634 --- /dev/null +++ b/pkg/monitor/tsdb/driver/victoriametrics/vm_test.go @@ -0,0 +1,70 @@ +package victoriametrics + +/*import ( + "context" + "testing" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/monitor/tsdb" +) + +func Test_queryByRaw(t *testing.T) { + ds := &tsdb.DataSource{ + Id: "vm", + Name: "vm", + Type: "victoricmetrics", + Url: "http://192.168.222.171:34795/", + } + q := &tsdb.TsdbQuery{ + TimeRange: tsdb.NewTimeRange("48h", "now"), + Queries: []*tsdb.Query{ + { + RefId: "A", + MetricQuery: monitor.MetricQuery{ + Database: "telegraf", + Measurement: "cpu", + Selects: []monitor.MetricQuerySelect{ + { + { + Type: "field", + Params: []string{"usage_active"}, + }, + { + Type: "mean", + }, + }, + }, + Tags: []monitor.MetricQueryTag{ + { + Key: "res_type", + Operator: "=", + Value: "host", + }, + }, + GroupBy: []monitor.MetricQueryPart{ + //{ + // Type: "tag", + // Params: []string{"host_id"}, + //}, + { + Type: "tag", + Params: []string{"*"}, + }, + }, + }, + }, + }, + Debug: false, + } + ep, _ := NewVMAdapter(ds) + + resp, err := ep.Query(context.Background(), ds, q) + if err != nil { + t.Fatalf("queryByRaw error: %v", err) + } + log.Infof("resp: %s", jsonutils.Marshal(resp).PrettyString()) +} +*/ diff --git a/pkg/monitor/tsdb/models.go b/pkg/monitor/tsdb/models.go index c953c71613..a728b2c610 100644 --- a/pkg/monitor/tsdb/models.go +++ b/pkg/monitor/tsdb/models.go @@ -15,7 +15,12 @@ package tsdb import ( + "fmt" + "sort" "strconv" + "strings" + + "yunion.io/x/pkg/util/sets" api "yunion.io/x/onecloud/pkg/apis/monitor" ) @@ -54,6 +59,7 @@ type QueryResult struct { } type TimeSeries struct { + // RawName is used to frontend displaying the curve name RawName string `json:"raw_name"` Columns []string `json:"columns"` Name string `json:"name"` @@ -61,6 +67,22 @@ type TimeSeries struct { Tags map[string]string `json:"tags,omitempty"` } +func NewTimeSeries( + name string, + rawName string, + columns []string, + points TimeSeriesPoints, + tags map[string]string, +) *TimeSeries { + return &TimeSeries{ + RawName: rawName, + Columns: columns, + Name: name, + Points: points, + Tags: tags, + } +} + type Table struct { Columns []TableColumn `json:"columns"` Rows []RowValues `json:"rows"` @@ -154,9 +176,50 @@ func NewTimeSeriesPointsFromArgs(values ...float64) TimeSeriesPoints { return points } -func NewTimeSeries(name string, points TimeSeriesPoints) *TimeSeries { - return &TimeSeries{ - Name: name, - Points: points, +func FormatRawName(idx int, name string, groupByTags []string, tags map[string]string) string { + // when group by tag specified + if len(groupByTags) != 0 { + for key, val := range tags { + if strings.Contains(strings.Join(groupByTags, ","), key) { + return val + } + } } + + genHint := func(k, v string) string { + return fmt.Sprintf("%s=%s", k, v) + } + + hintNames := sets.NewString() + hints := sets.NewString() + for _, tagKey := range api.MEASUREMENT_TAG_KEYWORD { + if tagV, ok := tags[tagKey]; ok { + gHint := genHint(tagKey, tagV) + if strings.Contains(tagKey, "name") { + hintNames.Insert(gHint) + } else { + hints.Insert(gHint) + } + } + } + + if len(hints) == 0 { + // try id + for key, val := range tags { + if strings.Contains(key, "id") && len(val) != 0 { + hints.Insert(genHint(key, val)) + } + } + } + + if len(hints) == 0 { + // if hints is empty at last, return index hint + return fmt.Sprintf("unknown-%d-%s", idx, name) + } + sortNL := hintNames.List() + sort.Strings(sortNL) + sortL := hints.List() + sort.Strings(sortL) + sortNL = append(sortNL, sortL...) + return fmt.Sprintf("{%s}", strings.Join(sortNL, ",")) } diff --git a/pkg/monitor/tsdb/query_endpoint.go b/pkg/monitor/tsdb/query_endpoint.go index 28d7c6024e..80c5af0994 100644 --- a/pkg/monitor/tsdb/query_endpoint.go +++ b/pkg/monitor/tsdb/query_endpoint.go @@ -18,10 +18,13 @@ import ( "context" "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis/monitor" ) type TsdbQueryEndpoint interface { Query(ctx context.Context, ds *DataSource, query *TsdbQuery) (*Response, error) + FilterMeasurement(ctx context.Context, ds *DataSource, from, to string, ms *monitor.InfluxMeasurement, tagFilter *monitor.MetricQueryTag) (*monitor.InfluxMeasurement, error) } var registry map[string]GetTsdbQueryEndpointFn @@ -32,19 +35,33 @@ func init() { registry = make(map[string]GetTsdbQueryEndpointFn) } -const ( - ErrorNotFoundExecutorDataSource = errors.Error("Not find executor for data source") +var ( + ErrorNotFoundExecutorDataSource error = errors.Error("Not find executor for data source") ) -func getTsdbQueryEndpointFor(dsInfo *DataSource) (TsdbQueryEndpoint, error) { - if fn, exists := registry[dsInfo.Type]; exists { - executor, err := fn(dsInfo) - if err != nil { - return nil, err - } - return executor, nil +func getDataSourceFunc(dsType string) (GetTsdbQueryEndpointFn, error) { + fn, exists := registry[dsType] + if !exists { + return nil, errors.Wrapf(ErrorNotFoundExecutorDataSource, "type: %s", dsType) } - return nil, errors.Wrapf(ErrorNotFoundExecutorDataSource, "type: %s", dsInfo.Type) + return fn, nil +} + +func GetTsdbQueryEndpointFor(dsInfo *DataSource) (TsdbQueryEndpoint, error) { + fn, err := getDataSourceFunc(dsInfo.Type) + if err != nil { + return nil, errors.Wrap(err, "getDataSourceFunc") + } + executor, err := fn(dsInfo) + if err != nil { + return nil, errors.Wrap(err, "construct datasource query endpoint") + } + return executor, nil +} + +func IsValidDataSource(dsType string) error { + _, err := getDataSourceFunc(dsType) + return err } func RegisterTsdbQueryEndpoint(dataSourceType string, fn GetTsdbQueryEndpointFn) { diff --git a/pkg/monitor/tsdb/request.go b/pkg/monitor/tsdb/request.go index adf8c8aecf..492e85120e 100644 --- a/pkg/monitor/tsdb/request.go +++ b/pkg/monitor/tsdb/request.go @@ -21,7 +21,7 @@ import ( type HandleRequestFunc func(ctx context.Context, dsInfo *DataSource, req *TsdbQuery) (*Response, error) func HandleRequest(ctx context.Context, dsInfo *DataSource, req *TsdbQuery) (*Response, error) { - endpoint, err := getTsdbQueryEndpointFor(dsInfo) + endpoint, err := GetTsdbQueryEndpointFor(dsInfo) if err != nil { return nil, err } diff --git a/vendor/github.com/cespare/xxhash/LICENSE.txt b/vendor/github.com/cespare/xxhash/LICENSE.txt new file mode 100644 index 0000000000..24b53065f4 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cespare/xxhash/README.md b/vendor/github.com/cespare/xxhash/README.md new file mode 100644 index 0000000000..0982fd25e5 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/README.md @@ -0,0 +1,50 @@ +# xxhash + +[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) + +xxhash is a Go implementation of the 64-bit +[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +high-quality hashing algorithm that is much faster than anything in the Go +standard library. + +The API is very small, taking its cue from the other hashing packages in the +standard library: + + $ go doc github.com/cespare/xxhash ! + package xxhash // import "github.com/cespare/xxhash" + + Package xxhash implements the 64-bit variant of xxHash (XXH64) as described + at http://cyan4973.github.io/xxHash/. + + func New() hash.Hash64 + func Sum64(b []byte) uint64 + func Sum64String(s string) uint64 + +This implementation provides a fast pure-Go implementation and an even faster +assembly implementation for amd64. + +## Benchmarks + +Here are some quick benchmarks comparing the pure-Go and assembly +implementations of Sum64 against another popular Go XXH64 implementation, +[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash): + +| input size | OneOfOne | cespare (purego) | cespare | +| --- | --- | --- | --- | +| 5 B | 416 MB/s | 720 MB/s | 872 MB/s | +| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s | +| 4 KB | 12727 MB/s | 12999 MB/s | 13026 MB/s | +| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s | + +These numbers were generated with: + +``` +$ go test -benchtime 10s -bench '/OneOfOne,' +$ go test -tags purego -benchtime 10s -bench '/xxhash,' +$ go test -benchtime 10s -bench '/xxhash,' +``` + +## Projects using this package + +- [InfluxDB](https://github.com/influxdata/influxdb) +- [Prometheus](https://github.com/prometheus/prometheus) diff --git a/vendor/github.com/cespare/xxhash/rotate.go b/vendor/github.com/cespare/xxhash/rotate.go new file mode 100644 index 0000000000..f3eac5ebc0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/rotate.go @@ -0,0 +1,14 @@ +// +build !go1.9 + +package xxhash + +// TODO(caleb): After Go 1.10 comes out, remove this fallback code. + +func rol1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) } +func rol7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) } +func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) } +func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) } +func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) } +func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) } +func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) } +func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) } diff --git a/vendor/github.com/cespare/xxhash/rotate19.go b/vendor/github.com/cespare/xxhash/rotate19.go new file mode 100644 index 0000000000..b99612bab8 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/rotate19.go @@ -0,0 +1,14 @@ +// +build go1.9 + +package xxhash + +import "math/bits" + +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/cespare/xxhash/xxhash.go b/vendor/github.com/cespare/xxhash/xxhash.go new file mode 100644 index 0000000000..f896bd28f0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash.go @@ -0,0 +1,168 @@ +// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described +// at http://cyan4973.github.io/xxHash/. +package xxhash + +import ( + "encoding/binary" + "hash" +) + +const ( + prime1 uint64 = 11400714785074694791 + prime2 uint64 = 14029467366897019727 + prime3 uint64 = 1609587929392839161 + prime4 uint64 = 9650029242287828579 + prime5 uint64 = 2870177450012600261 +) + +// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where +// possible in the Go code is worth a small (but measurable) performance boost +// by avoiding some MOVQs. Vars are needed for the asm and also are useful for +// convenience in the Go code in a few places where we need to intentionally +// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the +// result overflows a uint64). +var ( + prime1v = prime1 + prime2v = prime2 + prime3v = prime3 + prime4v = prime4 + prime5v = prime5 +) + +type xxh struct { + v1 uint64 + v2 uint64 + v3 uint64 + v4 uint64 + total int + mem [32]byte + n int // how much of mem is used +} + +// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm. +func New() hash.Hash64 { + var x xxh + x.Reset() + return &x +} + +func (x *xxh) Reset() { + x.n = 0 + x.total = 0 + x.v1 = prime1v + prime2 + x.v2 = prime2 + x.v3 = 0 + x.v4 = -prime1v +} + +func (x *xxh) Size() int { return 8 } +func (x *xxh) BlockSize() int { return 32 } + +// Write adds more data to x. It always returns len(b), nil. +func (x *xxh) Write(b []byte) (n int, err error) { + n = len(b) + x.total += len(b) + + if x.n+len(b) < 32 { + // This new data doesn't even fill the current block. + copy(x.mem[x.n:], b) + x.n += len(b) + return + } + + if x.n > 0 { + // Finish off the partial block. + copy(x.mem[x.n:], b) + x.v1 = round(x.v1, u64(x.mem[0:8])) + x.v2 = round(x.v2, u64(x.mem[8:16])) + x.v3 = round(x.v3, u64(x.mem[16:24])) + x.v4 = round(x.v4, u64(x.mem[24:32])) + b = b[32-x.n:] + x.n = 0 + } + + if len(b) >= 32 { + // One or more full blocks left. + b = writeBlocks(x, b) + } + + // Store any remaining partial block. + copy(x.mem[:], b) + x.n = len(b) + + return +} + +func (x *xxh) Sum(b []byte) []byte { + s := x.Sum64() + return append( + b, + byte(s>>56), + byte(s>>48), + byte(s>>40), + byte(s>>32), + byte(s>>24), + byte(s>>16), + byte(s>>8), + byte(s), + ) +} + +func (x *xxh) Sum64() uint64 { + var h uint64 + + if x.total >= 32 { + v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = x.v3 + prime5 + } + + h += uint64(x.total) + + i, end := 0, x.n + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(x.mem[i:i+8])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(x.mem[i:i+4])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for i < end { + h ^= uint64(x.mem[i]) * prime5 + h = rol11(h) * prime1 + i++ + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } + +func round(acc, input uint64) uint64 { + acc += input * prime2 + acc = rol31(acc) + acc *= prime1 + return acc +} + +func mergeRound(acc, val uint64) uint64 { + val = round(0, val) + acc ^= val + acc = acc*prime1 + prime4 + return acc +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/xxhash_amd64.go new file mode 100644 index 0000000000..d617652680 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.go @@ -0,0 +1,12 @@ +// +build !appengine +// +build gc +// +build !purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +// +//go:noescape +func Sum64(b []byte) uint64 + +func writeBlocks(x *xxh, b []byte) []byte diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/xxhash_amd64.s new file mode 100644 index 0000000000..757f2011f0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.s @@ -0,0 +1,233 @@ +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Register allocation: +// AX h +// CX pointer to advance through b +// DX n +// BX loop end +// R8 v1, k1 +// R9 v2 +// R10 v3 +// R11 v4 +// R12 tmp +// R13 prime1v +// R14 prime2v +// R15 prime4v + +// round reads from and advances the buffer pointer in CX. +// It assumes that R13 has prime1v and R14 has prime2v. +#define round(r) \ + MOVQ (CX), R12 \ + ADDQ $8, CX \ + IMULQ R14, R12 \ + ADDQ R12, r \ + ROLQ $31, r \ + IMULQ R13, r + +// mergeRound applies a merge round on the two registers acc and val. +// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. +#define mergeRound(acc, val) \ + IMULQ R14, val \ + ROLQ $31, val \ + IMULQ R13, val \ + XORQ val, acc \ + IMULQ R13, acc \ + ADDQ R15, acc + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT, $0-32 + // Load fixed primes. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + MOVQ ·prime4v(SB), R15 + + // Load slice. + MOVQ b_base+0(FP), CX + MOVQ b_len+8(FP), DX + LEAQ (CX)(DX*1), BX + + // The first loop limit will be len(b)-32. + SUBQ $32, BX + + // Check whether we have at least one block. + CMPQ DX, $32 + JLT noBlocks + + // Set up initial state (v1, v2, v3, v4). + MOVQ R13, R8 + ADDQ R14, R8 + MOVQ R14, R9 + XORQ R10, R10 + XORQ R11, R11 + SUBQ R13, R11 + + // Loop until CX > BX. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + MOVQ R8, AX + ROLQ $1, AX + MOVQ R9, R12 + ROLQ $7, R12 + ADDQ R12, AX + MOVQ R10, R12 + ROLQ $12, R12 + ADDQ R12, AX + MOVQ R11, R12 + ROLQ $18, R12 + ADDQ R12, AX + + mergeRound(AX, R8) + mergeRound(AX, R9) + mergeRound(AX, R10) + mergeRound(AX, R11) + + JMP afterBlocks + +noBlocks: + MOVQ ·prime5v(SB), AX + +afterBlocks: + ADDQ DX, AX + + // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. + ADDQ $24, BX + + CMPQ CX, BX + JG fourByte + +wordLoop: + // Calculate k1. + MOVQ (CX), R8 + ADDQ $8, CX + IMULQ R14, R8 + ROLQ $31, R8 + IMULQ R13, R8 + + XORQ R8, AX + ROLQ $27, AX + IMULQ R13, AX + ADDQ R15, AX + + CMPQ CX, BX + JLE wordLoop + +fourByte: + ADDQ $4, BX + CMPQ CX, BX + JG singles + + MOVL (CX), R8 + ADDQ $4, CX + IMULQ R13, R8 + XORQ R8, AX + + ROLQ $23, AX + IMULQ R14, AX + ADDQ ·prime3v(SB), AX + +singles: + ADDQ $4, BX + CMPQ CX, BX + JGE finalize + +singlesLoop: + MOVBQZX (CX), R12 + ADDQ $1, CX + IMULQ ·prime5v(SB), R12 + XORQ R12, AX + + ROLQ $11, AX + IMULQ R13, AX + + CMPQ CX, BX + JL singlesLoop + +finalize: + MOVQ AX, R12 + SHRQ $33, R12 + XORQ R12, AX + IMULQ R14, AX + MOVQ AX, R12 + SHRQ $29, R12 + XORQ R12, AX + IMULQ ·prime3v(SB), AX + MOVQ AX, R12 + SHRQ $32, R12 + XORQ R12, AX + + MOVQ AX, ret+24(FP) + RET + +// writeBlocks uses the same registers as above except that it uses AX to store +// the x pointer. + +// func writeBlocks(x *xxh, b []byte) []byte +TEXT ·writeBlocks(SB), NOSPLIT, $0-56 + // Load fixed primes needed for round. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + + // Load slice. + MOVQ b_base+8(FP), CX + MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below + MOVQ b_len+16(FP), DX + LEAQ (CX)(DX*1), BX + SUBQ $32, BX + + // Load vN from x. + MOVQ x+0(FP), AX + MOVQ 0(AX), R8 // v1 + MOVQ 8(AX), R9 // v2 + MOVQ 16(AX), R10 // v3 + MOVQ 24(AX), R11 // v4 + + // We don't need to check the loop condition here; this function is + // always called with at least one block of data to process. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + // Copy vN back to x. + MOVQ R8, 0(AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + + // Construct return slice. + // NOTE: It's important that we don't construct a slice that has a base + // pointer off the end of the original slice, as in Go 1.7+ this will + // cause runtime crashes. (See discussion in, for example, + // https://github.com/golang/go/issues/16772.) + // Therefore, we calculate the length/cap first, and if they're zero, we + // keep the old base. This is what the compiler does as well if you + // write code like + // b = b[len(b):] + + // New length is 32 - (CX - BX) -> BX+32 - CX. + ADDQ $32, BX + SUBQ CX, BX + JZ afterSetBase + + MOVQ CX, ret_base+32(FP) + +afterSetBase: + MOVQ BX, ret_len+40(FP) + MOVQ BX, ret_cap+48(FP) // set cap == len + + RET diff --git a/vendor/github.com/cespare/xxhash/xxhash_other.go b/vendor/github.com/cespare/xxhash/xxhash_other.go new file mode 100644 index 0000000000..c68d13f89e --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_other.go @@ -0,0 +1,75 @@ +// +build !amd64 appengine !gc purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 { + // A simpler version would be + // x := New() + // x.Write(b) + // return x.Sum64() + // but this is faster, particularly for small inputs. + + n := len(b) + var h uint64 + + if n >= 32 { + v1 := prime1v + prime2 + v2 := prime2 + v3 := uint64(0) + v4 := -prime1v + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = prime5 + } + + h += uint64(n) + + i, end := 0, len(b) + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(b[i:i+8:len(b)])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for ; i < end; i++ { + h ^= uint64(b[i]) * prime5 + h = rol11(h) * prime1 + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func writeBlocks(x *xxh, b []byte) []byte { + v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4 + return b +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_safe.go b/vendor/github.com/cespare/xxhash/xxhash_safe.go new file mode 100644 index 0000000000..dfa15ab7e2 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_safe.go @@ -0,0 +1,10 @@ +// +build appengine + +// This file contains the safe implementations of otherwise unsafe-using code. + +package xxhash + +// Sum64String computes the 64-bit xxHash digest of s. +func Sum64String(s string) uint64 { + return Sum64([]byte(s)) +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go new file mode 100644 index 0000000000..d2b64e8bb0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go @@ -0,0 +1,30 @@ +// +build !appengine + +// This file encapsulates usage of unsafe. +// xxhash_safe.go contains the safe implementations. + +package xxhash + +import ( + "reflect" + "unsafe" +) + +// Sum64String computes the 64-bit xxHash digest of s. +// It may be faster than Sum64([]byte(s)) by avoiding a copy. +// +// TODO(caleb): Consider removing this if an optimization is ever added to make +// it unnecessary: https://golang.org/issue/2205. +// +// TODO(caleb): We still have a function call; we could instead write Go/asm +// copies of Sum64 for strings to squeeze out a bit more speed. +func Sum64String(s string) uint64 { + // See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ + // for some discussion about this unsafe conversion. + var b []byte + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data + bh.Len = len(s) + bh.Cap = len(s) + return Sum64(b) +} diff --git a/vendor/github.com/influxdata/influxql/.gitignore b/vendor/github.com/influxdata/influxql/.gitignore new file mode 100644 index 0000000000..35903bd94a --- /dev/null +++ b/vendor/github.com/influxdata/influxql/.gitignore @@ -0,0 +1,14 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ \ No newline at end of file diff --git a/vendor/github.com/influxdata/influxql/Jenkinsfile b/vendor/github.com/influxdata/influxql/Jenkinsfile new file mode 100644 index 0000000000..9087a35a27 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/Jenkinsfile @@ -0,0 +1,35 @@ +pipeline { + agent { + docker { + image 'golang:1.9.2' + } + } + + stages { + stage('Test') { + steps { + sh """ + rm -f $WORKSPACE/test-results.{log,xml} + mkdir -p /go/src/github.com/influxdata + cp -a $WORKSPACE /go/src/github.com/influxdata/influxql + + cd /go/src/github.com/influxdata/influxql + go get -v -t + go test -v | tee $WORKSPACE/test-results.log + """ + } + + post { + always { + sh """ + if [ -e test-results.log ]; then + go get github.com/jstemmer/go-junit-report + go-junit-report < $WORKSPACE/test-results.log > test-results.xml + fi + """ + junit "test-results.xml" + } + } + } + } +} diff --git a/vendor/github.com/influxdata/influxql/LICENSE b/vendor/github.com/influxdata/influxql/LICENSE new file mode 100644 index 0000000000..63cef79ba6 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/influxdata/influxql/README.md b/vendor/github.com/influxdata/influxql/README.md new file mode 100644 index 0000000000..5fc340ce27 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/README.md @@ -0,0 +1,1170 @@ +# The Influx Query Language Specification + +## Introduction + +This is a reference for the Influx Query Language ("InfluxQL"). + +InfluxQL is a SQL-like query language for interacting with InfluxDB. It has +been lovingly crafted to feel familiar to those coming from other SQL or +SQL-like environments while providing features specific to storing and analyzing +time series data. + + +## Notation + +The syntax is specified using Extended Backus-Naur Form ("EBNF"). EBNF is the +same notation used in the [Go](http://golang.org) programming language +specification, which can be found [here](https://golang.org/ref/spec). Not so +coincidentally, InfluxDB is written in Go. + +``` +Production = production_name "=" [ Expression ] "." . +Expression = Alternative { "|" Alternative } . +Alternative = Term { Term } . +Term = production_name | token [ "…" token ] | Group | Option | Repetition . +Group = "(" Expression ")" . +Option = "[" Expression "]" . +Repetition = "{" Expression "}" . +``` + +Notation operators in order of increasing precedence: + +``` +| alternation +() grouping +[] option (0 or 1 times) +{} repetition (0 to n times) +``` + +## Comments + +Both single and multiline comments are supported. A comment is treated +the same as whitespace by the parser. + +``` +-- single line comment +/* + multiline comment +*/ +``` + +Single line comments will skip all text until the scanner hits a +newline. Multiline comments will skip all text until the end comment +marker is hit. Nested multiline comments are not supported so the +following does not work: + +``` +/* /* this does not work */ */ +``` + +## Query representation + +### Characters + +InfluxQL is Unicode text encoded in [UTF-8](http://en.wikipedia.org/wiki/UTF-8). + +``` +newline = /* the Unicode code point U+000A */ . +unicode_char = /* an arbitrary Unicode code point except newline */ . +``` + +## Letters and digits + +Letters are the set of ASCII characters plus the underscore character _ (U+005F) +is considered a letter. + +Only decimal digits are supported. + +``` +letter = ascii_letter | "_" . +ascii_letter = "A" … "Z" | "a" … "z" . +digit = "0" … "9" . +``` + +## Identifiers + +Identifiers are tokens which refer to database names, retention policy names, +user names, measurement names, tag keys, and field keys. + +The rules: + +- double quoted identifiers can contain any unicode character other than a new line +- double quoted identifiers can contain escaped `"` characters (i.e., `\"`) +- double quoted identifiers can contain InfluxQL keywords +- unquoted identifiers must start with an upper or lowercase ASCII character or "_" +- unquoted identifiers may contain only ASCII letters, decimal digits, and "_" + +``` +identifier = unquoted_identifier | quoted_identifier . +unquoted_identifier = ( letter ) { letter | digit } . +quoted_identifier = `"` unicode_char { unicode_char } `"` . +``` + +#### Examples: + +``` +cpu +_cpu_stats +"1h" +"anything really" +"1_Crazy-1337.identifier>NAME👍" +``` + +## Keywords + +``` +ALL ALTER ANALYZE ANY AS ASC +BEGIN BY CREATE CONTINUOUS DATABASE DATABASES +DEFAULT DELETE DESC DESTINATIONS DIAGNOSTICS DISTINCT +DROP DURATION END EVERY EXPLAIN FIELD +FOR FROM GRANT GRANTS GROUP GROUPS +IN INF INSERT INTO KEY KEYS +KILL LIMIT SHOW MEASUREMENT MEASUREMENTS NAME +OFFSET ON ORDER PASSWORD POLICY POLICIES +PRIVILEGES QUERIES QUERY READ REPLICATION RESAMPLE +RETENTION REVOKE SELECT SERIES SET SHARD +SHARDS SLIMIT SOFFSET STATS SUBSCRIPTION SUBSCRIPTIONS +TAG TO USER USERS VALUES WHERE +WITH WRITE +``` + +## Literals + +### Integers + +InfluxQL supports decimal integer literals. Hexadecimal and octal literals are +not currently supported. + +``` +int_lit = [ "+" | "-" ] ( "1" … "9" ) { digit } . +``` + +### Floats + +InfluxQL supports floating-point literals. Exponents are not currently supported. + +``` +float_lit = [ "+" | "-" ] ( "." digit { digit } | digit { digit } "." { digit } ) . +``` + +### Strings + +String literals must be surrounded by single quotes. Strings may contain `'` +characters as long as they are escaped (i.e., `\'`). + +``` +string_lit = `'` { unicode_char } `'` . +``` + +### Durations + +Duration literals specify a length of time. An integer literal followed +immediately (with no spaces) by a duration unit listed below is interpreted as +a duration literal. + +### Duration units +| Units | Meaning | +|--------|-----------------------------------------| +| u or µ | microseconds (1 millionth of a second) | +| ms | milliseconds (1 thousandth of a second) | +| s | second | +| m | minute | +| h | hour | +| d | day | +| w | week | + +``` +duration_lit = int_lit duration_unit . +duration_unit = "u" | "µ" | "ms" | "s" | "m" | "h" | "d" | "w" . +``` + +### Dates & Times + +The date and time literal format is not specified in EBNF like the rest of this document. It is specified using Go's date / time parsing format, which is a reference date written in the format required by InfluxQL. The reference date time is: + +InfluxQL reference date time: January 2nd, 2006 at 3:04:05 PM + +``` +time_lit = "2006-01-02 15:04:05.999999" | "2006-01-02" . +``` + +### Booleans + +``` +bool_lit = TRUE | FALSE . +``` + +### Regular Expressions + +``` +regex_lit = "/" { unicode_char } "/" . +``` + +**Comparators:** +`=~` matches against +`!~` doesn't match against + +> **Note:** Use regular expressions to match measurements and tags. +You cannot use regular expressions to match databases, retention policies, or fields. + +## Queries + +A query is composed of one or more statements separated by a semicolon. + +``` +query = statement { ";" statement } . + +statement = alter_retention_policy_stmt | + create_continuous_query_stmt | + create_database_stmt | + create_retention_policy_stmt | + create_subscription_stmt | + create_user_stmt | + delete_stmt | + drop_continuous_query_stmt | + drop_database_stmt | + drop_measurement_stmt | + drop_retention_policy_stmt | + drop_series_stmt | + drop_shard_stmt | + drop_subscription_stmt | + drop_user_stmt | + explain_stmt | + grant_stmt | + kill_query_statement | + show_continuous_queries_stmt | + show_databases_stmt | + show_field_keys_stmt | + show_grants_stmt | + show_measurements_stmt | + show_queries_stmt | + show_retention_policies | + show_series_stmt | + show_shard_groups_stmt | + show_shards_stmt | + show_subscriptions_stmt| + show_tag_keys_stmt | + show_tag_values_stmt | + show_users_stmt | + revoke_stmt | + select_stmt . +``` + +## Statements + +### ALTER RETENTION POLICY + +``` +alter_retention_policy_stmt = "ALTER RETENTION POLICY" policy_name on_clause + retention_policy_option + [ retention_policy_option ] + [ retention_policy_option ] + [ retention_policy_option ] . +``` + +> Replication factors do not serve a purpose with single node instances. + +#### Examples: + +```sql +-- Set default retention policy for mydb to 1h.cpu. +ALTER RETENTION POLICY "1h.cpu" ON "mydb" DEFAULT + +-- Change duration and replication factor. +ALTER RETENTION POLICY "policy1" ON "somedb" DURATION 1h REPLICATION 4 +``` + +### CREATE CONTINUOUS QUERY + +``` +create_continuous_query_stmt = "CREATE CONTINUOUS QUERY" query_name on_clause + [ "RESAMPLE" resample_opts ] + "BEGIN" select_stmt "END" . + +query_name = identifier . + +resample_opts = (every_stmt for_stmt | every_stmt | for_stmt) . +every_stmt = "EVERY" duration_lit +for_stmt = "FOR" duration_lit +``` + +#### Examples: + +```sql +-- selects from DEFAULT retention policy and writes into 6_months retention policy +CREATE CONTINUOUS QUERY "10m_event_count" +ON "db_name" +BEGIN + SELECT count("value") + INTO "6_months"."events" + FROM "events" + GROUP BY time(10m) +END; + +-- this selects from the output of one continuous query in one retention policy and outputs to another series in another retention policy +CREATE CONTINUOUS QUERY "1h_event_count" +ON "db_name" +BEGIN + SELECT sum("count") as "count" + INTO "2_years"."events" + FROM "6_months"."events" + GROUP BY time(1h) +END; + +-- this customizes the resample interval so the interval is queried every 10s and intervals are resampled until 2m after their start time +-- when resample is used, at least one of "EVERY" or "FOR" must be used +CREATE CONTINUOUS QUERY "cpu_mean" +ON "db_name" +RESAMPLE EVERY 10s FOR 2m +BEGIN + SELECT mean("value") + INTO "cpu_mean" + FROM "cpu" + GROUP BY time(1m) +END; +``` + +### CREATE DATABASE + +``` +create_database_stmt = "CREATE DATABASE" db_name + [ WITH + [ retention_policy_duration ] + [ retention_policy_replication ] + [ retention_policy_shard_group_duration ] + [ retention_policy_name ] + ] . +``` + +> Replication factors do not serve a purpose with single node instances. + +#### Examples: + +```sql +-- Create a database called foo +CREATE DATABASE "foo" + +-- Create a database called bar with a new DEFAULT retention policy and specify the duration, replication, shard group duration, and name of that retention policy +CREATE DATABASE "bar" WITH DURATION 1d REPLICATION 1 SHARD DURATION 30m NAME "myrp" + +-- Create a database called mydb with a new DEFAULT retention policy and specify the name of that retention policy +CREATE DATABASE "mydb" WITH NAME "myrp" +``` + +### CREATE RETENTION POLICY + +``` +create_retention_policy_stmt = "CREATE RETENTION POLICY" policy_name on_clause + retention_policy_duration + retention_policy_replication + [ retention_policy_shard_group_duration ] + [ "DEFAULT" ] . +``` + +> Replication factors do not serve a purpose with single node instances. + +#### Examples + +```sql +-- Create a retention policy. +CREATE RETENTION POLICY "10m.events" ON "somedb" DURATION 60m REPLICATION 2 + +-- Create a retention policy and set it as the DEFAULT. +CREATE RETENTION POLICY "10m.events" ON "somedb" DURATION 60m REPLICATION 2 DEFAULT + +-- Create a retention policy and specify the shard group duration. +CREATE RETENTION POLICY "10m.events" ON "somedb" DURATION 60m REPLICATION 2 SHARD DURATION 30m +``` + +### CREATE SUBSCRIPTION + +Subscriptions tell InfluxDB to send all the data it receives to Kapacitor or other third parties. + +``` +create_subscription_stmt = "CREATE SUBSCRIPTION" subscription_name "ON" db_name "." retention_policy "DESTINATIONS" ("ANY"|"ALL") host { "," host} . +``` + +#### Examples: + +```sql +-- Create a SUBSCRIPTION on database 'mydb' and retention policy 'autogen' that send data to 'example.com:9090' via UDP. +CREATE SUBSCRIPTION "sub0" ON "mydb"."autogen" DESTINATIONS ALL 'udp://example.com:9090' + +-- Create a SUBSCRIPTION on database 'mydb' and retention policy 'autogen' that round robins the data to 'h1.example.com:9090' and 'h2.example.com:9090'. +CREATE SUBSCRIPTION "sub0" ON "mydb"."autogen" DESTINATIONS ANY 'udp://h1.example.com:9090', 'udp://h2.example.com:9090' +``` + +### CREATE USER + +``` +create_user_stmt = "CREATE USER" user_name "WITH PASSWORD" password + [ "WITH ALL PRIVILEGES" ] . +``` + +#### Examples: + +```sql +-- Create a normal database user. +CREATE USER "jdoe" WITH PASSWORD '1337password' + +-- Create an admin user. +-- Note: Unlike the GRANT statement, the "PRIVILEGES" keyword is required here. +CREATE USER "jdoe" WITH PASSWORD '1337password' WITH ALL PRIVILEGES +``` + +> **Note:** The password string must be wrapped in single quotes. + +### DELETE + +``` +delete_stmt = "DELETE" ( from_clause | where_clause | from_clause where_clause ) . +``` + +#### Examples: + +```sql +DELETE FROM "cpu" +DELETE FROM "cpu" WHERE time < '2000-01-01T00:00:00Z' +DELETE WHERE time < '2000-01-01T00:00:00Z' +``` + +### DROP CONTINUOUS QUERY + +``` +drop_continuous_query_stmt = "DROP CONTINUOUS QUERY" query_name on_clause . +``` + +#### Example: + +```sql +DROP CONTINUOUS QUERY "myquery" ON "mydb" +``` + +### DROP DATABASE + +``` +drop_database_stmt = "DROP DATABASE" db_name . +``` + +#### Example: + +```sql +DROP DATABASE "mydb" +``` + +### DROP MEASUREMENT + +``` +drop_measurement_stmt = "DROP MEASUREMENT" measurement . +``` + +#### Examples: + +```sql +-- drop the cpu measurement +DROP MEASUREMENT "cpu" +``` + +### DROP RETENTION POLICY + +``` +drop_retention_policy_stmt = "DROP RETENTION POLICY" policy_name on_clause . +``` + +#### Example: + +```sql +-- drop the retention policy named 1h.cpu from mydb +DROP RETENTION POLICY "1h.cpu" ON "mydb" +``` + +### DROP SERIES + +``` +drop_series_stmt = "DROP SERIES" ( from_clause | where_clause | from_clause where_clause ) . +``` + +#### Example: + +```sql +DROP SERIES FROM "telegraf"."autogen"."cpu" WHERE cpu = 'cpu8' + +``` + +### DROP SHARD + +``` +drop_shard_stmt = "DROP SHARD" ( shard_id ) . +``` + +#### Example: + +``` +DROP SHARD 1 +``` + +### DROP SUBSCRIPTION + +``` +drop_subscription_stmt = "DROP SUBSCRIPTION" subscription_name "ON" db_name "." retention_policy . +``` + +#### Example: + +```sql +DROP SUBSCRIPTION "sub0" ON "mydb"."autogen" +``` + +### DROP USER + +``` +drop_user_stmt = "DROP USER" user_name . +``` + +#### Example: + +```sql +DROP USER "jdoe" +``` + +### EXPLAIN + +> **NOTE:** This functionality is unimplemented. + +``` +explain_stmt = "EXPLAIN" [ "ANALYZE" ] select_stmt . +``` + +### GRANT + +> **NOTE:** Users can be granted privileges on databases that do not exist. + +``` +grant_stmt = "GRANT" privilege [ on_clause ] to_clause . +``` + +#### Examples: + +```sql +-- grant admin privileges +GRANT ALL TO "jdoe" + +-- grant read access to a database +GRANT READ ON "mydb" TO "jdoe" +``` + +### KILL QUERY + +``` +kill_query_statement = "KILL QUERY" query_id . +``` + +#### Examples: + +``` +--- kill a query with the query_id 36 +KILL QUERY 36 +``` + +> **NOTE:** Identify the `query_id` from the `SHOW QUERIES` output. + +### SHOW CONTINUOUS QUERIES + +``` +show_continuous_queries_stmt = "SHOW CONTINUOUS QUERIES" . +``` + +#### Example: + +```sql +-- show all continuous queries +SHOW CONTINUOUS QUERIES +``` + +### SHOW DATABASES + +``` +show_databases_stmt = "SHOW DATABASES" . +``` + +#### Example: + +```sql +-- show all databases +SHOW DATABASES +``` + +### SHOW FIELD KEYS + +``` +show_field_keys_stmt = "SHOW FIELD KEYS" [ from_clause ] . +``` + +#### Examples: + +```sql +-- show field keys and field value data types from all measurements +SHOW FIELD KEYS + +-- show field keys and field value data types from specified measurement +SHOW FIELD KEYS FROM "cpu" +``` + +### SHOW GRANTS + +``` +show_grants_stmt = "SHOW GRANTS FOR" user_name . +``` + +#### Example: + +```sql +-- show grants for jdoe +SHOW GRANTS FOR "jdoe" +``` + +### SHOW MEASUREMENTS + +``` +show_measurements_stmt = "SHOW MEASUREMENTS" [ with_measurement_clause ] [ where_clause ] [ limit_clause ] [ offset_clause ] . +``` + +#### Examples: + +```sql +-- show all measurements +SHOW MEASUREMENTS + +-- show measurements where region tag = 'uswest' AND host tag = 'serverA' +SHOW MEASUREMENTS WHERE "region" = 'uswest' AND "host" = 'serverA' + +-- show measurements that start with 'h2o' +SHOW MEASUREMENTS WITH MEASUREMENT =~ /h2o.*/ +``` + +### SHOW QUERIES + +``` +show_queries_stmt = "SHOW QUERIES" . +``` + +#### Example: + +```sql +-- show all currently-running queries +SHOW QUERIES +``` + +### SHOW RETENTION POLICIES + +``` +show_retention_policies = "SHOW RETENTION POLICIES" on_clause . +``` + +#### Example: + +```sql +-- show all retention policies on a database +SHOW RETENTION POLICIES ON "mydb" +``` + +### SHOW SERIES + +``` +show_series_stmt = "SHOW SERIES" [ from_clause ] [ where_clause ] [ limit_clause ] [ offset_clause ] . +``` + +#### Example: + +```sql +SHOW SERIES FROM "telegraf"."autogen"."cpu" WHERE cpu = 'cpu8' +``` + +### SHOW SHARD GROUPS + +``` +show_shard_groups_stmt = "SHOW SHARD GROUPS" . +``` + +#### Example: + +```sql +SHOW SHARD GROUPS +``` + +### SHOW SHARDS + +``` +show_shards_stmt = "SHOW SHARDS" . +``` + +#### Example: + +```sql +SHOW SHARDS +``` + +### SHOW SUBSCRIPTIONS + +``` +show_subscriptions_stmt = "SHOW SUBSCRIPTIONS" . +``` + +#### Example: + +```sql +SHOW SUBSCRIPTIONS +``` + +### SHOW TAG KEYS + +``` +show_tag_keys_stmt = "SHOW TAG KEYS" [ from_clause ] [ where_clause ] [ group_by_clause ] + [ limit_clause ] [ offset_clause ] . +``` + +#### Examples: + +```sql +-- show all tag keys +SHOW TAG KEYS + +-- show all tag keys from the cpu measurement +SHOW TAG KEYS FROM "cpu" + +-- show all tag keys from the cpu measurement where the region key = 'uswest' +SHOW TAG KEYS FROM "cpu" WHERE "region" = 'uswest' + +-- show all tag keys where the host key = 'serverA' +SHOW TAG KEYS WHERE "host" = 'serverA' +``` + +### SHOW TAG VALUES + +``` +show_tag_values_stmt = "SHOW TAG VALUES" [ from_clause ] with_tag_clause [ where_clause ] + [ group_by_clause ] [ limit_clause ] [ offset_clause ] . +``` + +#### Examples: + +```sql +-- show all tag values across all measurements for the region tag +SHOW TAG VALUES WITH KEY = "region" + +-- show tag values from the cpu measurement for the region tag +SHOW TAG VALUES FROM "cpu" WITH KEY = "region" + +-- show tag values across all measurements for all tag keys that do not include the letter c +SHOW TAG VALUES WITH KEY !~ /.*c.*/ + +-- show tag values from the cpu measurement for region & host tag keys where service = 'redis' +SHOW TAG VALUES FROM "cpu" WITH KEY IN ("region", "host") WHERE "service" = 'redis' +``` + +### SHOW USERS + +``` +show_users_stmt = "SHOW USERS" . +``` + +#### Example: + +```sql +-- show all users +SHOW USERS +``` + +### REVOKE + +``` +revoke_stmt = "REVOKE" privilege [ on_clause ] "FROM" user_name . +``` + +#### Examples: + +```sql +-- revoke admin privileges from jdoe +REVOKE ALL PRIVILEGES FROM "jdoe" + +-- revoke read privileges from jdoe on mydb +REVOKE READ ON "mydb" FROM "jdoe" +``` + +### SELECT + +``` +select_stmt = "SELECT" fields from_clause [ into_clause ] [ where_clause ] + [ group_by_clause ] [ order_by_clause ] [ limit_clause ] + [ offset_clause ] [ slimit_clause ] [ soffset_clause ] + [ timezone_clause ] . +``` + +#### Examples: + +```sql +-- select mean value from the cpu measurement where region = 'uswest' grouped by 10 minute intervals +SELECT mean("value") FROM "cpu" WHERE "region" = 'uswest' GROUP BY time(10m) fill(0) + +-- select from all measurements beginning with cpu into the same measurement name in the cpu_1h retention policy +SELECT mean("value") INTO "cpu_1h".:MEASUREMENT FROM /cpu.*/ + +-- select from measurements grouped by the day with a timezone +SELECT mean("value") FROM "cpu" GROUP BY region, time(1d) fill(0) tz("America/Chicago") +``` + +## Clauses + +``` +from_clause = "FROM" measurements . + +group_by_clause = "GROUP BY" dimensions fill(fill_option). + +into_clause = "INTO" ( measurement | back_ref ). + +limit_clause = "LIMIT" int_lit . + +offset_clause = "OFFSET" int_lit . + +slimit_clause = "SLIMIT" int_lit . + +soffset_clause = "SOFFSET" int_lit . + +timezone_clause = tz(string_lit) . + +on_clause = "ON" db_name . + +order_by_clause = "ORDER BY" sort_fields . + +to_clause = "TO" user_name . + +where_clause = "WHERE" expr . + +with_measurement_clause = "WITH MEASUREMENT" ( "=" measurement | "=~" regex_lit ) . + +with_tag_clause = "WITH KEY" ( "=" tag_key | "!=" tag_key | "=~" regex_lit | "IN (" tag_keys ")" ) . +``` + +## Expressions + +``` +binary_op = "+" | "-" | "*" | "/" | "%" | "&" | "|" | "^" | "AND" | + "OR" | "=" | "!=" | "<>" | "<" | "<=" | ">" | ">=" . + +expr = unary_expr { binary_op unary_expr } . + +unary_expr = "(" expr ")" | var_ref | time_lit | string_lit | int_lit | + float_lit | bool_lit | duration_lit | regex_lit . +``` + +## Other + +``` +alias = "AS" identifier . + +back_ref = ( policy_name ".:MEASUREMENT" ) | + ( db_name "." [ policy_name ] ".:MEASUREMENT" ) . + +db_name = identifier . + +dimension = expr . + +dimensions = dimension { "," dimension } . + +field_key = identifier . + +field = expr [ alias ] . + +fields = field { "," field } . + +fill_option = "null" | "none" | "previous" | "linear" | int_lit | float_lit . + +host = string_lit . + +measurement = measurement_name | + ( policy_name "." measurement_name ) | + ( db_name "." [ policy_name ] "." measurement_name ) . + +measurements = measurement { "," measurement } . + +measurement_name = identifier | regex_lit . + +password = string_lit . + +policy_name = identifier . + +privilege = "ALL" [ "PRIVILEGES" ] | "READ" | "WRITE" . + +query_id = int_lit . + +query_name = identifier . + +retention_policy = identifier . + +retention_policy_option = retention_policy_duration | + retention_policy_replication | + retention_policy_shard_group_duration | + "DEFAULT" . + +retention_policy_duration = "DURATION" duration_lit . + +retention_policy_replication = "REPLICATION" int_lit . + +retention_policy_shard_group_duration = "SHARD DURATION" duration_lit . + +retention_policy_name = "NAME" identifier . + +series_id = int_lit . + +shard_id = int_lit . + +sort_field = field_key [ ASC | DESC ] . + +sort_fields = sort_field { "," sort_field } . + +subscription_name = identifier . + +tag_key = identifier . + +tag_keys = tag_key { "," tag_key } . + +user_name = identifier . + +var_ref = measurement . +``` + +## Query Engine Internals + +Once you understand the language itself, it's important to know how these +language constructs are implemented in the query engine. This gives you an +intuitive sense for how results will be processed and how to create efficient +queries. + +The life cycle of a query looks like this: + +1. InfluxQL query string is tokenized and then parsed into an abstract syntax + tree (AST). This is the code representation of the query itself. + +2. The AST is passed to the `QueryExecutor` which directs queries to the + appropriate handlers. For example, queries related to meta data are executed + by the meta service and `SELECT` statements are executed by the shards + themselves. + +3. The query engine then determines the shards that match the `SELECT` + statement's time range. From these shards, iterators are created for each + field in the statement. + +4. Iterators are passed to the emitter which drains them and joins the resulting + points. The emitter's job is to convert simple time/value points into the + more complex result objects that are returned to the client. + + +### Understanding Iterators + +Iterators are at the heart of the query engine. They provide a simple interface +for looping over a set of points. For example, this is an iterator over Float +points: + +``` +type FloatIterator interface { + Next() (*FloatPoint, error) +} +``` + +These iterators are created through the `IteratorCreator` interface: + +``` +type IteratorCreator interface { + CreateIterator(m *Measurement, opt IteratorOptions) (Iterator, error) +} +``` + +The `IteratorOptions` provide arguments about field selection, time ranges, +and dimensions that the iterator creator can use when planning an iterator. +The `IteratorCreator` interface is used at many levels such as the `Shards`, +`Shard`, and `Engine`. This allows optimizations to be performed when applicable +such as returning a precomputed `COUNT()`. + +Iterators aren't just for reading raw data from storage though. Iterators can be +composed so that they provided additional functionality around an input +iterator. For example, a `DistinctIterator` can compute the distinct values for +each time window for an input iterator. Or a `FillIterator` can generate +additional points that are missing from an input iterator. + +This composition also lends itself well to aggregation. For example, a statement +such as this: + +``` +SELECT MEAN(value) FROM cpu GROUP BY time(10m) +``` + +In this case, `MEAN(value)` is a `MeanIterator` wrapping an iterator from the +underlying shards. However, if we can add an additional iterator to determine +the derivative of the mean: + +``` +SELECT DERIVATIVE(MEAN(value), 20m) FROM cpu GROUP BY time(10m) +``` + + +### Understanding Auxiliary Fields + +Because InfluxQL allows users to use selector functions such as `FIRST()`, +`LAST()`, `MIN()`, and `MAX()`, the engine must provide a way to return related +data at the same time with the selected point. + +For example, in this query: + +``` +SELECT FIRST(value), host FROM cpu GROUP BY time(1h) +``` + +We are selecting the first `value` that occurs every hour but we also want to +retrieve the `host` associated with that point. Since the `Point` types only +specify a single typed `Value` for efficiency, we push the `host` into the +auxiliary fields of the point. These auxiliary fields are attached to the point +until it is passed to the emitter where the fields get split off to their own +iterator. + + +### Built-in Iterators + +There are many helper iterators that let us build queries: + +* Merge Iterator - This iterator combines one or more iterators into a single + new iterator of the same type. This iterator guarantees that all points + within a window will be output before starting the next window but does not + provide ordering guarantees within the window. This allows for fast access + for aggregate queries which do not need stronger sorting guarantees. + +* Sorted Merge Iterator - This iterator also combines one or more iterators + into a new iterator of the same type. However, this iterator guarantees + time ordering of every point. This makes it slower than the `MergeIterator` + but this ordering guarantee is required for non-aggregate queries which + return the raw data points. + +* Limit Iterator - This iterator limits the number of points per name/tag + group. This is the implementation of the `LIMIT` & `OFFSET` syntax. + +* Fill Iterator - This iterator injects extra points if they are missing from + the input iterator. It can provide `null` points, points with the previous + value, or points with a specific value. + +* Buffered Iterator - This iterator provides the ability to "unread" a point + back onto a buffer so it can be read again next time. This is used extensively + to provide lookahead for windowing. + +* Reduce Iterator - This iterator calls a reduction function for each point in + a window. When the window is complete then all points for that window are + output. This is used for simple aggregate functions such as `COUNT()`. + +* Reduce Slice Iterator - This iterator collects all points for a window first + and then passes them all to a reduction function at once. The results are + returned from the iterator. This is used for aggregate functions such as + `DERIVATIVE()`. + +* Transform Iterator - This iterator calls a transform function for each point + from an input iterator. This is used for executing binary expressions. + +* Dedupe Iterator - This iterator only outputs unique points. It is resource + intensive so it is only used for small queries such as meta query statements. + + +### Call Iterators + +Function calls in InfluxQL are implemented at two levels. Some calls can be +wrapped at multiple layers to improve efficiency. For example, a `COUNT()` can +be performed at the shard level and then multiple `CountIterator`s can be +wrapped with another `CountIterator` to compute the count of all shards. These +iterators can be created using `NewCallIterator()`. + +Some iterators are more complex or need to be implemented at a higher level. +For example, the `DERIVATIVE()` needs to retrieve all points for a window first +before performing the calculation. This iterator is created by the engine itself +and is never requested to be created by the lower levels. + +### Subqueries + +Subqueries are built on top of iterators. Most of the work involved in +supporting subqueries is in organizing how data is streamed to the +iterators that will process the data. + +The final ordering of the stream has to output all points from one +series before moving to the next series and it also needs to ensure +those points are printed in order. So there are two separate concepts we +need to consider when creating an iterator: ordering and grouping. + +When an inner query has a different grouping than the outermost query, +we still need to group together related points into buckets, but we do +not have to ensure that all points from one buckets are output before +the points in another bucket. In fact, if we do that, we will be unable +to perform the grouping for the outer query correctly. Instead, we group +all points by the outermost query for an interval and then, within that +interval, we group the points for the inner query. For example, here are +series keys and times in seconds (fields are omitted since they don't +matter in this example): + + cpu,host=server01 0 + cpu,host=server01 10 + cpu,host=server01 20 + cpu,host=server01 30 + cpu,host=server02 0 + cpu,host=server02 10 + cpu,host=server02 20 + cpu,host=server02 30 + +With the following query: + + SELECT mean(max) FROM (SELECT max(value) FROM cpu GROUP BY host, time(20s)) GROUP BY time(20s) + +The final grouping keeps all of the points together which means we need +to group `server01` with `server02`. That means we output the points +from the underlying engine like this: + + cpu,host=server01 0 + cpu,host=server01 10 + cpu,host=server02 0 + cpu,host=server02 10 + cpu,host=server01 20 + cpu,host=server01 30 + cpu,host=server02 20 + cpu,host=server02 30 + +Within each one of those time buckets, we calculate the `max()` value +for each unique host so the output stream gets transformed to look like +this: + + cpu,host=server01 0 + cpu,host=server02 0 + cpu,host=server01 20 + cpu,host=server02 20 + +Then we can process the `mean()` on this stream of data instead and it +will be output in the correct order. This is true of any order of +grouping since grouping can only go from more specific to less specific. + +When it comes to ordering, unordered data is faster to process, but we +always need to produce ordered data. When processing a raw query with no +aggregates, we need to ensure data coming from the engine is ordered so +the output is ordered. When we have an aggregate, we know one point is +being emitted for each interval and will always produce ordered output. +So for aggregates, we can take unordered data as the input and get +ordered output. Any ordered data as input will always result in ordered +data so we just need to look at how an iterator processes unordered +data. + +| | raw query | selector (without group by time) | selector (with group by time) | aggregator | +|-----------------|------------------|----------------------------------|-------------------------------|----------------| +| ordered input | ordered output | ordered output | ordered output | ordered output | +| unordered input | unordered output | unordered output | ordered output | ordered output | + +Since we always need ordered output, we just need to work backwards and +determine which pattern of input gives us ordered output. If both +ordered and unordered input produce ordered output, we prefer unordered +input since it is faster. + +There are also certain aggregates that require ordered input like +`median()` and `percentile()`. These functions will explicitly request +ordered input. It is also important to realize that selectors that are +grouped by time are the equivalent of an aggregator. It is only +selectors without a group by time that are different. diff --git a/vendor/github.com/influxdata/influxql/ast.go b/vendor/github.com/influxdata/influxql/ast.go new file mode 100644 index 0000000000..140a42c838 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/ast.go @@ -0,0 +1,5814 @@ +package influxql + +import ( + "bytes" + "errors" + "fmt" + "math" + "regexp" + "regexp/syntax" + "sort" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + internal "github.com/influxdata/influxql/internal" +) + +// DataType represents the primitive data types available in InfluxQL. +type DataType int + +const ( + // Unknown primitive data type. + Unknown DataType = 0 + // Float means the data type is a float. + Float DataType = 1 + // Integer means the data type is an integer. + Integer DataType = 2 + // String means the data type is a string of text. + String DataType = 3 + // Boolean means the data type is a boolean. + Boolean DataType = 4 + // Time means the data type is a time. + Time DataType = 5 + // Duration means the data type is a duration of time. + Duration DataType = 6 + // Tag means the data type is a tag. + Tag DataType = 7 + // AnyField means the data type is any field. + AnyField DataType = 8 + // Unsigned means the data type is an unsigned integer. + Unsigned DataType = 9 +) + +const ( + // MinTime is the minumum time that can be represented. + // + // 1677-09-21 00:12:43.145224194 +0000 UTC + // + // The two lowest minimum integers are used as sentinel values. The + // minimum value needs to be used as a value lower than any other value for + // comparisons and another separate value is needed to act as a sentinel + // default value that is unusable by the user, but usable internally. + // Because these two values need to be used for a special purpose, we do + // not allow users to write points at these two times. + MinTime = int64(math.MinInt64) + 2 + + // MaxTime is the maximum time that can be represented. + // + // 2262-04-11 23:47:16.854775806 +0000 UTC + // + // The highest time represented by a nanosecond needs to be used for an + // exclusive range in the shard group, so the maximum time needs to be one + // less than the possible maximum number of nanoseconds representable by an + // int64 so that we don't lose a point at that one time. + MaxTime = int64(math.MaxInt64) - 1 +) + +var ( + // ErrInvalidTime is returned when the timestamp string used to + // compare against time field is invalid. + ErrInvalidTime = errors.New("invalid timestamp string") +) + +// InspectDataType returns the data type of a given value. +func InspectDataType(v interface{}) DataType { + switch v.(type) { + case float64: + return Float + case int64, int32, int: + return Integer + case string: + return String + case bool: + return Boolean + case uint64: + return Unsigned + case time.Time: + return Time + case time.Duration: + return Duration + default: + return Unknown + } +} + +// DataTypeFromString returns a data type given the string representation of that +// data type. +func DataTypeFromString(s string) DataType { + switch s { + case "float": + return Float + case "integer": + return Integer + case "unsigned": + return Unsigned + case "string": + return String + case "boolean": + return Boolean + case "time": + return Time + case "duration": + return Duration + case "tag": + return Tag + case "field": + return AnyField + default: + return Unknown + } +} + +// LessThan returns true if the other DataType has greater precedence than the +// current data type. Unknown has the lowest precedence. +// +// NOTE: This is not the same as using the `<` or `>` operator because the +// integers used decrease with higher precedence, but Unknown is the lowest +// precedence at the zero value. +func (d DataType) LessThan(other DataType) bool { + if d == Unknown { + return true + } else if d == Unsigned { + return other != Unknown && other <= Integer + } else if other == Unsigned { + return d >= String + } + return other != Unknown && other < d +} + +var ( + zeroFloat64 interface{} = float64(0) + zeroInt64 interface{} = int64(0) + zeroUint64 interface{} = uint64(0) + zeroString interface{} = "" + zeroBoolean interface{} = false + zeroTime interface{} = time.Time{} + zeroDuration interface{} = time.Duration(0) +) + +// Zero returns the zero value for the DataType. +// The return value of this method, when sent back to InspectDataType, +// may not produce the same value. +func (d DataType) Zero() interface{} { + switch d { + case Float: + return zeroFloat64 + case Integer: + return zeroInt64 + case Unsigned: + return zeroUint64 + case String, Tag: + return zeroString + case Boolean: + return zeroBoolean + case Time: + return zeroTime + case Duration: + return zeroDuration + } + return nil +} + +// String returns the human-readable string representation of the DataType. +func (d DataType) String() string { + switch d { + case Float: + return "float" + case Integer: + return "integer" + case Unsigned: + return "unsigned" + case String: + return "string" + case Boolean: + return "boolean" + case Time: + return "time" + case Duration: + return "duration" + case Tag: + return "tag" + case AnyField: + return "field" + } + return "unknown" +} + +// Node represents a node in the InfluxDB abstract syntax tree. +type Node interface { + // node is unexported to ensure implementations of Node + // can only originate in this package. + node() + String() string +} + +func (*Query) node() {} +func (Statements) node() {} + +func (*AlterRetentionPolicyStatement) node() {} +func (*CreateContinuousQueryStatement) node() {} +func (*CreateDatabaseStatement) node() {} +func (*CreateRetentionPolicyStatement) node() {} +func (*CreateSubscriptionStatement) node() {} +func (*CreateUserStatement) node() {} +func (*Distinct) node() {} +func (*DeleteSeriesStatement) node() {} +func (*DeleteStatement) node() {} +func (*DropContinuousQueryStatement) node() {} +func (*DropDatabaseStatement) node() {} +func (*DropMeasurementStatement) node() {} +func (*DropRetentionPolicyStatement) node() {} +func (*DropSeriesStatement) node() {} +func (*DropShardStatement) node() {} +func (*DropSubscriptionStatement) node() {} +func (*DropUserStatement) node() {} +func (*ExplainStatement) node() {} +func (*GrantStatement) node() {} +func (*GrantAdminStatement) node() {} +func (*KillQueryStatement) node() {} +func (*RevokeStatement) node() {} +func (*RevokeAdminStatement) node() {} +func (*SelectStatement) node() {} +func (*SetPasswordUserStatement) node() {} +func (*ShowContinuousQueriesStatement) node() {} +func (*ShowGrantsForUserStatement) node() {} +func (*ShowDatabasesStatement) node() {} +func (*ShowFieldKeyCardinalityStatement) node() {} +func (*ShowFieldKeysStatement) node() {} +func (*ShowRetentionPoliciesStatement) node() {} +func (*ShowMeasurementCardinalityStatement) node() {} +func (*ShowMeasurementsStatement) node() {} +func (*ShowQueriesStatement) node() {} +func (*ShowSeriesStatement) node() {} +func (*ShowSeriesCardinalityStatement) node() {} +func (*ShowShardGroupsStatement) node() {} +func (*ShowShardsStatement) node() {} +func (*ShowStatsStatement) node() {} +func (*ShowSubscriptionsStatement) node() {} +func (*ShowDiagnosticsStatement) node() {} +func (*ShowTagKeyCardinalityStatement) node() {} +func (*ShowTagKeysStatement) node() {} +func (*ShowTagValuesCardinalityStatement) node() {} +func (*ShowTagValuesStatement) node() {} +func (*ShowUsersStatement) node() {} + +func (*BinaryExpr) node() {} +func (*BooleanLiteral) node() {} +func (*BoundParameter) node() {} +func (*Call) node() {} +func (*Dimension) node() {} +func (Dimensions) node() {} +func (*DurationLiteral) node() {} +func (*IntegerLiteral) node() {} +func (*UnsignedLiteral) node() {} +func (*Field) node() {} +func (Fields) node() {} +func (*Measurement) node() {} +func (Measurements) node() {} +func (*NilLiteral) node() {} +func (*NumberLiteral) node() {} +func (*ParenExpr) node() {} +func (*RegexLiteral) node() {} +func (*ListLiteral) node() {} +func (*SortField) node() {} +func (SortFields) node() {} +func (Sources) node() {} +func (*StringLiteral) node() {} +func (*SubQuery) node() {} +func (*Target) node() {} +func (*TimeLiteral) node() {} +func (*VarRef) node() {} +func (*Wildcard) node() {} + +// Query represents a collection of ordered statements. +type Query struct { + Statements Statements +} + +// String returns a string representation of the query. +func (q *Query) String() string { return q.Statements.String() } + +// Statements represents a list of statements. +type Statements []Statement + +// String returns a string representation of the statements. +func (a Statements) String() string { + var str []string + for _, stmt := range a { + str = append(str, stmt.String()) + } + return strings.Join(str, ";\n") +} + +// Statement represents a single command in InfluxQL. +type Statement interface { + Node + // stmt is unexported to ensure implementations of Statement + // can only originate in this package. + stmt() + RequiredPrivileges() (ExecutionPrivileges, error) +} + +// HasDefaultDatabase provides an interface to get the default database from a Statement. +type HasDefaultDatabase interface { + Node + // stmt is unexported to ensure implementations of HasDefaultDatabase + // can only originate in this package. + stmt() + DefaultDatabase() string +} + +// ExecutionPrivilege is a privilege required for a user to execute +// a statement on a database or resource. +type ExecutionPrivilege struct { + // Admin privilege required. + Admin bool + + // Name of the database. + Name string + + // Database privilege required. + Privilege Privilege +} + +// ExecutionPrivileges is a list of privileges required to execute a statement. +type ExecutionPrivileges []ExecutionPrivilege + +func (*AlterRetentionPolicyStatement) stmt() {} +func (*CreateContinuousQueryStatement) stmt() {} +func (*CreateDatabaseStatement) stmt() {} +func (*CreateRetentionPolicyStatement) stmt() {} +func (*CreateSubscriptionStatement) stmt() {} +func (*CreateUserStatement) stmt() {} +func (*DeleteSeriesStatement) stmt() {} +func (*DeleteStatement) stmt() {} +func (*DropContinuousQueryStatement) stmt() {} +func (*DropDatabaseStatement) stmt() {} +func (*DropMeasurementStatement) stmt() {} +func (*DropRetentionPolicyStatement) stmt() {} +func (*DropSeriesStatement) stmt() {} +func (*DropSubscriptionStatement) stmt() {} +func (*DropUserStatement) stmt() {} +func (*ExplainStatement) stmt() {} +func (*GrantStatement) stmt() {} +func (*GrantAdminStatement) stmt() {} +func (*KillQueryStatement) stmt() {} +func (*ShowContinuousQueriesStatement) stmt() {} +func (*ShowGrantsForUserStatement) stmt() {} +func (*ShowDatabasesStatement) stmt() {} +func (*ShowFieldKeyCardinalityStatement) stmt() {} +func (*ShowFieldKeysStatement) stmt() {} +func (*ShowMeasurementCardinalityStatement) stmt() {} +func (*ShowMeasurementsStatement) stmt() {} +func (*ShowQueriesStatement) stmt() {} +func (*ShowRetentionPoliciesStatement) stmt() {} +func (*ShowSeriesStatement) stmt() {} +func (*ShowSeriesCardinalityStatement) stmt() {} +func (*ShowShardGroupsStatement) stmt() {} +func (*ShowShardsStatement) stmt() {} +func (*ShowStatsStatement) stmt() {} +func (*DropShardStatement) stmt() {} +func (*ShowSubscriptionsStatement) stmt() {} +func (*ShowDiagnosticsStatement) stmt() {} +func (*ShowTagKeyCardinalityStatement) stmt() {} +func (*ShowTagKeysStatement) stmt() {} +func (*ShowTagValuesCardinalityStatement) stmt() {} +func (*ShowTagValuesStatement) stmt() {} +func (*ShowUsersStatement) stmt() {} +func (*RevokeStatement) stmt() {} +func (*RevokeAdminStatement) stmt() {} +func (*SelectStatement) stmt() {} +func (*SetPasswordUserStatement) stmt() {} + +// Expr represents an expression that can be evaluated to a value. +type Expr interface { + Node + // expr is unexported to ensure implementations of Expr + // can only originate in this package. + expr() +} + +func (*BinaryExpr) expr() {} +func (*BooleanLiteral) expr() {} +func (*BoundParameter) expr() {} +func (*Call) expr() {} +func (*Distinct) expr() {} +func (*DurationLiteral) expr() {} +func (*IntegerLiteral) expr() {} +func (*UnsignedLiteral) expr() {} +func (*NilLiteral) expr() {} +func (*NumberLiteral) expr() {} +func (*ParenExpr) expr() {} +func (*RegexLiteral) expr() {} +func (*ListLiteral) expr() {} +func (*StringLiteral) expr() {} +func (*TimeLiteral) expr() {} +func (*VarRef) expr() {} +func (*Wildcard) expr() {} + +// Literal represents a static literal. +type Literal interface { + Expr + // literal is unexported to ensure implementations of Literal + // can only originate in this package. + literal() +} + +func (*BooleanLiteral) literal() {} +func (*BoundParameter) literal() {} +func (*DurationLiteral) literal() {} +func (*IntegerLiteral) literal() {} +func (*UnsignedLiteral) literal() {} +func (*NilLiteral) literal() {} +func (*NumberLiteral) literal() {} +func (*RegexLiteral) literal() {} +func (*ListLiteral) literal() {} +func (*StringLiteral) literal() {} +func (*TimeLiteral) literal() {} + +// Source represents a source of data for a statement. +type Source interface { + Node + // source is unexported to ensure implementations of Source + // can only originate in this package. + source() +} + +func (*Measurement) source() {} +func (*SubQuery) source() {} + +// Sources represents a list of sources. +type Sources []Source + +// String returns a string representation of a Sources array. +func (a Sources) String() string { + var buf bytes.Buffer + + ubound := len(a) - 1 + for i, src := range a { + _, _ = buf.WriteString(src.String()) + if i < ubound { + _, _ = buf.WriteString(", ") + } + } + + return buf.String() +} + +// Measurements returns all measurements including ones embedded in subqueries. +func (a Sources) Measurements() []*Measurement { + mms := make([]*Measurement, 0, len(a)) + for _, src := range a { + switch src := src.(type) { + case *Measurement: + mms = append(mms, src) + case *SubQuery: + mms = append(mms, src.Statement.Sources.Measurements()...) + } + } + return mms +} + +// MarshalBinary encodes a list of sources to a binary format. +func (a Sources) MarshalBinary() ([]byte, error) { + var pb internal.Measurements + pb.Items = make([]*internal.Measurement, len(a)) + for i, source := range a { + pb.Items[i] = encodeMeasurement(source.(*Measurement)) + } + return proto.Marshal(&pb) +} + +// UnmarshalBinary decodes binary data into a list of sources. +func (a *Sources) UnmarshalBinary(buf []byte) error { + var pb internal.Measurements + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + *a = make(Sources, len(pb.GetItems())) + for i := range pb.GetItems() { + mm, err := decodeMeasurement(pb.GetItems()[i]) + if err != nil { + return err + } + (*a)[i] = mm + } + return nil +} + +// RequiredPrivileges recursively returns a list of execution privileges required. +func (a Sources) RequiredPrivileges() (ExecutionPrivileges, error) { + var ep ExecutionPrivileges + for _, source := range a { + switch source := source.(type) { + case *Measurement: + ep = append(ep, ExecutionPrivilege{ + Name: source.Database, + Privilege: ReadPrivilege, + }) + case *SubQuery: + privs, err := source.Statement.RequiredPrivileges() + if err != nil { + return nil, err + } + ep = append(ep, privs...) + default: + return nil, fmt.Errorf("invalid source: %s", source) + } + } + return ep, nil +} + +// IsSystemName returns true if name is an internal system name. +func IsSystemName(name string) bool { + switch name { + case "_fieldKeys", + "_measurements", + "_name", + "_series", + "_tagKey", + "_tagKeys", + "_tags": + return true + default: + return false + } +} + +// SortField represents a field to sort results by. +type SortField struct { + // Name of the field. + Name string + + // Sort order. + Ascending bool +} + +// String returns a string representation of a sort field. +func (field *SortField) String() string { + var buf bytes.Buffer + if field.Name != "" { + _, _ = buf.WriteString(field.Name) + _, _ = buf.WriteString(" ") + } + if field.Ascending { + _, _ = buf.WriteString("ASC") + } else { + _, _ = buf.WriteString("DESC") + } + return buf.String() +} + +// SortFields represents an ordered list of ORDER BY fields. +type SortFields []*SortField + +// String returns a string representation of sort fields. +func (a SortFields) String() string { + fields := make([]string, 0, len(a)) + for _, field := range a { + fields = append(fields, field.String()) + } + return strings.Join(fields, ", ") +} + +// CreateDatabaseStatement represents a command for creating a new database. +type CreateDatabaseStatement struct { + // Name of the database to be created. + Name string + + // RetentionPolicyCreate indicates whether the user explicitly wants to create a retention policy. + RetentionPolicyCreate bool + + // RetentionPolicyDuration indicates retention duration for the new database. + RetentionPolicyDuration *time.Duration + + // RetentionPolicyReplication indicates retention replication for the new database. + RetentionPolicyReplication *int + + // RetentionPolicyName indicates retention name for the new database. + RetentionPolicyName string + + // RetentionPolicyShardGroupDuration indicates shard group duration for the new database. + RetentionPolicyShardGroupDuration time.Duration +} + +// String returns a string representation of the create database statement. +func (s *CreateDatabaseStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("CREATE DATABASE ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + if s.RetentionPolicyCreate { + _, _ = buf.WriteString(" WITH") + if s.RetentionPolicyDuration != nil { + _, _ = buf.WriteString(" DURATION ") + _, _ = buf.WriteString(s.RetentionPolicyDuration.String()) + } + if s.RetentionPolicyReplication != nil { + _, _ = buf.WriteString(" REPLICATION ") + _, _ = buf.WriteString(strconv.Itoa(*s.RetentionPolicyReplication)) + } + if s.RetentionPolicyShardGroupDuration > 0 { + _, _ = buf.WriteString(" SHARD DURATION ") + _, _ = buf.WriteString(s.RetentionPolicyShardGroupDuration.String()) + } + if s.RetentionPolicyName != "" { + _, _ = buf.WriteString(" NAME ") + _, _ = buf.WriteString(QuoteIdent(s.RetentionPolicyName)) + } + } + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a CreateDatabaseStatement. +func (s *CreateDatabaseStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DropDatabaseStatement represents a command to drop a database. +type DropDatabaseStatement struct { + // Name of the database to be dropped. + Name string +} + +// String returns a string representation of the drop database statement. +func (s *DropDatabaseStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("DROP DATABASE ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a DropDatabaseStatement. +func (s *DropDatabaseStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DropRetentionPolicyStatement represents a command to drop a retention policy from a database. +type DropRetentionPolicyStatement struct { + // Name of the policy to drop. + Name string + + // Name of the database to drop the policy from. + Database string +} + +// String returns a string representation of the drop retention policy statement. +func (s *DropRetentionPolicyStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("DROP RETENTION POLICY ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a DropRetentionPolicyStatement. +func (s *DropRetentionPolicyStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: WritePrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *DropRetentionPolicyStatement) DefaultDatabase() string { + return s.Database +} + +// CreateUserStatement represents a command for creating a new user. +type CreateUserStatement struct { + // Name of the user to be created. + Name string + + // User's password. + Password string + + // User's admin privilege. + Admin bool +} + +// String returns a string representation of the create user statement. +func (s *CreateUserStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("CREATE USER ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" WITH PASSWORD ") + _, _ = buf.WriteString("[REDACTED]") + if s.Admin { + _, _ = buf.WriteString(" WITH ALL PRIVILEGES") + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a CreateUserStatement. +func (s *CreateUserStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DropUserStatement represents a command for dropping a user. +type DropUserStatement struct { + // Name of the user to drop. + Name string +} + +// String returns a string representation of the drop user statement. +func (s *DropUserStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("DROP USER ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a DropUserStatement. +func (s *DropUserStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// Privilege is a type of action a user can be granted the right to use. +type Privilege int + +const ( + // NoPrivileges means no privileges required / granted / revoked. + NoPrivileges Privilege = iota + // ReadPrivilege means read privilege required / granted / revoked. + ReadPrivilege + // WritePrivilege means write privilege required / granted / revoked. + WritePrivilege + // AllPrivileges means all privileges required / granted / revoked. + AllPrivileges +) + +// NewPrivilege returns an initialized *Privilege. +func NewPrivilege(p Privilege) *Privilege { return &p } + +// String returns a string representation of a Privilege. +func (p Privilege) String() string { + switch p { + case NoPrivileges: + return "NO PRIVILEGES" + case ReadPrivilege: + return "READ" + case WritePrivilege: + return "WRITE" + case AllPrivileges: + return "ALL PRIVILEGES" + } + return "" +} + +// GrantStatement represents a command for granting a privilege. +type GrantStatement struct { + // The privilege to be granted. + Privilege Privilege + + // Database to grant the privilege to. + On string + + // Who to grant the privilege to. + User string +} + +// String returns a string representation of the grant statement. +func (s *GrantStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("GRANT ") + _, _ = buf.WriteString(s.Privilege.String()) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.On)) + _, _ = buf.WriteString(" TO ") + _, _ = buf.WriteString(QuoteIdent(s.User)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a GrantStatement. +func (s *GrantStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *GrantStatement) DefaultDatabase() string { + return s.On +} + +// GrantAdminStatement represents a command for granting admin privilege. +type GrantAdminStatement struct { + // Who to grant the privilege to. + User string +} + +// String returns a string representation of the grant admin statement. +func (s *GrantAdminStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("GRANT ALL PRIVILEGES TO ") + _, _ = buf.WriteString(QuoteIdent(s.User)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a GrantAdminStatement. +func (s *GrantAdminStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// KillQueryStatement represents a command for killing a query. +type KillQueryStatement struct { + // The query to kill. + QueryID uint64 + + // The host to delegate the kill to. + Host string +} + +// String returns a string representation of the kill query statement. +func (s *KillQueryStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("KILL QUERY ") + _, _ = buf.WriteString(strconv.FormatUint(s.QueryID, 10)) + if s.Host != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Host)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a KillQueryStatement. +func (s *KillQueryStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// SetPasswordUserStatement represents a command for changing user password. +type SetPasswordUserStatement struct { + // Plain-text password. + Password string + + // Who to grant the privilege to. + Name string +} + +// String returns a string representation of the set password statement. +func (s *SetPasswordUserStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SET PASSWORD FOR ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" = ") + _, _ = buf.WriteString("[REDACTED]") + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a SetPasswordUserStatement. +func (s *SetPasswordUserStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// RevokeStatement represents a command to revoke a privilege from a user. +type RevokeStatement struct { + // The privilege to be revoked. + Privilege Privilege + + // Database to revoke the privilege from. + On string + + // Who to revoke privilege from. + User string +} + +// String returns a string representation of the revoke statement. +func (s *RevokeStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("REVOKE ") + _, _ = buf.WriteString(s.Privilege.String()) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.On)) + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(QuoteIdent(s.User)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a RevokeStatement. +func (s *RevokeStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *RevokeStatement) DefaultDatabase() string { + return s.On +} + +// RevokeAdminStatement represents a command to revoke admin privilege from a user. +type RevokeAdminStatement struct { + // Who to revoke admin privilege from. + User string +} + +// String returns a string representation of the revoke admin statement. +func (s *RevokeAdminStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("REVOKE ALL PRIVILEGES FROM ") + _, _ = buf.WriteString(QuoteIdent(s.User)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a RevokeAdminStatement. +func (s *RevokeAdminStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// CreateRetentionPolicyStatement represents a command to create a retention policy. +type CreateRetentionPolicyStatement struct { + // Name of policy to create. + Name string + + // Name of database this policy belongs to. + Database string + + // Duration data written to this policy will be retained. + Duration time.Duration + + // Replication factor for data written to this policy. + Replication int + + // Should this policy be set as default for the database? + Default bool + + // Shard Duration. + ShardGroupDuration time.Duration +} + +// String returns a string representation of the create retention policy. +func (s *CreateRetentionPolicyStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("CREATE RETENTION POLICY ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + _, _ = buf.WriteString(" DURATION ") + _, _ = buf.WriteString(FormatDuration(s.Duration)) + _, _ = buf.WriteString(" REPLICATION ") + _, _ = buf.WriteString(strconv.Itoa(s.Replication)) + if s.ShardGroupDuration > 0 { + _, _ = buf.WriteString(" SHARD DURATION ") + _, _ = buf.WriteString(FormatDuration(s.ShardGroupDuration)) + } + if s.Default { + _, _ = buf.WriteString(" DEFAULT") + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a CreateRetentionPolicyStatement. +func (s *CreateRetentionPolicyStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *CreateRetentionPolicyStatement) DefaultDatabase() string { + return s.Database +} + +// AlterRetentionPolicyStatement represents a command to alter an existing retention policy. +type AlterRetentionPolicyStatement struct { + // Name of policy to alter. + Name string + + // Name of the database this policy belongs to. + Database string + + // Duration data written to this policy will be retained. + Duration *time.Duration + + // Replication factor for data written to this policy. + Replication *int + + // Should this policy be set as defalut for the database? + Default bool + + // Duration of the Shard. + ShardGroupDuration *time.Duration +} + +// String returns a string representation of the alter retention policy statement. +func (s *AlterRetentionPolicyStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("ALTER RETENTION POLICY ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + + if s.Duration != nil { + _, _ = buf.WriteString(" DURATION ") + _, _ = buf.WriteString(FormatDuration(*s.Duration)) + } + + if s.Replication != nil { + _, _ = buf.WriteString(" REPLICATION ") + _, _ = buf.WriteString(strconv.Itoa(*s.Replication)) + } + + if s.ShardGroupDuration != nil { + _, _ = buf.WriteString(" SHARD DURATION ") + _, _ = buf.WriteString(FormatDuration(*s.ShardGroupDuration)) + } + + if s.Default { + _, _ = buf.WriteString(" DEFAULT") + } + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute an AlterRetentionPolicyStatement. +func (s *AlterRetentionPolicyStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *AlterRetentionPolicyStatement) DefaultDatabase() string { + return s.Database +} + +// FillOption represents different options for filling aggregate windows. +type FillOption int + +const ( + // NullFill means that empty aggregate windows will just have null values. + NullFill FillOption = iota + // NoFill means that empty aggregate windows will be purged from the result. + NoFill + // NumberFill means that empty aggregate windows will be filled with a provided number. + NumberFill + // PreviousFill means that empty aggregate windows will be filled with whatever the previous aggregate window had. + PreviousFill + // LinearFill means that empty aggregate windows will be filled with whatever a linear value between non null windows. + LinearFill +) + +// SelectStatement represents a command for extracting data from the database. +type SelectStatement struct { + // Expressions returned from the selection. + Fields Fields + + // Target (destination) for the result of a SELECT INTO query. + Target *Target + + // Expressions used for grouping the selection. + Dimensions Dimensions + + // Data sources (measurements) that fields are extracted from. + Sources Sources + + // An expression evaluated on data point. + Condition Expr + + // Fields to sort results by. + SortFields SortFields + + // Maximum number of rows to be returned. Unlimited if zero. + Limit int + + // Returns rows starting at an offset from the first row. + Offset int + + // Maxiumum number of series to be returned. Unlimited if zero. + SLimit int + + // Returns series starting at an offset from the first one. + SOffset int + + // Memoized group by interval from GroupBy(). + groupByInterval time.Duration + + // Whether it's a query for raw data values (i.e. not an aggregate). + IsRawQuery bool + + // What fill option the select statement uses, if any. + Fill FillOption + + // The value to fill empty aggregate buckets with, if any. + FillValue interface{} + + // The timezone for the query, if any. + Location *time.Location + + // Renames the implicit time field name. + TimeAlias string + + // Removes the "time" column from the output. + OmitTime bool + + // Removes measurement name from resulting query. Useful for meta queries. + StripName bool + + // Overrides the output measurement name. + EmitName string + + // Removes duplicate rows from raw queries. + Dedupe bool +} + +// TimeAscending returns true if the time field is sorted in chronological order. +func (s *SelectStatement) TimeAscending() bool { + return len(s.SortFields) == 0 || s.SortFields[0].Ascending +} + +// TimeFieldName returns the name of the time field. +func (s *SelectStatement) TimeFieldName() string { + if s.TimeAlias != "" { + return s.TimeAlias + } + return "time" +} + +// Clone returns a deep copy of the statement. +func (s *SelectStatement) Clone() *SelectStatement { + clone := *s + clone.Fields = make(Fields, 0, len(s.Fields)) + clone.Dimensions = make(Dimensions, 0, len(s.Dimensions)) + clone.Sources = cloneSources(s.Sources) + clone.SortFields = make(SortFields, 0, len(s.SortFields)) + clone.Condition = CloneExpr(s.Condition) + + if s.Target != nil { + clone.Target = &Target{ + Measurement: &Measurement{ + Database: s.Target.Measurement.Database, + RetentionPolicy: s.Target.Measurement.RetentionPolicy, + Name: s.Target.Measurement.Name, + Regex: CloneRegexLiteral(s.Target.Measurement.Regex), + }, + } + } + for _, f := range s.Fields { + clone.Fields = append(clone.Fields, &Field{Expr: CloneExpr(f.Expr), Alias: f.Alias}) + } + for _, d := range s.Dimensions { + clone.Dimensions = append(clone.Dimensions, &Dimension{Expr: CloneExpr(d.Expr)}) + } + for _, f := range s.SortFields { + clone.SortFields = append(clone.SortFields, &SortField{Name: f.Name, Ascending: f.Ascending}) + } + return &clone +} + +func cloneSources(sources Sources) Sources { + clone := make(Sources, 0, len(sources)) + for _, s := range sources { + clone = append(clone, cloneSource(s)) + } + return clone +} + +func cloneSource(s Source) Source { + if s == nil { + return nil + } + + switch s := s.(type) { + case *Measurement: + return s.Clone() + case *SubQuery: + return &SubQuery{Statement: s.Statement.Clone()} + default: + panic("unreachable") + } +} + +// FieldMapper returns the data type for the field inside of the measurement. +type FieldMapper interface { + FieldDimensions(m *Measurement) (fields map[string]DataType, dimensions map[string]struct{}, err error) + + TypeMapper +} + +// RewriteFields returns the re-written form of the select statement. Any wildcard query +// fields are replaced with the supplied fields, and any wildcard GROUP BY fields are replaced +// with the supplied dimensions. Any fields with no type specifier are rewritten with the +// appropriate type. +func (s *SelectStatement) RewriteFields(m FieldMapper) (*SelectStatement, error) { + // Clone the statement so we aren't rewriting the original. + other := s.Clone() + + // Iterate through the sources and rewrite any subqueries first. + for _, src := range other.Sources { + switch src := src.(type) { + case *SubQuery: + stmt, err := src.Statement.RewriteFields(m) + if err != nil { + return nil, err + } + src.Statement = stmt + } + } + + // Rewrite all variable references in the fields with their types if one + // hasn't been specified. + rewrite := func(n Node) { + ref, ok := n.(*VarRef) + if !ok || (ref.Type != Unknown && ref.Type != AnyField) { + return + } + + typ := EvalType(ref, other.Sources, m) + if typ == Tag && ref.Type == AnyField { + return + } + ref.Type = typ + } + WalkFunc(other.Fields, rewrite) + WalkFunc(other.Condition, rewrite) + + // Ignore if there are no wildcards. + hasFieldWildcard := other.HasFieldWildcard() + hasDimensionWildcard := other.HasDimensionWildcard() + if !hasFieldWildcard && !hasDimensionWildcard { + return other, nil + } + + fieldSet, dimensionSet, err := FieldDimensions(other.Sources, m) + if err != nil { + return nil, err + } + + // If there are no dimension wildcards then merge dimensions to fields. + if !hasDimensionWildcard { + // Remove the dimensions present in the group by so they don't get added as fields. + for _, d := range other.Dimensions { + switch expr := d.Expr.(type) { + case *VarRef: + delete(dimensionSet, expr.Val) + } + } + } + + // Sort the field and dimension names for wildcard expansion. + var fields []VarRef + if len(fieldSet) > 0 { + fields = make([]VarRef, 0, len(fieldSet)) + for name, typ := range fieldSet { + fields = append(fields, VarRef{Val: name, Type: typ}) + } + if !hasDimensionWildcard { + for name := range dimensionSet { + fields = append(fields, VarRef{Val: name, Type: Tag}) + } + dimensionSet = nil + } + sort.Sort(VarRefs(fields)) + } + dimensions := stringSetSlice(dimensionSet) + + // Rewrite all wildcard query fields + if hasFieldWildcard { + // Allocate a slice assuming there is exactly one wildcard for efficiency. + rwFields := make(Fields, 0, len(other.Fields)+len(fields)-1) + for _, f := range other.Fields { + switch expr := f.Expr.(type) { + case *Wildcard: + for _, ref := range fields { + if expr.Type == FIELD && ref.Type == Tag { + continue + } else if expr.Type == TAG && ref.Type != Tag { + continue + } + rwFields = append(rwFields, &Field{Expr: &VarRef{Val: ref.Val, Type: ref.Type}}) + } + case *RegexLiteral: + for _, ref := range fields { + if expr.Val.MatchString(ref.Val) { + rwFields = append(rwFields, &Field{Expr: &VarRef{Val: ref.Val, Type: ref.Type}}) + } + } + case *Call: + // Clone a template that we can modify and use for new fields. + template := CloneExpr(expr).(*Call) + + // Search for the call with a wildcard by continuously descending until + // we no longer have a call. + call := template + for len(call.Args) > 0 { + arg, ok := call.Args[0].(*Call) + if !ok { + break + } + call = arg + } + + // Check if this field value is a wildcard. + if len(call.Args) == 0 { + rwFields = append(rwFields, f) + continue + } + + // Retrieve if this is a wildcard or a regular expression. + var re *regexp.Regexp + switch expr := call.Args[0].(type) { + case *Wildcard: + if expr.Type == TAG { + return nil, fmt.Errorf("unable to use tag wildcard in %s()", call.Name) + } + case *RegexLiteral: + re = expr.Val + default: + rwFields = append(rwFields, f) + continue + } + + // All types that can expand wildcards support float, integer, and unsigned. + supportedTypes := map[DataType]struct{}{ + Float: {}, + Integer: {}, + Unsigned: {}, + } + + // Add additional types for certain functions. + switch call.Name { + case "count", "first", "last", "distinct", "elapsed", "mode", "sample": + supportedTypes[String] = struct{}{} + fallthrough + case "min", "max": + supportedTypes[Boolean] = struct{}{} + case "holt_winters", "holt_winters_with_fit": + delete(supportedTypes, Unsigned) + } + + for _, ref := range fields { + // Do not expand tags within a function call. It likely won't do anything + // anyway and will be the wrong thing in 99% of cases. + if ref.Type == Tag { + continue + } else if _, ok := supportedTypes[ref.Type]; !ok { + continue + } else if re != nil && !re.MatchString(ref.Val) { + continue + } + + // Make a new expression and replace the wildcard within this cloned expression. + call.Args[0] = &VarRef{Val: ref.Val, Type: ref.Type} + rwFields = append(rwFields, &Field{ + Expr: CloneExpr(template), + Alias: fmt.Sprintf("%s_%s", f.Name(), ref.Val), + }) + } + case *BinaryExpr: + // Search for regexes or wildcards within the binary + // expression. If we find any, throw an error indicating that + // it's illegal. + var regex, wildcard bool + WalkFunc(expr, func(n Node) { + switch n.(type) { + case *RegexLiteral: + regex = true + case *Wildcard: + wildcard = true + } + }) + + if wildcard { + return nil, fmt.Errorf("unsupported expression with wildcard: %s", f.Expr) + } else if regex { + return nil, fmt.Errorf("unsupported expression with regex field: %s", f.Expr) + } + rwFields = append(rwFields, f) + default: + rwFields = append(rwFields, f) + } + } + other.Fields = rwFields + } + + // Rewrite all wildcard GROUP BY fields + if hasDimensionWildcard { + // Allocate a slice assuming there is exactly one wildcard for efficiency. + rwDimensions := make(Dimensions, 0, len(other.Dimensions)+len(dimensions)-1) + for _, d := range other.Dimensions { + switch expr := d.Expr.(type) { + case *Wildcard: + for _, name := range dimensions { + rwDimensions = append(rwDimensions, &Dimension{Expr: &VarRef{Val: name}}) + } + case *RegexLiteral: + for _, name := range dimensions { + if expr.Val.MatchString(name) { + rwDimensions = append(rwDimensions, &Dimension{Expr: &VarRef{Val: name}}) + } + } + default: + rwDimensions = append(rwDimensions, d) + } + } + other.Dimensions = rwDimensions + } + + return other, nil +} + +// RewriteRegexConditions rewrites regex conditions to make better use of the +// database index. +// +// Conditions that can currently be simplified are: +// +// - host =~ /^foo$/ becomes host = 'foo' +// - host !~ /^foo$/ becomes host != 'foo' +// +// Note: if the regex contains groups, character classes, repetition or +// similar, it's likely it won't be rewritten. In order to support rewriting +// regexes with these characters would be a lot more work. +func (s *SelectStatement) RewriteRegexConditions() { + s.Condition = RewriteExpr(s.Condition, func(e Expr) Expr { + be, ok := e.(*BinaryExpr) + if !ok || (be.Op != EQREGEX && be.Op != NEQREGEX) { + // This expression is not a binary condition or doesn't have a + // regex based operator. + return e + } + + // Handle regex-based condition. + rhs := be.RHS.(*RegexLiteral) // This must be a regex. + + vals, ok := matchExactRegex(rhs.Val.String()) + if !ok { + // Regex didn't match. + return e + } + + // Update the condition operator. + var concatOp Token + if be.Op == EQREGEX { + be.Op = EQ + concatOp = OR + } else { + be.Op = NEQ + concatOp = AND + } + + // Remove leading and trailing ^ and $. + switch { + case len(vals) == 0: + be.RHS = &StringLiteral{} + case len(vals) == 1: + be.RHS = &StringLiteral{Val: vals[0]} + default: + expr := &BinaryExpr{ + Op: be.Op, + LHS: be.LHS, + RHS: &StringLiteral{Val: vals[0]}, + } + for i := 1; i < len(vals); i++ { + expr = &BinaryExpr{ + Op: concatOp, + LHS: expr, + RHS: &BinaryExpr{ + Op: be.Op, + LHS: be.LHS, + RHS: &StringLiteral{Val: vals[i]}, + }, + } + } + return &ParenExpr{Expr: expr} + } + return be + }) + + // Unwrap any top level parenthesis. + if cond, ok := s.Condition.(*ParenExpr); ok { + s.Condition = cond.Expr + } +} + +// matchExactRegex matches regexes into literals if possible. This will match the +// pattern /^foo$/ or /^(foo|bar)$/. It considers /^$/ to be a matching regex. +func matchExactRegex(v string) ([]string, bool) { + re, err := syntax.Parse(v, syntax.Perl) + if err != nil { + // Nothing we can do or log. + return nil, false + } + re = re.Simplify() + + if re.Op != syntax.OpConcat { + return nil, false + } + + if len(re.Sub) < 2 { + // Regex has too few subexpressions. + return nil, false + } + + start := re.Sub[0] + if !(start.Op == syntax.OpBeginLine || start.Op == syntax.OpBeginText) { + // Regex does not begin with ^ + return nil, false + } + + end := re.Sub[len(re.Sub)-1] + if !(end.Op == syntax.OpEndLine || end.Op == syntax.OpEndText) { + // Regex does not end with $ + return nil, false + } + + // Remove the begin and end text from the regex. + re.Sub = re.Sub[1 : len(re.Sub)-1] + + if len(re.Sub) == 0 { + // The regex /^$/ + return nil, true + } + return matchRegex(re) +} + +// matchRegex will match a regular expression to literals if possible. +func matchRegex(re *syntax.Regexp) ([]string, bool) { + + // Maximum number of literals that the expression should be expanded to. If + // this is exceeded, no expansion will be done. This allows reasonable + // optimizations of regex by expansion to literals but prevents cases + // where that expansion would result in a large number of literals. + const maxLiterals = 100 + + // Exit if we see a case-insensitive flag as it is not something we support at this time. + if re.Flags&syntax.FoldCase != 0 { + return nil, false + } + + switch re.Op { + case syntax.OpLiteral: + // We can rewrite this regex. + return []string{string(re.Rune)}, true + case syntax.OpCapture: + return matchRegex(re.Sub[0]) + case syntax.OpConcat: + // Go through each of the subs and concatenate the result to each one. + names, ok := matchRegex(re.Sub[0]) + if !ok { + return nil, false + } + + for _, sub := range re.Sub[1:] { + vals, ok := matchRegex(sub) + if !ok { + return nil, false + } + + // If there is only one value, concatenate it to all strings rather + // than allocate a new slice. + if len(vals) == 1 { + for i := range names { + names[i] += vals[0] + } + continue + } else if len(names) == 1 { + // If there is only one value, then do this concatenation in + // the opposite direction. + for i := range vals { + vals[i] = names[0] + vals[i] + } + names = vals + continue + } + + sz := len(names) * len(vals) + if sz > maxLiterals { + return nil, false + } + + // The long method of using multiple concatenations. + concat := make([]string, sz) + for i := range names { + for j := range vals { + concat[i*len(vals)+j] = names[i] + vals[j] + } + } + names = concat + } + return names, true + case syntax.OpCharClass: + var sz int + for i := 0; i < len(re.Rune); i += 2 { + sz += int(re.Rune[i+1]) - int(re.Rune[i]) + 1 + } + + if sz > maxLiterals { + return nil, false + } + + names := make([]string, 0, sz) + for i := 0; i < len(re.Rune); i += 2 { + for r := int(re.Rune[i]); r <= int(re.Rune[i+1]); r++ { + names = append(names, string([]rune{rune(r)})) + } + } + return names, true + case syntax.OpAlternate: + var names []string + for _, sub := range re.Sub { + vals, ok := matchRegex(sub) + if !ok { + return nil, false + } + names = append(names, vals...) + } + if len(names) > maxLiterals { + return nil, false + } + return names, true + } + return nil, false +} + +// RewriteDistinct rewrites the expression to be a call for map/reduce to work correctly. +// This method assumes all validation has passed. +func (s *SelectStatement) RewriteDistinct() { + WalkFunc(s.Fields, func(n Node) { + switch n := n.(type) { + case *Field: + if expr, ok := n.Expr.(*Distinct); ok { + n.Expr = expr.NewCall() + s.IsRawQuery = false + } + case *Call: + for i, arg := range n.Args { + if arg, ok := arg.(*Distinct); ok { + n.Args[i] = arg.NewCall() + } + } + } + }) +} + +// RewriteTimeFields removes any "time" field references. +func (s *SelectStatement) RewriteTimeFields() { + for i := 0; i < len(s.Fields); i++ { + switch expr := s.Fields[i].Expr.(type) { + case *VarRef: + if expr.Val == "time" { + s.TimeAlias = s.Fields[i].Alias + s.Fields = append(s.Fields[:i], s.Fields[i+1:]...) + } + } + } +} + +// ColumnNames will walk all fields and functions and return the appropriate field names for the select statement +// while maintaining order of the field names. +func (s *SelectStatement) ColumnNames() []string { + // First walk each field to determine the number of columns. + columnFields := Fields{} + for _, field := range s.Fields { + columnFields = append(columnFields, field) + + switch f := field.Expr.(type) { + case *Call: + if s.Target == nil && (f.Name == "top" || f.Name == "bottom") { + for _, arg := range f.Args[1:] { + ref, ok := arg.(*VarRef) + if ok { + columnFields = append(columnFields, &Field{Expr: ref}) + } + } + } + } + } + + // Determine if we should add an extra column for an implicit time. + offset := 0 + if !s.OmitTime { + offset++ + } + + columnNames := make([]string, len(columnFields)+offset) + if !s.OmitTime { + // Add the implicit time if requested. + columnNames[0] = s.TimeFieldName() + } + + // Keep track of the encountered column names. + names := make(map[string]int) + + // Resolve aliases first. + for i, col := range columnFields { + if col.Alias != "" { + columnNames[i+offset] = col.Alias + names[col.Alias] = 1 + } + } + + // Resolve any generated names and resolve conflicts. + for i, col := range columnFields { + if columnNames[i+offset] != "" { + continue + } + + name := col.Name() + count, conflict := names[name] + if conflict { + for { + resolvedName := fmt.Sprintf("%s_%d", name, count) + _, conflict = names[resolvedName] + if !conflict { + names[name] = count + 1 + name = resolvedName + break + } + count++ + } + } + names[name]++ + columnNames[i+offset] = name + } + return columnNames +} + +// FieldExprByName returns the expression that matches the field name and the +// index where this was found. If the name matches one of the arguments to +// "top" or "bottom", the variable reference inside of the function is returned +// and the index is of the function call rather than the variable reference. +// If no expression is found, -1 is returned for the index and the expression +// will be nil. +func (s *SelectStatement) FieldExprByName(name string) (int, Expr) { + for i, f := range s.Fields { + if f.Name() == name { + return i, f.Expr + } else if call, ok := f.Expr.(*Call); ok && (call.Name == "top" || call.Name == "bottom") && len(call.Args) > 2 { + for _, arg := range call.Args[1 : len(call.Args)-1] { + if arg, ok := arg.(*VarRef); ok && arg.Val == name { + return i, arg + } + } + } + } + return -1, nil +} + +// Reduce calls the Reduce function on the different components of the +// SelectStatement to reduce the statement. +func (s *SelectStatement) Reduce(valuer Valuer) *SelectStatement { + stmt := s.Clone() + stmt.Condition = Reduce(stmt.Condition, valuer) + for _, d := range stmt.Dimensions { + d.Expr = Reduce(d.Expr, valuer) + } + + for _, source := range stmt.Sources { + switch source := source.(type) { + case *SubQuery: + source.Statement = source.Statement.Reduce(valuer) + } + } + return stmt +} + +// String returns a string representation of the select statement. +func (s *SelectStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SELECT ") + _, _ = buf.WriteString(s.Fields.String()) + + if s.Target != nil { + _, _ = buf.WriteString(" ") + _, _ = buf.WriteString(s.Target.String()) + } + if len(s.Sources) > 0 { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + switch s.Fill { + case NoFill: + _, _ = buf.WriteString(" fill(none)") + case NumberFill: + _, _ = buf.WriteString(fmt.Sprintf(" fill(%v)", s.FillValue)) + case LinearFill: + _, _ = buf.WriteString(" fill(linear)") + case PreviousFill: + _, _ = buf.WriteString(" fill(previous)") + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + if s.SLimit > 0 { + _, _ = fmt.Fprintf(&buf, " SLIMIT %d", s.SLimit) + } + if s.SOffset > 0 { + _, _ = fmt.Fprintf(&buf, " SOFFSET %d", s.SOffset) + } + if s.Location != nil { + _, _ = fmt.Fprintf(&buf, ` TZ('%s')`, s.Location) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute the SelectStatement. +// NOTE: Statement should be normalized first (database name(s) in Sources and +// Target should be populated). If the statement has not been normalized, an +// empty string will be returned for the database name and it is up to the caller +// to interpret that as the default database. +func (s *SelectStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + ep, err := s.Sources.RequiredPrivileges() + if err != nil { + return nil, err + } + + if s.Target != nil { + ep = append(ep, ExecutionPrivilege{Admin: false, Name: s.Target.Measurement.Database, Privilege: WritePrivilege}) + } + return ep, nil +} + +// HasWildcard returns whether or not the select statement has at least 1 wildcard. +func (s *SelectStatement) HasWildcard() bool { + return s.HasFieldWildcard() || s.HasDimensionWildcard() +} + +// HasFieldWildcard returns whether or not the select statement has at least 1 wildcard in the fields. +func (s *SelectStatement) HasFieldWildcard() (hasWildcard bool) { + WalkFunc(s.Fields, func(n Node) { + if hasWildcard { + return + } + switch n.(type) { + case *Wildcard, *RegexLiteral: + hasWildcard = true + } + }) + return hasWildcard +} + +// HasDimensionWildcard returns whether or not the select statement has +// at least 1 wildcard in the dimensions aka `GROUP BY`. +func (s *SelectStatement) HasDimensionWildcard() bool { + for _, d := range s.Dimensions { + switch d.Expr.(type) { + case *Wildcard, *RegexLiteral: + return true + } + } + + return false +} + +// GroupByInterval extracts the time interval, if specified. +func (s *SelectStatement) GroupByInterval() (time.Duration, error) { + // return if we've already pulled it out + if s.groupByInterval != 0 { + return s.groupByInterval, nil + } + + // Ignore if there are no dimensions. + if len(s.Dimensions) == 0 { + return 0, nil + } + + for _, d := range s.Dimensions { + if call, ok := d.Expr.(*Call); ok && call.Name == "time" { + // Make sure there is exactly one argument. + if got := len(call.Args); got < 1 || got > 2 { + return 0, errors.New("time dimension expected 1 or 2 arguments") + } + + // Ensure the argument is a duration. + lit, ok := call.Args[0].(*DurationLiteral) + if !ok { + return 0, errors.New("time dimension must have duration argument") + } + s.groupByInterval = lit.Val + return lit.Val, nil + } + } + return 0, nil +} + +// GroupByOffset extracts the time interval offset, if specified. +func (s *SelectStatement) GroupByOffset() (time.Duration, error) { + interval, err := s.GroupByInterval() + if err != nil { + return 0, err + } + + // Ignore if there are no dimensions. + if len(s.Dimensions) == 0 { + return 0, nil + } + + for _, d := range s.Dimensions { + if call, ok := d.Expr.(*Call); ok && call.Name == "time" { + if len(call.Args) == 2 { + switch expr := call.Args[1].(type) { + case *DurationLiteral: + return expr.Val % interval, nil + case *TimeLiteral: + return expr.Val.Sub(expr.Val.Truncate(interval)), nil + default: + return 0, fmt.Errorf("invalid time dimension offset: %s", expr) + } + } + return 0, nil + } + } + return 0, nil +} + +// SetTimeRange sets the start and end time of the select statement to [start, end). i.e. start inclusive, end exclusive. +// This is used commonly for continuous queries so the start and end are in buckets. +func (s *SelectStatement) SetTimeRange(start, end time.Time) error { + cond := fmt.Sprintf("time >= '%s' AND time < '%s'", start.UTC().Format(time.RFC3339Nano), end.UTC().Format(time.RFC3339Nano)) + if s.Condition != nil { + cond = fmt.Sprintf("%s AND %s", s.rewriteWithoutTimeDimensions(), cond) + } + + expr, err := NewParser(strings.NewReader(cond)).ParseExpr() + if err != nil { + return err + } + + // Fold out any previously replaced time dimensions and set the condition. + s.Condition = Reduce(expr, nil) + + return nil +} + +// rewriteWithoutTimeDimensions will remove any WHERE time... clauses from the select statement. +// This is necessary when setting an explicit time range to override any that previously existed. +func (s *SelectStatement) rewriteWithoutTimeDimensions() string { + n := RewriteFunc(s.Condition, func(n Node) Node { + switch n := n.(type) { + case *BinaryExpr: + if n.LHS.String() == "time" { + return &BooleanLiteral{Val: true} + } + return n + case *Call: + return &BooleanLiteral{Val: true} + default: + return n + } + }) + + return n.String() +} + +func encodeMeasurement(mm *Measurement) *internal.Measurement { + pb := &internal.Measurement{ + Database: proto.String(mm.Database), + RetentionPolicy: proto.String(mm.RetentionPolicy), + Name: proto.String(mm.Name), + IsTarget: proto.Bool(mm.IsTarget), + } + if mm.Regex != nil { + pb.Regex = proto.String(mm.Regex.Val.String()) + } + return pb +} + +func decodeMeasurement(pb *internal.Measurement) (*Measurement, error) { + mm := &Measurement{ + Database: pb.GetDatabase(), + RetentionPolicy: pb.GetRetentionPolicy(), + Name: pb.GetName(), + IsTarget: pb.GetIsTarget(), + } + + if pb.Regex != nil { + regex, err := regexp.Compile(pb.GetRegex()) + if err != nil { + return nil, fmt.Errorf("invalid binary measurement regex: value=%q, err=%s", pb.GetRegex(), err) + } + mm.Regex = &RegexLiteral{Val: regex} + } + + return mm, nil +} + +// walkNames will walk the Expr and return the identifier names used. +func walkNames(exp Expr) []string { + switch expr := exp.(type) { + case *VarRef: + return []string{expr.Val} + case *Call: + var a []string + for _, expr := range expr.Args { + if ref, ok := expr.(*VarRef); ok { + a = append(a, ref.Val) + } + } + return a + case *BinaryExpr: + var ret []string + ret = append(ret, walkNames(expr.LHS)...) + ret = append(ret, walkNames(expr.RHS)...) + return ret + case *ParenExpr: + return walkNames(expr.Expr) + } + + return nil +} + +// walkRefs will walk the Expr and return the var refs used. +func walkRefs(exp Expr) []VarRef { + refs := make(map[VarRef]struct{}) + var walk func(exp Expr) + walk = func(exp Expr) { + switch expr := exp.(type) { + case *VarRef: + refs[*expr] = struct{}{} + case *Call: + for _, expr := range expr.Args { + if ref, ok := expr.(*VarRef); ok { + refs[*ref] = struct{}{} + } + } + case *BinaryExpr: + walk(expr.LHS) + walk(expr.RHS) + case *ParenExpr: + walk(expr.Expr) + } + } + walk(exp) + + // Turn the map into a slice. + a := make([]VarRef, 0, len(refs)) + for ref := range refs { + a = append(a, ref) + } + return a +} + +// ExprNames returns a list of non-"time" field names from an expression. +func ExprNames(expr Expr) []VarRef { + m := make(map[VarRef]struct{}) + for _, ref := range walkRefs(expr) { + if ref.Val == "time" { + continue + } + m[ref] = struct{}{} + } + + a := make([]VarRef, 0, len(m)) + for k := range m { + a = append(a, k) + } + sort.Sort(VarRefs(a)) + + return a +} + +// Target represents a target (destination) policy, measurement, and DB. +type Target struct { + // Measurement to write into. + Measurement *Measurement +} + +// String returns a string representation of the Target. +func (t *Target) String() string { + if t == nil { + return "" + } + + var buf bytes.Buffer + _, _ = buf.WriteString("INTO ") + _, _ = buf.WriteString(t.Measurement.String()) + if t.Measurement.Name == "" { + _, _ = buf.WriteString(":MEASUREMENT") + } + + return buf.String() +} + +// ExplainStatement represents a command for explaining a select statement. +type ExplainStatement struct { + Statement *SelectStatement + + Analyze bool +} + +// String returns a string representation of the explain statement. +func (e *ExplainStatement) String() string { + var buf bytes.Buffer + buf.WriteString("EXPLAIN ") + if e.Analyze { + buf.WriteString("ANALYZE ") + } + buf.WriteString(e.Statement.String()) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ExplainStatement. +func (e *ExplainStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return e.Statement.RequiredPrivileges() +} + +// DeleteStatement represents a command for deleting data from the database. +type DeleteStatement struct { + // Data source that values are removed from. + Source Source + + // An expression evaluated on data point. + Condition Expr +} + +// String returns a string representation of the delete statement. +func (s *DeleteStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("DELETE FROM ") + _, _ = buf.WriteString(s.Source.String()) + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a DeleteStatement. +func (s *DeleteStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: WritePrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *DeleteStatement) DefaultDatabase() string { + if m, ok := s.Source.(*Measurement); ok { + return m.Database + } + return "" +} + +// ShowSeriesStatement represents a command for listing series in the database. +type ShowSeriesStatement struct { + // Database to query. If blank, use the default database. + // The database can also be specified per source in the Sources. + Database string + + // Measurement(s) the series are listed for. + Sources Sources + + // An expression evaluated on a series name or tag. + Condition Expr + + // Fields to sort results by + SortFields SortFields + + // Maximum number of rows to be returned. + // Unlimited if zero. + Limit int + + // Returns rows starting at an offset from the first row. + Offset int +} + +// String returns a string representation of the list series statement. +func (s *ShowSeriesStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW SERIES") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = buf.WriteString(" LIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.Limit)) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowSeriesStatement. +func (s *ShowSeriesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowSeriesStatement) DefaultDatabase() string { + return s.Database +} + +// DropSeriesStatement represents a command for removing a series from the database. +type DropSeriesStatement struct { + // Data source that fields are extracted from (optional) + Sources Sources + + // An expression evaluated on data point (optional) + Condition Expr +} + +// String returns a string representation of the drop series statement. +func (s *DropSeriesStatement) String() string { + var buf bytes.Buffer + buf.WriteString("DROP SERIES") + + if s.Sources != nil { + buf.WriteString(" FROM ") + buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + buf.WriteString(" WHERE ") + buf.WriteString(s.Condition.String()) + } + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a DropSeriesStatement. +func (s DropSeriesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: WritePrivilege}}, nil +} + +// DeleteSeriesStatement represents a command for deleting all or part of a series from a database. +type DeleteSeriesStatement struct { + // Data source that fields are extracted from (optional) + Sources Sources + + // An expression evaluated on data point (optional) + Condition Expr +} + +// String returns a string representation of the delete series statement. +func (s *DeleteSeriesStatement) String() string { + var buf bytes.Buffer + buf.WriteString("DELETE") + + if s.Sources != nil { + buf.WriteString(" FROM ") + buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + buf.WriteString(" WHERE ") + buf.WriteString(s.Condition.String()) + } + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a DeleteSeriesStatement. +func (s DeleteSeriesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: WritePrivilege}}, nil +} + +// DropShardStatement represents a command for removing a shard from +// the node. +type DropShardStatement struct { + // ID of the shard to be dropped. + ID uint64 +} + +// String returns a string representation of the drop series statement. +func (s *DropShardStatement) String() string { + var buf bytes.Buffer + buf.WriteString("DROP SHARD ") + buf.WriteString(strconv.FormatUint(s.ID, 10)) + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a +// DropShardStatement. +func (s *DropShardStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowSeriesCardinalityStatement represents a command for listing series cardinality. +type ShowSeriesCardinalityStatement struct { + // Database to query. If blank, use the default database. + // The database can also be specified per source in the Sources. + Database string + + // Specifies whether the user requires exact counting or not. + Exact bool + + // Measurement(s) the series are listed for. + Sources Sources + + // An expression evaluated on a series name or tag. + Condition Expr + + // Expressions used for grouping the selection. + Dimensions Dimensions + + Limit, Offset int +} + +// String returns a string representation of the show continuous queries statement. +func (s *ShowSeriesCardinalityStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW SERIES") + + if s.Exact { + _, _ = buf.WriteString(" EXACT") + } + _, _ = buf.WriteString(" CARDINALITY") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowSeriesCardinalityStatement. +func (s *ShowSeriesCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + if !s.Exact { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil + } + return s.Sources.RequiredPrivileges() +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowSeriesCardinalityStatement) DefaultDatabase() string { + return s.Database +} + +// ShowContinuousQueriesStatement represents a command for listing continuous queries. +type ShowContinuousQueriesStatement struct{} + +// String returns a string representation of the show continuous queries statement. +func (s *ShowContinuousQueriesStatement) String() string { return "SHOW CONTINUOUS QUERIES" } + +// RequiredPrivileges returns the privilege required to execute a ShowContinuousQueriesStatement. +func (s *ShowContinuousQueriesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: ReadPrivilege}}, nil +} + +// ShowGrantsForUserStatement represents a command for listing user privileges. +type ShowGrantsForUserStatement struct { + // Name of the user to display privileges. + Name string +} + +// String returns a string representation of the show grants for user. +func (s *ShowGrantsForUserStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW GRANTS FOR ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowGrantsForUserStatement +func (s *ShowGrantsForUserStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowDatabasesStatement represents a command for listing all databases in the cluster. +type ShowDatabasesStatement struct{} + +// String returns a string representation of the show databases command. +func (s *ShowDatabasesStatement) String() string { return "SHOW DATABASES" } + +// RequiredPrivileges returns the privilege required to execute a ShowDatabasesStatement. +func (s *ShowDatabasesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + // SHOW DATABASES is one of few statements that have no required privileges. + // Anyone is allowed to execute it, but the returned results depend on the user's + // individual database permissions. + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: NoPrivileges}}, nil +} + +// CreateContinuousQueryStatement represents a command for creating a continuous query. +type CreateContinuousQueryStatement struct { + // Name of the continuous query to be created. + Name string + + // Name of the database to create the continuous query on. + Database string + + // Source of data (SELECT statement). + Source *SelectStatement + + // Interval to resample previous queries. + ResampleEvery time.Duration + + // Maximum duration to resample previous queries. + ResampleFor time.Duration +} + +// String returns a string representation of the statement. +func (s *CreateContinuousQueryStatement) String() string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "CREATE CONTINUOUS QUERY %s ON %s ", QuoteIdent(s.Name), QuoteIdent(s.Database)) + + if s.ResampleEvery > 0 || s.ResampleFor > 0 { + buf.WriteString("RESAMPLE ") + if s.ResampleEvery > 0 { + fmt.Fprintf(&buf, "EVERY %s ", FormatDuration(s.ResampleEvery)) + } + if s.ResampleFor > 0 { + fmt.Fprintf(&buf, "FOR %s ", FormatDuration(s.ResampleFor)) + } + } + fmt.Fprintf(&buf, "BEGIN %s END", s.Source.String()) + return buf.String() +} + +// DefaultDatabase returns the default database from the statement. +func (s *CreateContinuousQueryStatement) DefaultDatabase() string { + return s.Database +} + +// RequiredPrivileges returns the privilege required to execute a CreateContinuousQueryStatement. +func (s *CreateContinuousQueryStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + ep := ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}} + + // Selecting into a database that's different from the source? + if s.Source.Target.Measurement.Database != "" { + // Change source database privilege requirement to read. + ep[0].Privilege = ReadPrivilege + + // Add destination database privilege requirement and set it to write. + p := ExecutionPrivilege{ + Admin: false, + Name: s.Source.Target.Measurement.Database, + Privilege: WritePrivilege, + } + ep = append(ep, p) + } + + return ep, nil +} + +func (s *CreateContinuousQueryStatement) validate() error { + interval, err := s.Source.GroupByInterval() + if err != nil { + return err + } + + if s.ResampleFor != 0 { + if s.ResampleEvery != 0 && s.ResampleEvery > interval { + interval = s.ResampleEvery + } + if interval > s.ResampleFor { + return fmt.Errorf("FOR duration must be >= GROUP BY time duration: must be a minimum of %s, got %s", FormatDuration(interval), FormatDuration(s.ResampleFor)) + } + } + return nil +} + +// DropContinuousQueryStatement represents a command for removing a continuous query. +type DropContinuousQueryStatement struct { + Name string + Database string +} + +// String returns a string representation of the statement. +func (s *DropContinuousQueryStatement) String() string { + return fmt.Sprintf("DROP CONTINUOUS QUERY %s ON %s", QuoteIdent(s.Name), QuoteIdent(s.Database)) +} + +// RequiredPrivileges returns the privilege(s) required to execute a DropContinuousQueryStatement +func (s *DropContinuousQueryStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: WritePrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *DropContinuousQueryStatement) DefaultDatabase() string { + return s.Database +} + +// ShowMeasurementCardinalityStatement represents a command for listing measurement cardinality. +type ShowMeasurementCardinalityStatement struct { + Exact bool // If false then cardinality estimation will be used. + Database string + Sources Sources + Condition Expr + Dimensions Dimensions + Limit, Offset int +} + +// String returns a string representation of the statement. +func (s *ShowMeasurementCardinalityStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW MEASUREMENT") + + if s.Exact { + _, _ = buf.WriteString(" EXACT") + } + _, _ = buf.WriteString(" CARDINALITY") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowMeasurementCardinalityStatement. +func (s *ShowMeasurementCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + if !s.Exact { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil + } + return s.Sources.RequiredPrivileges() +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowMeasurementCardinalityStatement) DefaultDatabase() string { + return s.Database +} + +// ShowMeasurementsStatement represents a command for listing measurements. +type ShowMeasurementsStatement struct { + // Database to query. If blank, use the default database. + Database string + + // Measurement name or regex. + Source Source + + // An expression evaluated on data point. + Condition Expr + + // Fields to sort results by + SortFields SortFields + + // Maximum number of rows to be returned. + // Unlimited if zero. + Limit int + + // Returns rows starting at an offset from the first row. + Offset int +} + +// String returns a string representation of the statement. +func (s *ShowMeasurementsStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW MEASUREMENTS") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(s.Database) + } + if s.Source != nil { + _, _ = buf.WriteString(" WITH MEASUREMENT ") + if m, ok := s.Source.(*Measurement); ok && m.Regex != nil { + _, _ = buf.WriteString("=~ ") + } else { + _, _ = buf.WriteString("= ") + } + _, _ = buf.WriteString(s.Source.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = buf.WriteString(" LIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.Limit)) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowMeasurementsStatement. +func (s *ShowMeasurementsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowMeasurementsStatement) DefaultDatabase() string { + return s.Database +} + +// DropMeasurementStatement represents a command to drop a measurement. +type DropMeasurementStatement struct { + // Name of the measurement to be dropped. + Name string +} + +// String returns a string representation of the drop measurement statement. +func (s *DropMeasurementStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("DROP MEASUREMENT ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a DropMeasurementStatement +func (s *DropMeasurementStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowQueriesStatement represents a command for listing all running queries. +type ShowQueriesStatement struct{} + +// String returns a string representation of the show queries statement. +func (s *ShowQueriesStatement) String() string { + return "SHOW QUERIES" +} + +// RequiredPrivileges returns the privilege required to execute a ShowQueriesStatement. +func (s *ShowQueriesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: "", Privilege: ReadPrivilege}}, nil +} + +// ShowRetentionPoliciesStatement represents a command for listing retention policies. +type ShowRetentionPoliciesStatement struct { + // Name of the database to list policies for. + Database string +} + +// String returns a string representation of a ShowRetentionPoliciesStatement. +func (s *ShowRetentionPoliciesStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW RETENTION POLICIES") + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowRetentionPoliciesStatement +func (s *ShowRetentionPoliciesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowRetentionPoliciesStatement) DefaultDatabase() string { + return s.Database +} + +// ShowStatsStatement displays statistics for a given module. +type ShowStatsStatement struct { + Module string +} + +// String returns a string representation of a ShowStatsStatement. +func (s *ShowStatsStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW STATS") + if s.Module != "" { + _, _ = buf.WriteString(" FOR ") + _, _ = buf.WriteString(QuoteString(s.Module)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowStatsStatement +func (s *ShowStatsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowShardGroupsStatement represents a command for displaying shard groups in the cluster. +type ShowShardGroupsStatement struct{} + +// String returns a string representation of the SHOW SHARD GROUPS command. +func (s *ShowShardGroupsStatement) String() string { return "SHOW SHARD GROUPS" } + +// RequiredPrivileges returns the privileges required to execute the statement. +func (s *ShowShardGroupsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowShardsStatement represents a command for displaying shards in the cluster. +type ShowShardsStatement struct{} + +// String returns a string representation. +func (s *ShowShardsStatement) String() string { return "SHOW SHARDS" } + +// RequiredPrivileges returns the privileges required to execute the statement. +func (s *ShowShardsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowDiagnosticsStatement represents a command for show node diagnostics. +type ShowDiagnosticsStatement struct { + // Module + Module string +} + +// String returns a string representation of the ShowDiagnosticsStatement. +func (s *ShowDiagnosticsStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW DIAGNOSTICS") + if s.Module != "" { + _, _ = buf.WriteString(" FOR ") + _, _ = buf.WriteString(QuoteString(s.Module)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowDiagnosticsStatement +func (s *ShowDiagnosticsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// CreateSubscriptionStatement represents a command to add a subscription to the incoming data stream. +type CreateSubscriptionStatement struct { + Name string + Database string + RetentionPolicy string + Destinations []string + Mode string +} + +// String returns a string representation of the CreateSubscriptionStatement. +func (s *CreateSubscriptionStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("CREATE SUBSCRIPTION ") + _, _ = buf.WriteString(QuoteIdent(s.Name)) + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + _, _ = buf.WriteString(".") + _, _ = buf.WriteString(QuoteIdent(s.RetentionPolicy)) + _, _ = buf.WriteString(" DESTINATIONS ") + _, _ = buf.WriteString(s.Mode) + _, _ = buf.WriteString(" ") + for i, dest := range s.Destinations { + if i != 0 { + _, _ = buf.WriteString(", ") + } + _, _ = buf.WriteString(QuoteString(dest)) + } + + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a CreateSubscriptionStatement. +func (s *CreateSubscriptionStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *CreateSubscriptionStatement) DefaultDatabase() string { + return s.Database +} + +// DropSubscriptionStatement represents a command to drop a subscription to the incoming data stream. +type DropSubscriptionStatement struct { + Name string + Database string + RetentionPolicy string +} + +// String returns a string representation of the DropSubscriptionStatement. +func (s *DropSubscriptionStatement) String() string { + return fmt.Sprintf(`DROP SUBSCRIPTION %s ON %s.%s`, QuoteIdent(s.Name), QuoteIdent(s.Database), QuoteIdent(s.RetentionPolicy)) +} + +// RequiredPrivileges returns the privilege required to execute a DropSubscriptionStatement +func (s *DropSubscriptionStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *DropSubscriptionStatement) DefaultDatabase() string { + return s.Database +} + +// ShowSubscriptionsStatement represents a command to show a list of subscriptions. +type ShowSubscriptionsStatement struct { +} + +// String returns a string representation of the ShowSubscriptionsStatement. +func (s *ShowSubscriptionsStatement) String() string { + return "SHOW SUBSCRIPTIONS" +} + +// RequiredPrivileges returns the privilege required to execute a ShowSubscriptionsStatement. +func (s *ShowSubscriptionsStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowTagKeysStatement represents a command for listing tag keys. +type ShowTagKeysStatement struct { + // Database to query. If blank, use the default database. + // The database can also be specified per source in the Sources. + Database string + + // Data sources that fields are extracted from. + Sources Sources + + // An expression evaluated on data point. + Condition Expr + + // Fields to sort results by. + SortFields SortFields + + // Maximum number of tag keys per measurement. Unlimited if zero. + Limit int + + // Returns tag keys starting at an offset from the first row. + Offset int + + // Maxiumum number of series to be returned. Unlimited if zero. + SLimit int + + // Returns series starting at an offset from the first one. + SOffset int +} + +// String returns a string representation of the statement. +func (s *ShowTagKeysStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW TAG KEYS") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = buf.WriteString(" LIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.Limit)) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + if s.SLimit > 0 { + _, _ = buf.WriteString(" SLIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.SLimit)) + } + if s.SOffset > 0 { + _, _ = buf.WriteString(" SOFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.SOffset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowTagKeysStatement. +func (s *ShowTagKeysStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowTagKeysStatement) DefaultDatabase() string { + return s.Database +} + +// ShowTagKeyCardinalityStatement represents a command for listing tag key cardinality. +type ShowTagKeyCardinalityStatement struct { + Database string + Exact bool + Sources Sources + Condition Expr + Dimensions Dimensions + Limit, Offset int +} + +// String returns a string representation of the statement. +func (s *ShowTagKeyCardinalityStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW TAG KEY ") + if s.Exact { + _, _ = buf.WriteString("EXACT ") + } + _, _ = buf.WriteString("CARDINALITY") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowTagKeyCardinalityStatement. +func (s *ShowTagKeyCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return s.Sources.RequiredPrivileges() +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowTagKeyCardinalityStatement) DefaultDatabase() string { + return s.Database +} + +// ShowTagValuesStatement represents a command for listing tag values. +type ShowTagValuesStatement struct { + // Database to query. If blank, use the default database. + // The database can also be specified per source in the Sources. + Database string + + // Data source that fields are extracted from. + Sources Sources + + // Operation to use when selecting tag key(s). + Op Token + + // Literal to compare the tag key(s) with. + TagKeyExpr Literal + + // An expression evaluated on data point. + Condition Expr + + // Fields to sort results by. + SortFields SortFields + + // Maximum number of rows to be returned. + // Unlimited if zero. + Limit int + + // Returns rows starting at an offset from the first row. + Offset int +} + +// String returns a string representation of the statement. +func (s *ShowTagValuesStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW TAG VALUES") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + _, _ = buf.WriteString(" WITH KEY ") + _, _ = buf.WriteString(s.Op.String()) + _, _ = buf.WriteString(" ") + if lit, ok := s.TagKeyExpr.(*StringLiteral); ok { + _, _ = buf.WriteString(QuoteIdent(lit.Val)) + } else { + _, _ = buf.WriteString(s.TagKeyExpr.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = buf.WriteString(" LIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.Limit)) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowTagValuesStatement. +func (s *ShowTagValuesStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowTagValuesStatement) DefaultDatabase() string { + return s.Database +} + +// ShowTagValuesCardinalityStatement represents a command for listing tag value cardinality. +type ShowTagValuesCardinalityStatement struct { + Database string + Exact bool + Sources Sources + Op Token + TagKeyExpr Literal + Condition Expr + Dimensions Dimensions + Limit, Offset int +} + +// String returns a string representation of the statement. +func (s *ShowTagValuesCardinalityStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW TAG VALUES ") + if s.Exact { + _, _ = buf.WriteString("EXACT ") + } + _, _ = buf.WriteString("CARDINALITY") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + _, _ = buf.WriteString(" WITH KEY ") + _, _ = buf.WriteString(s.Op.String()) + _, _ = buf.WriteString(" ") + if lit, ok := s.TagKeyExpr.(*StringLiteral); ok { + _, _ = buf.WriteString(QuoteIdent(lit.Val)) + } else { + _, _ = buf.WriteString(s.TagKeyExpr.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowTagValuesCardinalityStatement. +func (s *ShowTagValuesCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return s.Sources.RequiredPrivileges() +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowTagValuesCardinalityStatement) DefaultDatabase() string { + return s.Database +} + +// ShowUsersStatement represents a command for listing users. +type ShowUsersStatement struct{} + +// String returns a string representation of the ShowUsersStatement. +func (s *ShowUsersStatement) String() string { + return "SHOW USERS" +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowUsersStatement +func (s *ShowUsersStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: true, Name: "", Privilege: AllPrivileges}}, nil +} + +// ShowFieldKeyCardinalityStatement represents a command for listing field key cardinality. +type ShowFieldKeyCardinalityStatement struct { + Database string + Exact bool + Sources Sources + Condition Expr + Dimensions Dimensions + Limit, Offset int +} + +// String returns a string representation of the statement. +func (s *ShowFieldKeyCardinalityStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW FIELD KEY ") + + if s.Exact { + _, _ = buf.WriteString("EXACT ") + } + _, _ = buf.WriteString("CARDINALITY") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if s.Condition != nil { + _, _ = buf.WriteString(" WHERE ") + _, _ = buf.WriteString(s.Condition.String()) + } + if len(s.Dimensions) > 0 { + _, _ = buf.WriteString(" GROUP BY ") + _, _ = buf.WriteString(s.Dimensions.String()) + } + if s.Limit > 0 { + _, _ = fmt.Fprintf(&buf, " LIMIT %d", s.Limit) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege required to execute a ShowFieldKeyCardinalityStatement. +func (s *ShowFieldKeyCardinalityStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return s.Sources.RequiredPrivileges() +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowFieldKeyCardinalityStatement) DefaultDatabase() string { + return s.Database +} + +// ShowFieldKeysStatement represents a command for listing field keys. +type ShowFieldKeysStatement struct { + // Database to query. If blank, use the default database. + // The database can also be specified per source in the Sources. + Database string + + // Data sources that fields are extracted from. + Sources Sources + + // Fields to sort results by + SortFields SortFields + + // Maximum number of rows to be returned. + // Unlimited if zero. + Limit int + + // Returns rows starting at an offset from the first row. + Offset int +} + +// String returns a string representation of the statement. +func (s *ShowFieldKeysStatement) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("SHOW FIELD KEYS") + + if s.Database != "" { + _, _ = buf.WriteString(" ON ") + _, _ = buf.WriteString(QuoteIdent(s.Database)) + } + if s.Sources != nil { + _, _ = buf.WriteString(" FROM ") + _, _ = buf.WriteString(s.Sources.String()) + } + if len(s.SortFields) > 0 { + _, _ = buf.WriteString(" ORDER BY ") + _, _ = buf.WriteString(s.SortFields.String()) + } + if s.Limit > 0 { + _, _ = buf.WriteString(" LIMIT ") + _, _ = buf.WriteString(strconv.Itoa(s.Limit)) + } + if s.Offset > 0 { + _, _ = buf.WriteString(" OFFSET ") + _, _ = buf.WriteString(strconv.Itoa(s.Offset)) + } + return buf.String() +} + +// RequiredPrivileges returns the privilege(s) required to execute a ShowFieldKeysStatement. +func (s *ShowFieldKeysStatement) RequiredPrivileges() (ExecutionPrivileges, error) { + return ExecutionPrivileges{{Admin: false, Name: s.Database, Privilege: ReadPrivilege}}, nil +} + +// DefaultDatabase returns the default database from the statement. +func (s *ShowFieldKeysStatement) DefaultDatabase() string { + return s.Database +} + +// Fields represents a list of fields. +type Fields []*Field + +// AliasNames returns a list of calculated field names in +// order of alias, function name, then field. +func (a Fields) AliasNames() []string { + names := []string{} + for _, f := range a { + names = append(names, f.Name()) + } + return names +} + +// Names returns a list of field names. +func (a Fields) Names() []string { + names := []string{} + for _, f := range a { + switch expr := f.Expr.(type) { + case *Call: + names = append(names, expr.Name) + case *VarRef: + names = append(names, expr.Val) + case *BinaryExpr: + names = append(names, walkNames(expr)...) + case *ParenExpr: + names = append(names, walkNames(expr)...) + } + } + return names +} + +// String returns a string representation of the fields. +func (a Fields) String() string { + var str []string + for _, f := range a { + str = append(str, f.String()) + } + return strings.Join(str, ", ") +} + +// Field represents an expression retrieved from a select statement. +type Field struct { + Expr Expr + Alias string +} + +// Name returns the name of the field. Returns alias, if set. +// Otherwise uses the function name or variable name. +func (f *Field) Name() string { + // Return alias, if set. + if f.Alias != "" { + return f.Alias + } + + // Return the function name or variable name, if available. + switch expr := f.Expr.(type) { + case *Call: + return expr.Name + case *BinaryExpr: + return BinaryExprName(expr) + case *ParenExpr: + f := Field{Expr: expr.Expr} + return f.Name() + case *VarRef: + return expr.Val + } + + // Otherwise return a blank name. + return "" +} + +// String returns a string representation of the field. +func (f *Field) String() string { + str := f.Expr.String() + + if f.Alias == "" { + return str + } + return fmt.Sprintf("%s AS %s", str, QuoteIdent(f.Alias)) +} + +// Len implements sort.Interface. +func (a Fields) Len() int { return len(a) } + +// Less implements sort.Interface. +func (a Fields) Less(i, j int) bool { return a[i].Name() < a[j].Name() } + +// Swap implements sort.Interface. +func (a Fields) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// Dimensions represents a list of dimensions. +type Dimensions []*Dimension + +// String returns a string representation of the dimensions. +func (a Dimensions) String() string { + var str []string + for _, d := range a { + str = append(str, d.String()) + } + return strings.Join(str, ", ") +} + +// Normalize returns the interval and tag dimensions separately. +// Returns 0 if no time interval is specified. +func (a Dimensions) Normalize() (time.Duration, []string) { + var dur time.Duration + var tags []string + + for _, dim := range a { + switch expr := dim.Expr.(type) { + case *Call: + lit, _ := expr.Args[0].(*DurationLiteral) + dur = lit.Val + case *VarRef: + tags = append(tags, expr.Val) + } + } + + return dur, tags +} + +// Dimension represents an expression that a select statement is grouped by. +type Dimension struct { + Expr Expr +} + +// String returns a string representation of the dimension. +func (d *Dimension) String() string { return d.Expr.String() } + +// Measurements represents a list of measurements. +type Measurements []*Measurement + +// String returns a string representation of the measurements. +func (a Measurements) String() string { + var str []string + for _, m := range a { + str = append(str, m.String()) + } + return strings.Join(str, ", ") +} + +// Measurement represents a single measurement used as a datasource. +type Measurement struct { + Database string + RetentionPolicy string + Name string + Regex *RegexLiteral + IsTarget bool + + // This field indicates that the measurement should read be read from the + // specified system iterator. + SystemIterator string +} + +// Clone returns a deep clone of the Measurement. +func (m *Measurement) Clone() *Measurement { + var regexp *RegexLiteral + if m.Regex != nil && m.Regex.Val != nil { + regexp = &RegexLiteral{Val: m.Regex.Val.Copy()} + } + return &Measurement{ + Database: m.Database, + RetentionPolicy: m.RetentionPolicy, + Name: m.Name, + Regex: regexp, + IsTarget: m.IsTarget, + SystemIterator: m.SystemIterator, + } +} + +// String returns a string representation of the measurement. +func (m *Measurement) String() string { + var buf bytes.Buffer + if m.Database != "" { + _, _ = buf.WriteString(QuoteIdent(m.Database)) + _, _ = buf.WriteString(".") + } + + if m.RetentionPolicy != "" { + _, _ = buf.WriteString(QuoteIdent(m.RetentionPolicy)) + } + + if m.Database != "" || m.RetentionPolicy != "" { + _, _ = buf.WriteString(`.`) + } + + if m.Name != "" && m.SystemIterator == "" { + _, _ = buf.WriteString(QuoteIdent(m.Name)) + } else if m.SystemIterator != "" { + _, _ = buf.WriteString(QuoteIdent(m.SystemIterator)) + } else if m.Regex != nil { + _, _ = buf.WriteString(m.Regex.String()) + } + + return buf.String() +} + +// SubQuery is a source with a SelectStatement as the backing store. +type SubQuery struct { + Statement *SelectStatement +} + +// String returns a string representation of the subquery. +func (s *SubQuery) String() string { + return fmt.Sprintf("(%s)", s.Statement.String()) +} + +// VarRef represents a reference to a variable. +type VarRef struct { + Val string + Type DataType +} + +// String returns a string representation of the variable reference. +func (r *VarRef) String() string { + buf := bytes.NewBufferString(QuoteIdent(r.Val)) + if r.Type != Unknown { + buf.WriteString("::") + buf.WriteString(r.Type.String()) + } + return buf.String() +} + +// VarRefs represents a slice of VarRef types. +type VarRefs []VarRef + +// Len implements sort.Interface. +func (a VarRefs) Len() int { return len(a) } + +// Less implements sort.Interface. +func (a VarRefs) Less(i, j int) bool { + if a[i].Val != a[j].Val { + return a[i].Val < a[j].Val + } + return a[i].Type < a[j].Type +} + +// Swap implements sort.Interface. +func (a VarRefs) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// Strings returns a slice of the variable names. +func (a VarRefs) Strings() []string { + s := make([]string, len(a)) + for i, ref := range a { + s[i] = ref.Val + } + return s +} + +// Call represents a function call. +type Call struct { + Name string + Args []Expr +} + +// String returns a string representation of the call. +func (c *Call) String() string { + // Join arguments. + var str []string + for _, arg := range c.Args { + str = append(str, arg.String()) + } + + // Write function name and args. + return fmt.Sprintf("%s(%s)", c.Name, strings.Join(str, ", ")) +} + +// Distinct represents a DISTINCT expression. +type Distinct struct { + // Identifier following DISTINCT + Val string +} + +// String returns a string representation of the expression. +func (d *Distinct) String() string { + return fmt.Sprintf("DISTINCT %s", d.Val) +} + +// NewCall returns a new call expression from this expressions. +func (d *Distinct) NewCall() *Call { + return &Call{ + Name: "distinct", + Args: []Expr{ + &VarRef{Val: d.Val}, + }, + } +} + +// NumberLiteral represents a numeric literal. +type NumberLiteral struct { + Val float64 +} + +// String returns a string representation of the literal. +func (l *NumberLiteral) String() string { return strconv.FormatFloat(l.Val, 'f', 3, 64) } + +// IntegerLiteral represents an integer literal. +type IntegerLiteral struct { + Val int64 +} + +// String returns a string representation of the literal. +func (l *IntegerLiteral) String() string { return fmt.Sprintf("%d", l.Val) } + +// UnsignedLiteral represents an unsigned literal. The parser will only use an unsigned literal if the parsed +// integer is greater than math.MaxInt64. +type UnsignedLiteral struct { + Val uint64 +} + +// String returns a string representation of the literal. +func (l *UnsignedLiteral) String() string { return strconv.FormatUint(l.Val, 10) } + +// BooleanLiteral represents a boolean literal. +type BooleanLiteral struct { + Val bool +} + +// String returns a string representation of the literal. +func (l *BooleanLiteral) String() string { + if l.Val { + return "true" + } + return "false" +} + +// isTrueLiteral returns true if the expression is a literal "true" value. +func isTrueLiteral(expr Expr) bool { + if expr, ok := expr.(*BooleanLiteral); ok { + return expr.Val == true + } + return false +} + +// isFalseLiteral returns true if the expression is a literal "false" value. +func isFalseLiteral(expr Expr) bool { + if expr, ok := expr.(*BooleanLiteral); ok { + return expr.Val == false + } + return false +} + +// ListLiteral represents a list of tag key literals. +type ListLiteral struct { + Vals []string +} + +// String returns a string representation of the literal. +func (s *ListLiteral) String() string { + var buf bytes.Buffer + _, _ = buf.WriteString("(") + for idx, tagKey := range s.Vals { + if idx != 0 { + _, _ = buf.WriteString(", ") + } + _, _ = buf.WriteString(QuoteIdent(tagKey)) + } + _, _ = buf.WriteString(")") + return buf.String() +} + +// StringLiteral represents a string literal. +type StringLiteral struct { + Val string +} + +// String returns a string representation of the literal. +func (l *StringLiteral) String() string { return QuoteString(l.Val) } + +// IsTimeLiteral returns if this string can be interpreted as a time literal. +func (l *StringLiteral) IsTimeLiteral() bool { + return isDateTimeString(l.Val) || isDateString(l.Val) +} + +// ToTimeLiteral returns a time literal if this string can be converted to a time literal. +func (l *StringLiteral) ToTimeLiteral(loc *time.Location) (*TimeLiteral, error) { + if loc == nil { + loc = time.UTC + } + + if isDateTimeString(l.Val) { + t, err := time.ParseInLocation(DateTimeFormat, l.Val, loc) + if err != nil { + // try to parse it as an RFCNano time + t, err = time.ParseInLocation(time.RFC3339Nano, l.Val, loc) + if err != nil { + return nil, ErrInvalidTime + } + } + return &TimeLiteral{Val: t}, nil + } else if isDateString(l.Val) { + t, err := time.ParseInLocation(DateFormat, l.Val, loc) + if err != nil { + return nil, ErrInvalidTime + } + return &TimeLiteral{Val: t}, nil + } + return nil, ErrInvalidTime +} + +// TimeLiteral represents a point-in-time literal. +type TimeLiteral struct { + Val time.Time +} + +// String returns a string representation of the literal. +func (l *TimeLiteral) String() string { + return `'` + l.Val.UTC().Format(time.RFC3339Nano) + `'` +} + +// DurationLiteral represents a duration literal. +type DurationLiteral struct { + Val time.Duration +} + +// String returns a string representation of the literal. +func (l *DurationLiteral) String() string { return FormatDuration(l.Val) } + +// NilLiteral represents a nil literal. +// This is not available to the query language itself. It's only used internally. +type NilLiteral struct{} + +// String returns a string representation of the literal. +func (l *NilLiteral) String() string { return `nil` } + +// BoundParameter represents a bound parameter literal. +// This is not available to the query language itself, but can be used when +// constructing a query string from an AST. +type BoundParameter struct { + Name string +} + +// String returns a string representation of the bound parameter. +func (bp *BoundParameter) String() string { + return fmt.Sprintf("$%s", QuoteIdent(bp.Name)) +} + +// BinaryExpr represents an operation between two expressions. +type BinaryExpr struct { + Op Token + LHS Expr + RHS Expr +} + +// String returns a string representation of the binary expression. +func (e *BinaryExpr) String() string { + return fmt.Sprintf("%s %s %s", e.LHS.String(), e.Op.String(), e.RHS.String()) +} + +// BinaryExprName returns the name of a binary expression by concatenating +// the variables in the binary expression with underscores. +func BinaryExprName(expr *BinaryExpr) string { + v := binaryExprNameVisitor{} + Walk(&v, expr) + return strings.Join(v.names, "_") +} + +type binaryExprNameVisitor struct { + names []string +} + +func (v *binaryExprNameVisitor) Visit(n Node) Visitor { + switch n := n.(type) { + case *VarRef: + v.names = append(v.names, n.Val) + case *Call: + v.names = append(v.names, n.Name) + return nil + } + return v +} + +// ParenExpr represents a parenthesized expression. +type ParenExpr struct { + Expr Expr +} + +// String returns a string representation of the parenthesized expression. +func (e *ParenExpr) String() string { return fmt.Sprintf("(%s)", e.Expr.String()) } + +// RegexLiteral represents a regular expression. +type RegexLiteral struct { + Val *regexp.Regexp +} + +// String returns a string representation of the literal. +func (r *RegexLiteral) String() string { + if r.Val != nil { + return fmt.Sprintf("/%s/", strings.Replace(r.Val.String(), `/`, `\/`, -1)) + } + return "" +} + +// CloneRegexLiteral returns a clone of the RegexLiteral. +func CloneRegexLiteral(r *RegexLiteral) *RegexLiteral { + if r == nil { + return nil + } + + clone := &RegexLiteral{} + if r.Val != nil { + clone.Val = regexp.MustCompile(r.Val.String()) + } + + return clone +} + +// Wildcard represents a wild card expression. +type Wildcard struct { + Type Token +} + +// String returns a string representation of the wildcard. +func (e *Wildcard) String() string { + switch e.Type { + case FIELD: + return "*::field" + case TAG: + return "*::tag" + default: + return "*" + } +} + +// CloneExpr returns a deep copy of the expression. +func CloneExpr(expr Expr) Expr { + if expr == nil { + return nil + } + switch expr := expr.(type) { + case *BinaryExpr: + return &BinaryExpr{Op: expr.Op, LHS: CloneExpr(expr.LHS), RHS: CloneExpr(expr.RHS)} + case *BooleanLiteral: + return &BooleanLiteral{Val: expr.Val} + case *Call: + args := make([]Expr, len(expr.Args)) + for i, arg := range expr.Args { + args[i] = CloneExpr(arg) + } + return &Call{Name: expr.Name, Args: args} + case *Distinct: + return &Distinct{Val: expr.Val} + case *DurationLiteral: + return &DurationLiteral{Val: expr.Val} + case *IntegerLiteral: + return &IntegerLiteral{Val: expr.Val} + case *UnsignedLiteral: + return &UnsignedLiteral{Val: expr.Val} + case *NumberLiteral: + return &NumberLiteral{Val: expr.Val} + case *ParenExpr: + return &ParenExpr{Expr: CloneExpr(expr.Expr)} + case *RegexLiteral: + return &RegexLiteral{Val: expr.Val} + case *StringLiteral: + return &StringLiteral{Val: expr.Val} + case *TimeLiteral: + return &TimeLiteral{Val: expr.Val} + case *VarRef: + return &VarRef{Val: expr.Val, Type: expr.Type} + case *Wildcard: + return &Wildcard{Type: expr.Type} + } + panic("unreachable") +} + +// HasTimeExpr returns true if the expression has a time term. +func HasTimeExpr(expr Expr) bool { + switch n := expr.(type) { + case *BinaryExpr: + if n.Op == AND || n.Op == OR { + return HasTimeExpr(n.LHS) || HasTimeExpr(n.RHS) + } + if ref, ok := n.LHS.(*VarRef); ok && strings.ToLower(ref.Val) == "time" { + return true + } + return false + case *ParenExpr: + // walk down the tree + return HasTimeExpr(n.Expr) + default: + return false + } +} + +// Visitor can be called by Walk to traverse an AST hierarchy. +// The Visit() function is called once per node. +type Visitor interface { + Visit(Node) Visitor +} + +// Walk traverses a node hierarchy in depth-first order. +func Walk(v Visitor, node Node) { + if node == nil { + return + } + + if v = v.Visit(node); v == nil { + return + } + + switch n := node.(type) { + case *BinaryExpr: + Walk(v, n.LHS) + Walk(v, n.RHS) + + case *Call: + for _, expr := range n.Args { + Walk(v, expr) + } + + case *CreateContinuousQueryStatement: + Walk(v, n.Source) + + case *Dimension: + Walk(v, n.Expr) + + case Dimensions: + for _, c := range n { + Walk(v, c) + } + + case *DeleteSeriesStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *DropSeriesStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ExplainStatement: + Walk(v, n.Statement) + + case *Field: + Walk(v, n.Expr) + + case Fields: + for _, c := range n { + Walk(v, c) + } + + case *ParenExpr: + Walk(v, n.Expr) + + case *Query: + Walk(v, n.Statements) + + case *SelectStatement: + Walk(v, n.Fields) + Walk(v, n.Target) + Walk(v, n.Dimensions) + Walk(v, n.Sources) + Walk(v, n.Condition) + Walk(v, n.SortFields) + + case *ShowFieldKeyCardinalityStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowSeriesStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowSeriesCardinalityStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowMeasurementCardinalityStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowTagKeyCardinalityStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowTagKeysStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + Walk(v, n.SortFields) + + case *ShowTagValuesCardinalityStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + + case *ShowTagValuesStatement: + Walk(v, n.Sources) + Walk(v, n.Condition) + Walk(v, n.SortFields) + + case *ShowFieldKeysStatement: + Walk(v, n.Sources) + Walk(v, n.SortFields) + + case SortFields: + for _, sf := range n { + Walk(v, sf) + } + + case Sources: + for _, s := range n { + Walk(v, s) + } + + case *SubQuery: + Walk(v, n.Statement) + + case Statements: + for _, s := range n { + Walk(v, s) + } + + case *Target: + if n != nil { + Walk(v, n.Measurement) + } + } +} + +// WalkFunc traverses a node hierarchy in depth-first order. +func WalkFunc(node Node, fn func(Node)) { + Walk(walkFuncVisitor(fn), node) +} + +type walkFuncVisitor func(Node) + +func (fn walkFuncVisitor) Visit(n Node) Visitor { fn(n); return fn } + +// Rewriter can be called by Rewrite to replace nodes in the AST hierarchy. +// The Rewrite() function is called once per node. +type Rewriter interface { + Rewrite(Node) Node +} + +// Rewrite recursively invokes the rewriter to replace each node. +// Nodes are traversed depth-first and rewritten from leaf to root. +func Rewrite(r Rewriter, node Node) Node { + switch n := node.(type) { + case *Query: + n.Statements = Rewrite(r, n.Statements).(Statements) + + case Statements: + for i, s := range n { + n[i] = Rewrite(r, s).(Statement) + } + + case *SelectStatement: + n.Fields = Rewrite(r, n.Fields).(Fields) + n.Dimensions = Rewrite(r, n.Dimensions).(Dimensions) + n.Sources = Rewrite(r, n.Sources).(Sources) + + // Rewrite may return nil. Nil does not satisfy the Expr + // interface. We only assert the rewritten result to be an + // Expr if it is not nil: + if cond := Rewrite(r, n.Condition); cond != nil { + n.Condition = cond.(Expr) + } else { + n.Condition = nil + } + + case *SubQuery: + n.Statement = Rewrite(r, n.Statement).(*SelectStatement) + + case Fields: + for i, f := range n { + n[i] = Rewrite(r, f).(*Field) + } + + case *Field: + n.Expr = Rewrite(r, n.Expr).(Expr) + + case Dimensions: + for i, d := range n { + n[i] = Rewrite(r, d).(*Dimension) + } + + case *Dimension: + n.Expr = Rewrite(r, n.Expr).(Expr) + + case *BinaryExpr: + n.LHS = Rewrite(r, n.LHS).(Expr) + n.RHS = Rewrite(r, n.RHS).(Expr) + + case *ParenExpr: + n.Expr = Rewrite(r, n.Expr).(Expr) + + case *Call: + for i, expr := range n.Args { + n.Args[i] = Rewrite(r, expr).(Expr) + } + } + + return r.Rewrite(node) +} + +// RewriteFunc rewrites a node hierarchy. +func RewriteFunc(node Node, fn func(Node) Node) Node { + return Rewrite(rewriterFunc(fn), node) +} + +type rewriterFunc func(Node) Node + +func (fn rewriterFunc) Rewrite(n Node) Node { return fn(n) } + +// RewriteExpr recursively invokes the function to replace each expr. +// Nodes are traversed depth-first and rewritten from leaf to root. +func RewriteExpr(expr Expr, fn func(Expr) Expr) Expr { + switch e := expr.(type) { + case *BinaryExpr: + e.LHS = RewriteExpr(e.LHS, fn) + e.RHS = RewriteExpr(e.RHS, fn) + if e.LHS != nil && e.RHS == nil { + expr = e.LHS + } else if e.RHS != nil && e.LHS == nil { + expr = e.RHS + } else if e.LHS == nil && e.RHS == nil { + return nil + } + + case *ParenExpr: + e.Expr = RewriteExpr(e.Expr, fn) + if e.Expr == nil { + return nil + } + + case *Call: + for i, expr := range e.Args { + e.Args[i] = RewriteExpr(expr, fn) + } + } + + return fn(expr) +} + +// Eval evaluates expr against a map. +func Eval(expr Expr, m map[string]interface{}) interface{} { + eval := ValuerEval{Valuer: MapValuer(m)} + return eval.Eval(expr) +} + +// MapValuer is a valuer that substitutes values for the mapped interface. +type MapValuer map[string]interface{} + +// Value returns the value for a key in the MapValuer. +func (m MapValuer) Value(key string) (interface{}, bool) { + v, ok := m[key] + return v, ok +} + +// ValuerEval will evaluate an expression using the Valuer. +type ValuerEval struct { + Valuer Valuer + + // IntegerFloatDivision will set the eval system to treat + // a division between two integers as a floating point division. + IntegerFloatDivision bool +} + +// Eval evaluates an expression and returns a value. +func (v *ValuerEval) Eval(expr Expr) interface{} { + if expr == nil { + return nil + } + + switch expr := expr.(type) { + case *BinaryExpr: + return v.evalBinaryExpr(expr) + case *BooleanLiteral: + return expr.Val + case *IntegerLiteral: + return expr.Val + case *NumberLiteral: + return expr.Val + case *UnsignedLiteral: + return expr.Val + case *ParenExpr: + return v.Eval(expr.Expr) + case *RegexLiteral: + return expr.Val + case *StringLiteral: + return expr.Val + case *Call: + if valuer, ok := v.Valuer.(CallValuer); ok { + var args []interface{} + if len(expr.Args) > 0 { + args = make([]interface{}, len(expr.Args)) + for i := range expr.Args { + args[i] = v.Eval(expr.Args[i]) + } + } + val, _ := valuer.Call(expr.Name, args) + return val + } + return nil + case *VarRef: + val, _ := v.Valuer.Value(expr.Val) + return val + default: + return nil + } +} + +// EvalBool evaluates expr and returns true if result is a boolean true. +// Otherwise returns false. +func (v *ValuerEval) EvalBool(expr Expr) bool { + val, _ := v.Eval(expr).(bool) + return val +} + +func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} { + lhs := v.Eval(expr.LHS) + rhs := v.Eval(expr.RHS) + if lhs == nil && rhs != nil { + // When the LHS is nil and the RHS is a boolean, implicitly cast the + // nil to false. + if _, ok := rhs.(bool); ok { + lhs = false + } + } else if lhs != nil && rhs == nil { + // Implicit cast of the RHS nil to false when the LHS is a boolean. + if _, ok := lhs.(bool); ok { + rhs = false + } + } + + // Evaluate if both sides are simple types. + switch lhs := lhs.(type) { + case bool: + rhs, ok := rhs.(bool) + switch expr.Op { + case AND: + return ok && (lhs && rhs) + case OR: + return ok && (lhs || rhs) + case BITWISE_AND: + return ok && (lhs && rhs) + case BITWISE_OR: + return ok && (lhs || rhs) + case BITWISE_XOR: + return ok && (lhs != rhs) + case EQ: + return ok && (lhs == rhs) + case NEQ: + return ok && (lhs != rhs) + } + case float64: + // Try the rhs as a float64, int64, or uint64 + rhsf, ok := rhs.(float64) + if !ok { + switch val := rhs.(type) { + case int64: + rhsf, ok = float64(val), true + case uint64: + rhsf, ok = float64(val), true + } + } + + rhs := rhsf + switch expr.Op { + case EQ: + return ok && (lhs == rhs) + case NEQ: + return ok && (lhs != rhs) + case LT: + return ok && (lhs < rhs) + case LTE: + return ok && (lhs <= rhs) + case GT: + return ok && (lhs > rhs) + case GTE: + return ok && (lhs >= rhs) + case ADD: + if !ok { + return nil + } + return lhs + rhs + case SUB: + if !ok { + return nil + } + return lhs - rhs + case MUL: + if !ok { + return nil + } + return lhs * rhs + case DIV: + if !ok { + return nil + } else if rhs == 0 { + return float64(0) + } + return lhs / rhs + case MOD: + if !ok { + return nil + } + return math.Mod(lhs, rhs) + } + case int64: + // Try as a float64 to see if a float cast is required. + switch rhs := rhs.(type) { + case float64: + lhs := float64(lhs) + switch expr.Op { + case EQ: + return lhs == rhs + case NEQ: + return lhs != rhs + case LT: + return lhs < rhs + case LTE: + return lhs <= rhs + case GT: + return lhs > rhs + case GTE: + return lhs >= rhs + case ADD: + return lhs + rhs + case SUB: + return lhs - rhs + case MUL: + return lhs * rhs + case DIV: + if rhs == 0 { + return float64(0) + } + return lhs / rhs + case MOD: + return math.Mod(lhs, rhs) + } + case int64: + switch expr.Op { + case EQ: + return lhs == rhs + case NEQ: + return lhs != rhs + case LT: + return lhs < rhs + case LTE: + return lhs <= rhs + case GT: + return lhs > rhs + case GTE: + return lhs >= rhs + case ADD: + return lhs + rhs + case SUB: + return lhs - rhs + case MUL: + return lhs * rhs + case DIV: + if v.IntegerFloatDivision { + if rhs == 0 { + return float64(0) + } + return float64(lhs) / float64(rhs) + } + + if rhs == 0 { + return int64(0) + } + return lhs / rhs + case MOD: + if rhs == 0 { + return int64(0) + } + return lhs % rhs + case BITWISE_AND: + return lhs & rhs + case BITWISE_OR: + return lhs | rhs + case BITWISE_XOR: + return lhs ^ rhs + } + case uint64: + switch expr.Op { + case EQ: + return uint64(lhs) == rhs + case NEQ: + return uint64(lhs) != rhs + case LT: + if lhs < 0 { + return true + } + return uint64(lhs) < rhs + case LTE: + if lhs < 0 { + return true + } + return uint64(lhs) <= rhs + case GT: + if lhs < 0 { + return false + } + return uint64(lhs) > rhs + case GTE: + if lhs < 0 { + return false + } + return uint64(lhs) >= rhs + case ADD: + return uint64(lhs) + rhs + case SUB: + return uint64(lhs) - rhs + case MUL: + return uint64(lhs) * rhs + case DIV: + if rhs == 0 { + return uint64(0) + } + return uint64(lhs) / rhs + case MOD: + if rhs == 0 { + return uint64(0) + } + return uint64(lhs) % rhs + case BITWISE_AND: + return uint64(lhs) & rhs + case BITWISE_OR: + return uint64(lhs) | rhs + case BITWISE_XOR: + return uint64(lhs) ^ rhs + } + } + case uint64: + // Try as a float64 to see if a float cast is required. + switch rhs := rhs.(type) { + case float64: + lhs := float64(lhs) + switch expr.Op { + case EQ: + return lhs == rhs + case NEQ: + return lhs != rhs + case LT: + return lhs < rhs + case LTE: + return lhs <= rhs + case GT: + return lhs > rhs + case GTE: + return lhs >= rhs + case ADD: + return lhs + rhs + case SUB: + return lhs - rhs + case MUL: + return lhs * rhs + case DIV: + if rhs == 0 { + return float64(0) + } + return lhs / rhs + case MOD: + return math.Mod(lhs, rhs) + } + case int64: + switch expr.Op { + case EQ: + return lhs == uint64(rhs) + case NEQ: + return lhs != uint64(rhs) + case LT: + if rhs < 0 { + return false + } + return lhs < uint64(rhs) + case LTE: + if rhs < 0 { + return false + } + return lhs <= uint64(rhs) + case GT: + if rhs < 0 { + return true + } + return lhs > uint64(rhs) + case GTE: + if rhs < 0 { + return true + } + return lhs >= uint64(rhs) + case ADD: + return lhs + uint64(rhs) + case SUB: + return lhs - uint64(rhs) + case MUL: + return lhs * uint64(rhs) + case DIV: + if rhs == 0 { + return uint64(0) + } + return lhs / uint64(rhs) + case MOD: + if rhs == 0 { + return uint64(0) + } + return lhs % uint64(rhs) + case BITWISE_AND: + return lhs & uint64(rhs) + case BITWISE_OR: + return lhs | uint64(rhs) + case BITWISE_XOR: + return lhs ^ uint64(rhs) + } + case uint64: + switch expr.Op { + case EQ: + return lhs == rhs + case NEQ: + return lhs != rhs + case LT: + return lhs < rhs + case LTE: + return lhs <= rhs + case GT: + return lhs > rhs + case GTE: + return lhs >= rhs + case ADD: + return lhs + rhs + case SUB: + return lhs - rhs + case MUL: + return lhs * rhs + case DIV: + if rhs == 0 { + return uint64(0) + } + return lhs / rhs + case MOD: + if rhs == 0 { + return uint64(0) + } + return lhs % rhs + case BITWISE_AND: + return lhs & rhs + case BITWISE_OR: + return lhs | rhs + case BITWISE_XOR: + return lhs ^ rhs + } + } + case string: + switch expr.Op { + case EQ: + rhs, ok := rhs.(string) + if !ok { + return false + } + return lhs == rhs + case NEQ: + rhs, ok := rhs.(string) + if !ok { + return false + } + return lhs != rhs + case EQREGEX: + rhs, ok := rhs.(*regexp.Regexp) + if !ok { + return false + } + return rhs.MatchString(lhs) + case NEQREGEX: + rhs, ok := rhs.(*regexp.Regexp) + if !ok { + return false + } + return !rhs.MatchString(lhs) + } + } + + // The types were not comparable. If our operation was an equality operation, + // return false instead of true. + switch expr.Op { + case EQ, NEQ, LT, LTE, GT, GTE: + return false + } + return nil +} + +// EvalBool evaluates expr and returns true if result is a boolean true. +// Otherwise returns false. +func EvalBool(expr Expr, m map[string]interface{}) bool { + v, _ := Eval(expr, m).(bool) + return v +} + +// TypeMapper maps a data type to the measurement and field. +type TypeMapper interface { + MapType(measurement *Measurement, field string) DataType +} + +// CallTypeMapper maps a data type to the function call. +type CallTypeMapper interface { + TypeMapper + + CallType(name string, args []DataType) (DataType, error) +} + +type nilTypeMapper struct{} + +func (nilTypeMapper) MapType(*Measurement, string) DataType { return Unknown } + +type multiTypeMapper []TypeMapper + +// MultiTypeMapper combines multiple TypeMappers into a single one. +// The MultiTypeMapper will return the first type that is not Unknown. +// It will not iterate through all of them to find the highest priority one. +func MultiTypeMapper(mappers ...TypeMapper) TypeMapper { + return multiTypeMapper(mappers) +} + +func (a multiTypeMapper) MapType(measurement *Measurement, field string) DataType { + for _, m := range a { + if typ := m.MapType(measurement, field); typ != Unknown { + return typ + } + } + return Unknown +} + +func (a multiTypeMapper) CallType(name string, args []DataType) (DataType, error) { + for _, m := range a { + call, ok := m.(CallTypeMapper) + if ok { + typ, err := call.CallType(name, args) + if err != nil { + return Unknown, err + } else if typ != Unknown { + return typ, nil + } + } + } + return Unknown, nil +} + +// TypeValuerEval evaluates an expression to determine its output type. +type TypeValuerEval struct { + TypeMapper TypeMapper + Sources Sources +} + +// EvalType returns the type for an expression. If the expression cannot +// be evaluated for some reason, like incompatible types, it is returned +// as a TypeError in the error. If the error is non-fatal so we can continue +// even though an error happened, true will be returned. +// This function assumes that the expression has already been reduced. +func (v *TypeValuerEval) EvalType(expr Expr) (DataType, error) { + switch expr := expr.(type) { + case *VarRef: + return v.evalVarRefExprType(expr) + case *Call: + return v.evalCallExprType(expr) + case *BinaryExpr: + return v.evalBinaryExprType(expr) + case *ParenExpr: + return v.EvalType(expr.Expr) + case *NumberLiteral: + return Float, nil + case *IntegerLiteral: + return Integer, nil + case *UnsignedLiteral: + return Unsigned, nil + case *StringLiteral: + return String, nil + case *BooleanLiteral: + return Boolean, nil + } + return Unknown, nil +} + +func (v *TypeValuerEval) evalVarRefExprType(expr *VarRef) (DataType, error) { + // If this variable already has an assigned type, just use that. + if expr.Type != Unknown && expr.Type != AnyField { + return expr.Type, nil + } + + var typ DataType + if v.TypeMapper != nil { + for _, src := range v.Sources { + switch src := src.(type) { + case *Measurement: + if t := v.TypeMapper.MapType(src, expr.Val); typ.LessThan(t) { + typ = t + } + case *SubQuery: + _, e := src.Statement.FieldExprByName(expr.Val) + if e != nil { + valuer := TypeValuerEval{ + TypeMapper: v.TypeMapper, + Sources: src.Statement.Sources, + } + if t, err := valuer.EvalType(e); err != nil { + return Unknown, err + } else if typ.LessThan(t) { + typ = t + } + } + + if typ == Unknown { + for _, d := range src.Statement.Dimensions { + if d, ok := d.Expr.(*VarRef); ok && expr.Val == d.Val { + typ = Tag + } + } + } + } + } + } + return typ, nil +} + +func (v *TypeValuerEval) evalCallExprType(expr *Call) (DataType, error) { + typmap, ok := v.TypeMapper.(CallTypeMapper) + if !ok { + return Unknown, nil + } + + // Evaluate all of the data types for the arguments. + args := make([]DataType, len(expr.Args)) + for i, arg := range expr.Args { + typ, err := v.EvalType(arg) + if err != nil { + return Unknown, err + } + args[i] = typ + } + + // Pass in the data types for the call so it can be type checked and + // the resulting type can be returned. + return typmap.CallType(expr.Name, args) +} + +func (v *TypeValuerEval) evalBinaryExprType(expr *BinaryExpr) (DataType, error) { + // Find the data type for both sides of the expression. + lhs, err := v.EvalType(expr.LHS) + if err != nil { + return Unknown, err + } + rhs, err := v.EvalType(expr.RHS) + if err != nil { + return Unknown, err + } + + // If one of the two is unsigned and the other is an integer, we cannot add + // the two without an explicit cast unless the integer is a literal. + if lhs == Unsigned && rhs == Integer { + if isLiteral(expr.LHS) { + return Unknown, &TypeError{ + Expr: expr, + Message: fmt.Sprintf("cannot use %s with an integer and unsigned literal", expr.Op), + } + } else if !isLiteral(expr.RHS) { + return Unknown, &TypeError{ + Expr: expr, + Message: fmt.Sprintf("cannot use %s between an integer and unsigned, an explicit cast is required", expr.Op), + } + } + } else if lhs == Integer && rhs == Unsigned { + if isLiteral(expr.RHS) { + return Unknown, &TypeError{ + Expr: expr, + Message: fmt.Sprintf("cannot use %s with an integer and unsigned literal", expr.Op), + } + } else if !isLiteral(expr.LHS) { + return Unknown, &TypeError{ + Expr: expr, + Message: fmt.Sprintf("cannot use %s between an integer and unsigned, an explicit cast is required", expr.Op), + } + } + } + + // If one of the two is unknown, then return the other as the type. + if lhs == Unknown { + return rhs, nil + } else if rhs == Unknown { + return lhs, nil + } + + // Rather than re-implement the ValuerEval here, we create a dummy binary + // expression with the zero values and inspect the resulting value back into + // a data type to determine the output. + e := BinaryExpr{ + LHS: &VarRef{Val: "lhs"}, + RHS: &VarRef{Val: "rhs"}, + Op: expr.Op, + } + result := Eval(&e, map[string]interface{}{ + "lhs": lhs.Zero(), + "rhs": rhs.Zero(), + }) + + typ := InspectDataType(result) + if typ == Unknown { + // If the type is unknown, then the two types were not compatible. + return Unknown, &TypeError{ + Expr: expr, + Message: fmt.Sprintf("incompatible types: %s and %s", lhs, rhs), + } + } + return typ, nil +} + +// TypeError is an error when two types are incompatible. +type TypeError struct { + // Expr contains the expression that generated the type error. + Expr Expr + // Message contains the informational message about the type error. + Message string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("type error: %s: %s", e.Expr, e.Message) +} + +// EvalType evaluates the expression's type. +func EvalType(expr Expr, sources Sources, typmap TypeMapper) DataType { + if typmap == nil { + typmap = nilTypeMapper{} + } + + valuer := TypeValuerEval{ + TypeMapper: typmap, + Sources: sources, + } + typ, _ := valuer.EvalType(expr) + return typ +} + +func FieldDimensions(sources Sources, m FieldMapper) (fields map[string]DataType, dimensions map[string]struct{}, err error) { + fields = make(map[string]DataType) + dimensions = make(map[string]struct{}) + + for _, src := range sources { + switch src := src.(type) { + case *Measurement: + f, d, err := m.FieldDimensions(src) + if err != nil { + return nil, nil, err + } + + for k, typ := range f { + if fields[k].LessThan(typ) { + fields[k] = typ + } + } + for k := range d { + dimensions[k] = struct{}{} + } + case *SubQuery: + for _, f := range src.Statement.Fields { + k := f.Name() + typ := EvalType(f.Expr, src.Statement.Sources, m) + + if fields[k].LessThan(typ) { + fields[k] = typ + } + } + + for _, d := range src.Statement.Dimensions { + if expr, ok := d.Expr.(*VarRef); ok { + dimensions[expr.Val] = struct{}{} + } + } + } + } + return +} + +// Reduce evaluates expr using the available values in valuer. +// References that don't exist in valuer are ignored. +func Reduce(expr Expr, valuer Valuer) Expr { + expr = reduce(expr, valuer) + + // Unwrap parens at top level. + if expr, ok := expr.(*ParenExpr); ok { + return expr.Expr + } + return expr +} + +func reduce(expr Expr, valuer Valuer) Expr { + if expr == nil { + return nil + } + + switch expr := expr.(type) { + case *BinaryExpr: + return reduceBinaryExpr(expr, valuer) + case *Call: + return reduceCall(expr, valuer) + case *ParenExpr: + return reduceParenExpr(expr, valuer) + case *VarRef: + return reduceVarRef(expr, valuer) + case *NilLiteral: + return expr + default: + return CloneExpr(expr) + } +} + +func reduceBinaryExpr(expr *BinaryExpr, valuer Valuer) Expr { + // Reduce both sides first. + op := expr.Op + lhs := reduce(expr.LHS, valuer) + rhs := reduce(expr.RHS, valuer) + + loc := time.UTC + if valuer, ok := valuer.(ZoneValuer); ok { + if l := valuer.Zone(); l != nil { + loc = l + } + } + + // Do not evaluate if one side is nil. + if lhs == nil || rhs == nil { + return &BinaryExpr{LHS: lhs, RHS: rhs, Op: expr.Op} + } + + // If we have a logical operator (AND, OR) and one side is a boolean literal + // then we need to have special handling. + if op == AND { + if isFalseLiteral(lhs) || isFalseLiteral(rhs) { + return &BooleanLiteral{Val: false} + } else if isTrueLiteral(lhs) { + return rhs + } else if isTrueLiteral(rhs) { + return lhs + } + } else if op == OR { + if isTrueLiteral(lhs) || isTrueLiteral(rhs) { + return &BooleanLiteral{Val: true} + } else if isFalseLiteral(lhs) { + return rhs + } else if isFalseLiteral(rhs) { + return lhs + } + } + + // Evaluate if both sides are simple types. + switch lhs := lhs.(type) { + case *BooleanLiteral: + return reduceBinaryExprBooleanLHS(op, lhs, rhs) + case *DurationLiteral: + return reduceBinaryExprDurationLHS(op, lhs, rhs, loc) + case *IntegerLiteral: + return reduceBinaryExprIntegerLHS(op, lhs, rhs, loc) + case *UnsignedLiteral: + return reduceBinaryExprUnsignedLHS(op, lhs, rhs) + case *NilLiteral: + return reduceBinaryExprNilLHS(op, lhs, rhs) + case *NumberLiteral: + return reduceBinaryExprNumberLHS(op, lhs, rhs) + case *StringLiteral: + return reduceBinaryExprStringLHS(op, lhs, rhs, loc) + case *TimeLiteral: + return reduceBinaryExprTimeLHS(op, lhs, rhs, loc) + default: + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} + } +} + +func reduceBinaryExprBooleanLHS(op Token, lhs *BooleanLiteral, rhs Expr) Expr { + switch rhs := rhs.(type) { + case *BooleanLiteral: + switch op { + case EQ: + return &BooleanLiteral{Val: lhs.Val == rhs.Val} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + case AND: + return &BooleanLiteral{Val: lhs.Val && rhs.Val} + case OR: + return &BooleanLiteral{Val: lhs.Val || rhs.Val} + case BITWISE_AND: + return &BooleanLiteral{Val: lhs.Val && rhs.Val} + case BITWISE_OR: + return &BooleanLiteral{Val: lhs.Val || rhs.Val} + case BITWISE_XOR: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + } + case *NilLiteral: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprDurationLHS(op Token, lhs *DurationLiteral, rhs Expr, loc *time.Location) Expr { + switch rhs := rhs.(type) { + case *DurationLiteral: + switch op { + case ADD: + return &DurationLiteral{Val: lhs.Val + rhs.Val} + case SUB: + return &DurationLiteral{Val: lhs.Val - rhs.Val} + case EQ: + return &BooleanLiteral{Val: lhs.Val == rhs.Val} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + case GT: + return &BooleanLiteral{Val: lhs.Val > rhs.Val} + case GTE: + return &BooleanLiteral{Val: lhs.Val >= rhs.Val} + case LT: + return &BooleanLiteral{Val: lhs.Val < rhs.Val} + case LTE: + return &BooleanLiteral{Val: lhs.Val <= rhs.Val} + } + case *NumberLiteral: + switch op { + case MUL: + return &DurationLiteral{Val: lhs.Val * time.Duration(rhs.Val)} + case DIV: + if rhs.Val == 0 { + return &DurationLiteral{Val: 0} + } + return &DurationLiteral{Val: lhs.Val / time.Duration(rhs.Val)} + } + case *IntegerLiteral: + switch op { + case MUL: + return &DurationLiteral{Val: lhs.Val * time.Duration(rhs.Val)} + case DIV: + if rhs.Val == 0 { + return &DurationLiteral{Val: 0} + } + return &DurationLiteral{Val: lhs.Val / time.Duration(rhs.Val)} + } + case *TimeLiteral: + switch op { + case ADD: + return &TimeLiteral{Val: rhs.Val.Add(lhs.Val)} + } + case *StringLiteral: + t, err := rhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprDurationLHS(op, lhs, t, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *NilLiteral: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprIntegerLHS(op Token, lhs *IntegerLiteral, rhs Expr, loc *time.Location) Expr { + switch rhs := rhs.(type) { + case *NumberLiteral: + return reduceBinaryExprNumberLHS(op, &NumberLiteral{Val: float64(lhs.Val)}, rhs) + case *IntegerLiteral: + switch op { + case ADD: + return &IntegerLiteral{Val: lhs.Val + rhs.Val} + case SUB: + return &IntegerLiteral{Val: lhs.Val - rhs.Val} + case MUL: + return &IntegerLiteral{Val: lhs.Val * rhs.Val} + case DIV: + if rhs.Val == 0 { + return &NumberLiteral{Val: 0} + } + return &NumberLiteral{Val: float64(lhs.Val) / float64(rhs.Val)} + case MOD: + if rhs.Val == 0 { + return &IntegerLiteral{Val: 0} + } + return &IntegerLiteral{Val: lhs.Val % rhs.Val} + case BITWISE_AND: + return &IntegerLiteral{Val: lhs.Val & rhs.Val} + case BITWISE_OR: + return &IntegerLiteral{Val: lhs.Val | rhs.Val} + case BITWISE_XOR: + return &IntegerLiteral{Val: lhs.Val ^ rhs.Val} + case EQ: + return &BooleanLiteral{Val: lhs.Val == rhs.Val} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + case GT: + return &BooleanLiteral{Val: lhs.Val > rhs.Val} + case GTE: + return &BooleanLiteral{Val: lhs.Val >= rhs.Val} + case LT: + return &BooleanLiteral{Val: lhs.Val < rhs.Val} + case LTE: + return &BooleanLiteral{Val: lhs.Val <= rhs.Val} + } + case *UnsignedLiteral: + // Comparisons between an unsigned and integer literal will not involve + // a cast if the integer is negative as that will have an improper result. + // Look for those situations here. + if lhs.Val < 0 { + switch op { + case LT, LTE: + return &BooleanLiteral{Val: true} + case GT, GTE: + return &BooleanLiteral{Val: false} + } + } + return reduceBinaryExprUnsignedLHS(op, &UnsignedLiteral{Val: uint64(lhs.Val)}, rhs) + case *DurationLiteral: + // Treat the integer as a timestamp. + switch op { + case ADD: + return &TimeLiteral{Val: time.Unix(0, lhs.Val).Add(rhs.Val)} + case SUB: + return &TimeLiteral{Val: time.Unix(0, lhs.Val).Add(-rhs.Val)} + } + case *TimeLiteral: + d := &DurationLiteral{Val: time.Duration(lhs.Val)} + expr := reduceBinaryExprDurationLHS(op, d, rhs, loc) + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *StringLiteral: + t, err := rhs.ToTimeLiteral(loc) + if err != nil { + break + } + d := &DurationLiteral{Val: time.Duration(lhs.Val)} + expr := reduceBinaryExprDurationLHS(op, d, t, loc) + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *NilLiteral: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprUnsignedLHS(op Token, lhs *UnsignedLiteral, rhs Expr) Expr { + switch rhs := rhs.(type) { + case *NumberLiteral: + return reduceBinaryExprNumberLHS(op, &NumberLiteral{Val: float64(lhs.Val)}, rhs) + case *IntegerLiteral: + // Comparisons between an unsigned and integer literal will not involve + // a cast if the integer is negative as that will have an improper result. + // Look for those situations here. + if rhs.Val < 0 { + switch op { + case LT, LTE: + return &BooleanLiteral{Val: false} + case GT, GTE: + return &BooleanLiteral{Val: true} + } + } + return reduceBinaryExprUnsignedLHS(op, lhs, &UnsignedLiteral{Val: uint64(rhs.Val)}) + case *UnsignedLiteral: + switch op { + case ADD: + return &UnsignedLiteral{Val: lhs.Val + rhs.Val} + case SUB: + return &UnsignedLiteral{Val: lhs.Val - rhs.Val} + case MUL: + return &UnsignedLiteral{Val: lhs.Val * rhs.Val} + case DIV: + if rhs.Val == 0 { + return &UnsignedLiteral{Val: 0} + } + return &UnsignedLiteral{Val: lhs.Val / rhs.Val} + case MOD: + if rhs.Val == 0 { + return &UnsignedLiteral{Val: 0} + } + return &UnsignedLiteral{Val: lhs.Val % rhs.Val} + case EQ: + return &BooleanLiteral{Val: lhs.Val == rhs.Val} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + case GT: + return &BooleanLiteral{Val: lhs.Val > rhs.Val} + case GTE: + return &BooleanLiteral{Val: lhs.Val >= rhs.Val} + case LT: + return &BooleanLiteral{Val: lhs.Val < rhs.Val} + case LTE: + return &BooleanLiteral{Val: lhs.Val <= rhs.Val} + } + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprNilLHS(op Token, lhs *NilLiteral, rhs Expr) Expr { + switch op { + case EQ, NEQ: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprNumberLHS(op Token, lhs *NumberLiteral, rhs Expr) Expr { + switch rhs := rhs.(type) { + case *NumberLiteral: + switch op { + case ADD: + return &NumberLiteral{Val: lhs.Val + rhs.Val} + case SUB: + return &NumberLiteral{Val: lhs.Val - rhs.Val} + case MUL: + return &NumberLiteral{Val: lhs.Val * rhs.Val} + case DIV: + if rhs.Val == 0 { + return &NumberLiteral{Val: 0} + } + return &NumberLiteral{Val: lhs.Val / rhs.Val} + case MOD: + return &NumberLiteral{Val: math.Mod(lhs.Val, rhs.Val)} + case EQ: + return &BooleanLiteral{Val: lhs.Val == rhs.Val} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != rhs.Val} + case GT: + return &BooleanLiteral{Val: lhs.Val > rhs.Val} + case GTE: + return &BooleanLiteral{Val: lhs.Val >= rhs.Val} + case LT: + return &BooleanLiteral{Val: lhs.Val < rhs.Val} + case LTE: + return &BooleanLiteral{Val: lhs.Val <= rhs.Val} + } + case *IntegerLiteral: + switch op { + case ADD: + return &NumberLiteral{Val: lhs.Val + float64(rhs.Val)} + case SUB: + return &NumberLiteral{Val: lhs.Val - float64(rhs.Val)} + case MUL: + return &NumberLiteral{Val: lhs.Val * float64(rhs.Val)} + case DIV: + if float64(rhs.Val) == 0 { + return &NumberLiteral{Val: 0} + } + return &NumberLiteral{Val: lhs.Val / float64(rhs.Val)} + case MOD: + return &NumberLiteral{Val: math.Mod(lhs.Val, float64(rhs.Val))} + case EQ: + return &BooleanLiteral{Val: lhs.Val == float64(rhs.Val)} + case NEQ: + return &BooleanLiteral{Val: lhs.Val != float64(rhs.Val)} + case GT: + return &BooleanLiteral{Val: lhs.Val > float64(rhs.Val)} + case GTE: + return &BooleanLiteral{Val: lhs.Val >= float64(rhs.Val)} + case LT: + return &BooleanLiteral{Val: lhs.Val < float64(rhs.Val)} + case LTE: + return &BooleanLiteral{Val: lhs.Val <= float64(rhs.Val)} + } + case *UnsignedLiteral: + return reduceBinaryExprNumberLHS(op, lhs, &NumberLiteral{Val: float64(rhs.Val)}) + case *NilLiteral: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprStringLHS(op Token, lhs *StringLiteral, rhs Expr, loc *time.Location) Expr { + switch rhs := rhs.(type) { + case *StringLiteral: + switch op { + case EQ: + var expr Expr = &BooleanLiteral{Val: lhs.Val == rhs.Val} + // This might be a comparison between time literals. + // If it is, parse the time literals and then compare since it + // could be a different result if they use different formats + // for the same time. + if lhs.IsTimeLiteral() && rhs.IsTimeLiteral() { + tlhs, err := lhs.ToTimeLiteral(loc) + if err != nil { + return expr + } + + trhs, err := rhs.ToTimeLiteral(loc) + if err != nil { + return expr + } + + t := reduceBinaryExprTimeLHS(op, tlhs, trhs, loc) + if _, ok := t.(*BinaryExpr); !ok { + expr = t + } + } + return expr + case NEQ: + var expr Expr = &BooleanLiteral{Val: lhs.Val != rhs.Val} + // This might be a comparison between time literals. + // If it is, parse the time literals and then compare since it + // could be a different result if they use different formats + // for the same time. + if lhs.IsTimeLiteral() && rhs.IsTimeLiteral() { + tlhs, err := lhs.ToTimeLiteral(loc) + if err != nil { + return expr + } + + trhs, err := rhs.ToTimeLiteral(loc) + if err != nil { + return expr + } + + t := reduceBinaryExprTimeLHS(op, tlhs, trhs, loc) + if _, ok := t.(*BinaryExpr); !ok { + expr = t + } + } + return expr + case ADD: + return &StringLiteral{Val: lhs.Val + rhs.Val} + default: + // Attempt to convert the string literal to a time literal. + t, err := lhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprTimeLHS(op, t, rhs, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + } + case *DurationLiteral: + // Attempt to convert the string literal to a time literal. + t, err := lhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprTimeLHS(op, t, rhs, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *TimeLiteral: + // Attempt to convert the string literal to a time literal. + t, err := lhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprTimeLHS(op, t, rhs, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *IntegerLiteral: + // Attempt to convert the string literal to a time literal. + t, err := lhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprTimeLHS(op, t, rhs, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *NilLiteral: + switch op { + case EQ, NEQ: + return &BooleanLiteral{Val: false} + } + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceBinaryExprTimeLHS(op Token, lhs *TimeLiteral, rhs Expr, loc *time.Location) Expr { + switch rhs := rhs.(type) { + case *DurationLiteral: + switch op { + case ADD: + return &TimeLiteral{Val: lhs.Val.Add(rhs.Val)} + case SUB: + return &TimeLiteral{Val: lhs.Val.Add(-rhs.Val)} + } + case *IntegerLiteral: + d := &DurationLiteral{Val: time.Duration(rhs.Val)} + expr := reduceBinaryExprTimeLHS(op, lhs, d, loc) + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *TimeLiteral: + switch op { + case SUB: + return &DurationLiteral{Val: lhs.Val.Sub(rhs.Val)} + case EQ: + return &BooleanLiteral{Val: lhs.Val.Equal(rhs.Val)} + case NEQ: + return &BooleanLiteral{Val: !lhs.Val.Equal(rhs.Val)} + case GT: + return &BooleanLiteral{Val: lhs.Val.After(rhs.Val)} + case GTE: + return &BooleanLiteral{Val: lhs.Val.After(rhs.Val) || lhs.Val.Equal(rhs.Val)} + case LT: + return &BooleanLiteral{Val: lhs.Val.Before(rhs.Val)} + case LTE: + return &BooleanLiteral{Val: lhs.Val.Before(rhs.Val) || lhs.Val.Equal(rhs.Val)} + } + case *StringLiteral: + t, err := rhs.ToTimeLiteral(loc) + if err != nil { + break + } + expr := reduceBinaryExprTimeLHS(op, lhs, t, loc) + + // If the returned expression is still a binary expr, that means + // we couldn't reduce it so this wasn't used in a time literal context. + if _, ok := expr.(*BinaryExpr); !ok { + return expr + } + case *NilLiteral: + return &BooleanLiteral{Val: false} + } + return &BinaryExpr{Op: op, LHS: lhs, RHS: rhs} +} + +func reduceCall(expr *Call, valuer Valuer) Expr { + // Otherwise reduce arguments. + var args []Expr + literalsOnly := true + if len(expr.Args) > 0 { + args = make([]Expr, len(expr.Args)) + for i, arg := range expr.Args { + args[i] = reduce(arg, valuer) + if !isLiteral(args[i]) { + literalsOnly = false + } + } + } + + // Evaluate a function call if the valuer is a CallValuer and + // the arguments are only literals. + if literalsOnly { + if valuer, ok := valuer.(CallValuer); ok { + argVals := make([]interface{}, len(args)) + for i := range args { + argVals[i] = Eval(args[i], nil) + } + if v, ok := valuer.Call(expr.Name, argVals); ok { + return asLiteral(v) + } + } + } + return &Call{Name: expr.Name, Args: args} +} + +func reduceParenExpr(expr *ParenExpr, valuer Valuer) Expr { + subexpr := reduce(expr.Expr, valuer) + if subexpr, ok := subexpr.(*BinaryExpr); ok { + return &ParenExpr{Expr: subexpr} + } + return subexpr +} + +func reduceVarRef(expr *VarRef, valuer Valuer) Expr { + // Ignore if there is no valuer. + if valuer == nil { + return &VarRef{Val: expr.Val, Type: expr.Type} + } + + // Retrieve the value of the ref. + // Ignore if the value doesn't exist. + v, ok := valuer.Value(expr.Val) + if !ok { + return &VarRef{Val: expr.Val, Type: expr.Type} + } + + // Return the value as a literal. + return asLiteral(v) +} + +// asLiteral takes an interface and converts it into an influxql literal. +func asLiteral(v interface{}) Literal { + switch v := v.(type) { + case bool: + return &BooleanLiteral{Val: v} + case time.Duration: + return &DurationLiteral{Val: v} + case float64: + return &NumberLiteral{Val: v} + case int64: + return &IntegerLiteral{Val: v} + case string: + return &StringLiteral{Val: v} + case time.Time: + return &TimeLiteral{Val: v} + default: + return &NilLiteral{} + } +} + +// isLiteral returns if the expression is a literal. +func isLiteral(expr Expr) bool { + _, ok := expr.(Literal) + return ok +} + +// Valuer is the interface that wraps the Value() method. +type Valuer interface { + // Value returns the value and existence flag for a given key. + Value(key string) (interface{}, bool) +} + +// CallValuer implements the Call method for evaluating function calls. +type CallValuer interface { + Valuer + + // Call is invoked to evaluate a function call (if possible). + Call(name string, args []interface{}) (interface{}, bool) +} + +// ZoneValuer is the interface that specifies the current time zone. +type ZoneValuer interface { + Valuer + + // Zone returns the time zone location. This function may return nil + // if no time zone is known. + Zone() *time.Location +} + +var _ CallValuer = (*NowValuer)(nil) +var _ ZoneValuer = (*NowValuer)(nil) + +// NowValuer returns only the value for "now()". +type NowValuer struct { + Now time.Time + Location *time.Location +} + +// Value is a method that returns the value and existence flag for a given key. +func (v *NowValuer) Value(key string) (interface{}, bool) { + if !v.Now.IsZero() && key == "now()" { + return v.Now, true + } + return nil, false +} + +// Call evaluates the now() function to replace now() with the current time. +func (v *NowValuer) Call(name string, args []interface{}) (interface{}, bool) { + if name == "now" && len(args) == 0 { + return v.Now, true + } + return nil, false +} + +// Zone is a method that returns the time.Location. +func (v *NowValuer) Zone() *time.Location { + if v.Location != nil { + return v.Location + } + return nil +} + +// MultiValuer returns a Valuer that iterates over multiple Valuer instances +// to find a match. +func MultiValuer(valuers ...Valuer) Valuer { + return multiValuer(valuers) +} + +type multiValuer []Valuer + +var _ CallValuer = multiValuer(nil) +var _ ZoneValuer = multiValuer(nil) + +func (a multiValuer) Value(key string) (interface{}, bool) { + for _, valuer := range a { + if v, ok := valuer.Value(key); ok { + return v, true + } + } + return nil, false +} + +func (a multiValuer) Call(name string, args []interface{}) (interface{}, bool) { + for _, valuer := range a { + if valuer, ok := valuer.(CallValuer); ok { + if v, ok := valuer.Call(name, args); ok { + return v, true + } + } + } + return nil, false +} + +func (a multiValuer) Zone() *time.Location { + for _, valuer := range a { + if valuer, ok := valuer.(ZoneValuer); ok { + if v := valuer.Zone(); v != nil { + return v + } + } + } + return nil +} + +// ContainsVarRef returns true if expr is a VarRef or contains one. +func ContainsVarRef(expr Expr) bool { + var v containsVarRefVisitor + Walk(&v, expr) + return v.contains +} + +type containsVarRefVisitor struct { + contains bool +} + +func (v *containsVarRefVisitor) Visit(n Node) Visitor { + switch n.(type) { + case *Call: + return nil + case *VarRef: + v.contains = true + } + return v +} + +func IsSelector(expr Expr) bool { + if call, ok := expr.(*Call); ok { + switch call.Name { + case "first", "last", "min", "max", "percentile", "sample", "top", "bottom": + return true + } + } + return false +} + +// stringSetSlice returns a sorted slice of keys from a string set. +func stringSetSlice(m map[string]struct{}) []string { + if m == nil { + return nil + } + + a := make([]string, 0, len(m)) + for k := range m { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// TimeRange represents a range of time from Min to Max. The times are inclusive. +type TimeRange struct { + Min, Max time.Time +} + +// Intersect joins this TimeRange with another TimeRange. +func (t TimeRange) Intersect(other TimeRange) TimeRange { + if !other.Min.IsZero() { + if t.Min.IsZero() || other.Min.After(t.Min) { + t.Min = other.Min + } + } + if !other.Max.IsZero() { + if t.Max.IsZero() || other.Max.Before(t.Max) { + t.Max = other.Max + } + } + return t +} + +// IsZero is true if the min and max of the time range are zero. +func (t TimeRange) IsZero() bool { + return t.Min.IsZero() && t.Max.IsZero() +} + +// Used by TimeRange methods. +var minTime = time.Unix(0, MinTime) +var maxTime = time.Unix(0, MaxTime) + +// MinTime returns the minimum time of the TimeRange. +// If the minimum time is zero, this returns the minimum possible time. +func (t TimeRange) MinTime() time.Time { + if t.Min.IsZero() { + return minTime + } + return t.Min +} + +// MaxTime returns the maximum time of the TimeRange. +// If the maximum time is zero, this returns the maximum possible time. +func (t TimeRange) MaxTime() time.Time { + if t.Max.IsZero() { + return maxTime + } + return t.Max +} + +// MinTimeNano returns the minimum time in nanoseconds since the epoch. +// If the minimum time is zero, this returns the minimum possible time. +func (t TimeRange) MinTimeNano() int64 { + if t.Min.IsZero() { + return MinTime + } + return t.Min.UnixNano() +} + +// MaxTimeNano returns the maximum time in nanoseconds since the epoch. +// If the maximum time is zero, this returns the maximum possible time. +func (t TimeRange) MaxTimeNano() int64 { + if t.Max.IsZero() { + return MaxTime + } + return t.Max.UnixNano() +} + +// ConditionExpr extracts the time range and the condition from an expression. +// We only support simple time ranges that are constrained with AND and are not nested. +// This throws an error when we encounter a time condition that is combined with OR +// to prevent returning unexpected results that we do not support. +func ConditionExpr(cond Expr, valuer Valuer) (Expr, TimeRange, error) { + expr, tr, err := conditionExpr(cond, valuer) + + // Remove top level parentheses + if e, ok := expr.(*ParenExpr); ok { + expr = e.Expr + } + + if e, ok := expr.(*BooleanLiteral); ok && e.Val { + // If the condition is true, return nil instead to indicate there + // is no condition. + expr = nil + } + return expr, tr, err +} + +func conditionExpr(cond Expr, valuer Valuer) (Expr, TimeRange, error) { + if cond == nil { + return nil, TimeRange{}, nil + } + + switch cond := cond.(type) { + case *BinaryExpr: + if cond.Op == AND || cond.Op == OR { + lhsExpr, lhsTime, err := conditionExpr(cond.LHS, valuer) + if err != nil { + return nil, TimeRange{}, err + } + + rhsExpr, rhsTime, err := conditionExpr(cond.RHS, valuer) + if err != nil { + return nil, TimeRange{}, err + } + + // Always intersect the time range even if it makes no sense. + // There is no such thing as using OR with a time range. + timeRange := lhsTime.Intersect(rhsTime) + + // Combine the left and right expression. + if rhsExpr == nil { + return lhsExpr, timeRange, nil + } else if lhsExpr == nil { + return rhsExpr, timeRange, nil + } + return reduce(&BinaryExpr{ + Op: cond.Op, + LHS: lhsExpr, + RHS: rhsExpr, + }, nil), timeRange, nil + } + + // If either the left or the right side is "time", we are looking at + // a time range. + if lhs, ok := cond.LHS.(*VarRef); ok && strings.ToLower(lhs.Val) == "time" { + timeRange, err := getTimeRange(cond.Op, cond.RHS, valuer) + return nil, timeRange, err + } else if rhs, ok := cond.RHS.(*VarRef); ok && strings.ToLower(rhs.Val) == "time" { + // Swap the op for the opposite if it is a comparison. + op := cond.Op + switch op { + case GT: + op = LT + case LT: + op = GT + case GTE: + op = LTE + case LTE: + op = GTE + } + timeRange, err := getTimeRange(op, cond.LHS, valuer) + return nil, timeRange, err + } + return reduce(cond, valuer), TimeRange{}, nil + case *ParenExpr: + expr, timeRange, err := conditionExpr(cond.Expr, valuer) + if err != nil { + return nil, TimeRange{}, err + } else if expr == nil { + return nil, timeRange, nil + } + return reduce(&ParenExpr{Expr: expr}, nil), timeRange, nil + case *BooleanLiteral: + return cond, TimeRange{}, nil + default: + return nil, TimeRange{}, fmt.Errorf("invalid condition expression: %s", cond) + } +} + +// getTimeRange returns the time range associated with this comparison. +// op is the operation that is used for comparison and rhs is the right hand side +// of the expression. The left hand side is always assumed to be "time". +func getTimeRange(op Token, rhs Expr, valuer Valuer) (TimeRange, error) { + // If literal looks like a date time then parse it as a time literal. + if strlit, ok := rhs.(*StringLiteral); ok { + if strlit.IsTimeLiteral() { + var loc *time.Location + if valuer, ok := valuer.(ZoneValuer); ok { + loc = valuer.Zone() + } + t, err := strlit.ToTimeLiteral(loc) + if err != nil { + return TimeRange{}, err + } + rhs = t + } + } + + // Evaluate the RHS to replace "now()" with the current time. + rhs = Reduce(rhs, valuer) + + var value time.Time + switch lit := rhs.(type) { + case *TimeLiteral: + if lit.Val.After(time.Unix(0, MaxTime)) { + return TimeRange{}, fmt.Errorf("time %s overflows time literal", lit.Val.Format(time.RFC3339)) + } else if lit.Val.Before(time.Unix(0, MinTime+1)) { + // The minimum allowable time literal is one greater than the minimum time because the minimum time + // is a sentinel value only used internally. + return TimeRange{}, fmt.Errorf("time %s underflows time literal", lit.Val.Format(time.RFC3339)) + } + value = lit.Val + case *DurationLiteral: + value = time.Unix(0, int64(lit.Val)).UTC() + case *NumberLiteral: + value = time.Unix(0, int64(lit.Val)).UTC() + case *IntegerLiteral: + value = time.Unix(0, lit.Val).UTC() + default: + return TimeRange{}, fmt.Errorf("invalid operation: time and %T are not compatible", lit) + } + + timeRange := TimeRange{} + switch op { + case GT: + timeRange.Min = value.Add(time.Nanosecond) + case GTE: + timeRange.Min = value + case LT: + timeRange.Max = value.Add(-time.Nanosecond) + case LTE: + timeRange.Max = value + case EQ: + timeRange.Min, timeRange.Max = value, value + default: + return TimeRange{}, fmt.Errorf("invalid time comparison operator: %s", op) + } + return timeRange, nil +} diff --git a/vendor/github.com/influxdata/influxql/doc.go b/vendor/github.com/influxdata/influxql/doc.go new file mode 100644 index 0000000000..8b814c9cea --- /dev/null +++ b/vendor/github.com/influxdata/influxql/doc.go @@ -0,0 +1,12 @@ +/* +Package influxql implements a parser for the InfluxDB query language. + +InfluxQL is a DML and DDL language for the InfluxDB time series database. +It provides the ability to query for aggregate statistics as well as create +and configure the InfluxDB server. + +See https://docs.influxdata.com/influxdb/latest/query_language/ +for a reference on using InfluxQL. + +*/ +package influxql diff --git a/vendor/github.com/influxdata/influxql/influxql.go b/vendor/github.com/influxdata/influxql/influxql.go new file mode 100644 index 0000000000..037c3a2220 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/influxql.go @@ -0,0 +1,3 @@ +package influxql // import "github.com/influxdata/influxql" + +//go:generate protoc --gogo_out=. internal/internal.proto diff --git a/vendor/github.com/influxdata/influxql/internal/internal.pb.go b/vendor/github.com/influxdata/influxql/internal/internal.pb.go new file mode 100644 index 0000000000..548c3e2ef4 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/internal/internal.pb.go @@ -0,0 +1,120 @@ +// Code generated by protoc-gen-gogo. +// source: internal/internal.proto +// DO NOT EDIT! + +/* +Package influxql is a generated protocol buffer package. + +It is generated from these files: + internal/internal.proto + +It has these top-level messages: + Measurements + Measurement +*/ +package influxql + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Measurements struct { + Items []*Measurement `protobuf:"bytes,1,rep,name=Items" json:"Items,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Measurements) Reset() { *m = Measurements{} } +func (m *Measurements) String() string { return proto.CompactTextString(m) } +func (*Measurements) ProtoMessage() {} +func (*Measurements) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } + +func (m *Measurements) GetItems() []*Measurement { + if m != nil { + return m.Items + } + return nil +} + +type Measurement struct { + Database *string `protobuf:"bytes,1,opt,name=Database" json:"Database,omitempty"` + RetentionPolicy *string `protobuf:"bytes,2,opt,name=RetentionPolicy" json:"RetentionPolicy,omitempty"` + Name *string `protobuf:"bytes,3,opt,name=Name" json:"Name,omitempty"` + Regex *string `protobuf:"bytes,4,opt,name=Regex" json:"Regex,omitempty"` + IsTarget *bool `protobuf:"varint,5,opt,name=IsTarget" json:"IsTarget,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Measurement) Reset() { *m = Measurement{} } +func (m *Measurement) String() string { return proto.CompactTextString(m) } +func (*Measurement) ProtoMessage() {} +func (*Measurement) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } + +func (m *Measurement) GetDatabase() string { + if m != nil && m.Database != nil { + return *m.Database + } + return "" +} + +func (m *Measurement) GetRetentionPolicy() string { + if m != nil && m.RetentionPolicy != nil { + return *m.RetentionPolicy + } + return "" +} + +func (m *Measurement) GetName() string { + if m != nil && m.Name != nil { + return *m.Name + } + return "" +} + +func (m *Measurement) GetRegex() string { + if m != nil && m.Regex != nil { + return *m.Regex + } + return "" +} + +func (m *Measurement) GetIsTarget() bool { + if m != nil && m.IsTarget != nil { + return *m.IsTarget + } + return false +} + +func init() { + proto.RegisterType((*Measurements)(nil), "influxql.Measurements") + proto.RegisterType((*Measurement)(nil), "influxql.Measurement") +} + +func init() { proto.RegisterFile("internal/internal.proto", fileDescriptorInternal) } + +var fileDescriptorInternal = []byte{ + // 195 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xcf, 0xcc, 0x2b, 0x49, + 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x87, 0x31, 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0x38, 0x32, + 0xf3, 0xd2, 0x72, 0x4a, 0x2b, 0x0a, 0x73, 0x94, 0xac, 0xb9, 0x78, 0x7c, 0x53, 0x13, 0x8b, 0x4b, + 0x8b, 0x52, 0x73, 0x53, 0xf3, 0x4a, 0x8a, 0x85, 0xb4, 0xb9, 0x58, 0x3d, 0x4b, 0x52, 0x73, 0x8b, + 0x25, 0x18, 0x15, 0x98, 0x35, 0xb8, 0x8d, 0x44, 0xf5, 0x60, 0x2a, 0xf5, 0x90, 0x94, 0x05, 0x41, + 0xd4, 0x28, 0xcd, 0x64, 0xe4, 0xe2, 0x46, 0x12, 0x16, 0x92, 0xe2, 0xe2, 0x70, 0x49, 0x2c, 0x49, + 0x4c, 0x4a, 0x2c, 0x4e, 0x95, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0xf3, 0x85, 0x34, 0xb8, + 0xf8, 0x83, 0x52, 0x4b, 0x52, 0xf3, 0x4a, 0x32, 0xf3, 0xf3, 0x02, 0xf2, 0x73, 0x32, 0x93, 0x2b, + 0x25, 0x98, 0xc0, 0x4a, 0xd0, 0x85, 0x85, 0x84, 0xb8, 0x58, 0xfc, 0x12, 0x73, 0x53, 0x25, 0x98, + 0xc1, 0xd2, 0x60, 0xb6, 0x90, 0x08, 0x17, 0x6b, 0x50, 0x6a, 0x7a, 0x6a, 0x85, 0x04, 0x0b, 0x58, + 0x10, 0xc2, 0x01, 0xd9, 0xe7, 0x59, 0x1c, 0x92, 0x58, 0x94, 0x9e, 0x5a, 0x22, 0xc1, 0xaa, 0xc0, + 0xa8, 0xc1, 0x11, 0x04, 0xe7, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x16, 0x06, 0x23, 0xfc, + 0x00, 0x00, 0x00, +} diff --git a/vendor/github.com/influxdata/influxql/internal/internal.proto b/vendor/github.com/influxdata/influxql/internal/internal.proto new file mode 100644 index 0000000000..198ad91e98 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/internal/internal.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; +package influxql; + +message Measurements { + repeated Measurement Items = 1; +} + +message Measurement { + optional string Database = 1; + optional string RetentionPolicy = 2; + optional string Name = 3; + optional string Regex = 4; + optional bool IsTarget = 5; +} diff --git a/vendor/github.com/influxdata/influxql/params.go b/vendor/github.com/influxdata/influxql/params.go new file mode 100644 index 0000000000..e16b72788b --- /dev/null +++ b/vendor/github.com/influxdata/influxql/params.go @@ -0,0 +1,180 @@ +package influxql + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +// Value represents a value that can be bound +// to a parameter when parsing the query. +type Value interface { + TokenType() Token + Value() string +} + +type ( + // Identifier is an identifier value. + Identifier string + + // StringValue is a string literal. + StringValue string + + // RegexValue is a regexp literal. + RegexValue string + + // NumberValue is a number literal. + NumberValue float64 + + // IntegerValue is an integer literal. + IntegerValue int64 + + // BooleanValue is a boolean literal. + BooleanValue bool + + // DurationValue is a duration literal. + DurationValue string + + // ErrorValue is a special value that returns an error during parsing + // when it is used. + ErrorValue string +) + +// BindValue will bind an interface value to its influxql value. +// This method of binding values only supports literals. +func BindValue(v interface{}) Value { + if jv, ok := v.(json.Number); ok { + var err error + v, err = jsonNumberToValue(jv) + if err != nil { + return ErrorValue(err.Error()) + } + } + + switch v := v.(type) { + case float64: + return NumberValue(v) + case int64: + return IntegerValue(v) + case string: + return StringValue(v) + case bool: + return BooleanValue(v) + case map[string]interface{}: + return bindObjectValue(v) + default: + s := fmt.Sprintf("unable to bind parameter with type %T", v) + return ErrorValue(s) + } +} + +// bindObjectValue will bind an object to a value. +func bindObjectValue(m map[string]interface{}) Value { + if len(m) != 1 { + return ErrorValue("bound object parameter value must have exactly one entry") + } + + var ( + k string + v interface{} + ) + for k, v = range m { + // Nothing done here. + } + + if jv, ok := v.(json.Number); ok { + var err error + v, err = jsonNumberToValue(jv) + if err != nil { + return ErrorValue(err.Error()) + } + } + + switch k { + case "ident", "identifier": + s, ok := v.(string) + if !ok { + return ErrorValue("identifier must be a string value") + } + return Identifier(s) + case "regex": + s, ok := v.(string) + if !ok { + return ErrorValue("regex literal must be a string value") + } + return RegexValue(s) + case "string": + s, ok := v.(string) + if !ok { + return ErrorValue("string literal must be a string value") + } + return StringValue(s) + case "float", "number": + switch f := v.(type) { + case float64: + return NumberValue(f) + case int64: + return NumberValue(f) + default: + return ErrorValue("number literal must be a float value") + } + case "int", "integer": + i, ok := v.(int64) + if !ok { + return ErrorValue("integer literal must be an integer value") + } + return IntegerValue(i) + case "duration": + switch d := v.(type) { + case string: + return DurationValue(d) + case int64: + return DurationValue(FormatDuration(time.Duration(d))) + default: + return ErrorValue("duration literal must be a string or integer value") + } + default: + return ErrorValue(fmt.Sprintf("unknown bind object type: %s", k)) + } +} + +func (v Identifier) TokenType() Token { return IDENT } +func (v Identifier) Value() string { return string(v) } +func (v StringValue) TokenType() Token { return STRING } +func (v StringValue) Value() string { return string(v) } +func (v RegexValue) TokenType() Token { return REGEX } +func (v RegexValue) Value() string { return string(v) } +func (v NumberValue) TokenType() Token { return NUMBER } +func (v NumberValue) Value() string { return strconv.FormatFloat(float64(v), 'f', -1, 64) } +func (v IntegerValue) TokenType() Token { return INTEGER } +func (v IntegerValue) Value() string { return strconv.FormatInt(int64(v), 10) } +func (v BooleanValue) TokenType() Token { + if v { + return TRUE + } else { + return FALSE + } +} +func (v BooleanValue) Value() string { return "" } +func (v DurationValue) TokenType() Token { return DURATIONVAL } +func (v DurationValue) Value() string { return string(v) } +func (e ErrorValue) TokenType() Token { return BOUNDPARAM } +func (e ErrorValue) Value() string { return string(e) } + +func jsonNumberToValue(v json.Number) (interface{}, error) { + if strings.Contains(string(v), ".") { + f, err := v.Float64() + if err != nil { + return nil, err + } + return f, nil + } else { + i, err := v.Int64() + if err != nil { + return nil, err + } + return i, nil + } +} diff --git a/vendor/github.com/influxdata/influxql/parse_tree.go b/vendor/github.com/influxdata/influxql/parse_tree.go new file mode 100644 index 0000000000..761fca105d --- /dev/null +++ b/vendor/github.com/influxdata/influxql/parse_tree.go @@ -0,0 +1,236 @@ +package influxql + +import ( + "fmt" +) + +var Language = &ParseTree{} + +type ParseTree struct { + Handlers map[Token]func(*Parser) (Statement, error) + Tokens map[Token]*ParseTree + Keys []string +} + +// With passes the current parse tree to a function to allow nested functions. +func (t *ParseTree) With(fn func(*ParseTree)) { + fn(t) +} + +// Group groups together a set of related handlers with a common token prefix. +func (t *ParseTree) Group(tokens ...Token) *ParseTree { + for _, tok := range tokens { + // Look for the parse tree for this token. + if subtree := t.Tokens[tok]; subtree != nil { + t = subtree + continue + } + + // No subtree exists yet. Verify that we don't have a conflicting + // statement. + if _, conflict := t.Handlers[tok]; conflict { + panic(fmt.Sprintf("conflict for token %s", tok)) + } + + // Create the new parse tree and register it inside of this one for + // later reference. + newT := &ParseTree{} + if t.Tokens == nil { + t.Tokens = make(map[Token]*ParseTree) + } + t.Tokens[tok] = newT + t.Keys = append(t.Keys, tok.String()) + t = newT + } + return t +} + +// Handle registers a handler to be invoked when seeing the given token. +func (t *ParseTree) Handle(tok Token, fn func(*Parser) (Statement, error)) { + // Verify that there is no conflict for this token in this parse tree. + if _, conflict := t.Tokens[tok]; conflict { + panic(fmt.Sprintf("conflict for token %s", tok)) + } + + if _, conflict := t.Handlers[tok]; conflict { + panic(fmt.Sprintf("conflict for token %s", tok)) + } + + if t.Handlers == nil { + t.Handlers = make(map[Token]func(*Parser) (Statement, error)) + } + t.Handlers[tok] = fn + t.Keys = append(t.Keys, tok.String()) +} + +// Parse parses a statement using the language defined in the parse tree. +func (t *ParseTree) Parse(p *Parser) (Statement, error) { + for { + tok, pos, lit := p.ScanIgnoreWhitespace() + if subtree := t.Tokens[tok]; subtree != nil { + t = subtree + continue + } + + if stmt := t.Handlers[tok]; stmt != nil { + return stmt(p) + } + + // There were no registered handlers. Return the valid tokens in the order they were added. + return nil, newParseError(tokstr(tok, lit), t.Keys, pos) + } +} + +func (t *ParseTree) Clone() *ParseTree { + newT := &ParseTree{} + if t.Handlers != nil { + newT.Handlers = make(map[Token]func(*Parser) (Statement, error), len(t.Handlers)) + for tok, handler := range t.Handlers { + newT.Handlers[tok] = handler + } + } + + if t.Tokens != nil { + newT.Tokens = make(map[Token]*ParseTree, len(t.Tokens)) + for tok, subtree := range t.Tokens { + newT.Tokens[tok] = subtree.Clone() + } + } + return newT +} + +func init() { + Language.Handle(SELECT, func(p *Parser) (Statement, error) { + return p.parseSelectStatement(targetNotRequired) + }) + Language.Handle(DELETE, func(p *Parser) (Statement, error) { + return p.parseDeleteStatement() + }) + Language.Group(SHOW).With(func(show *ParseTree) { + show.Group(CONTINUOUS).Handle(QUERIES, func(p *Parser) (Statement, error) { + return p.parseShowContinuousQueriesStatement() + }) + show.Handle(DATABASES, func(p *Parser) (Statement, error) { + return p.parseShowDatabasesStatement() + }) + show.Handle(DIAGNOSTICS, func(p *Parser) (Statement, error) { + return p.parseShowDiagnosticsStatement() + }) + show.Group(FIELD).With(func(field *ParseTree) { + field.Handle(KEY, func(p *Parser) (Statement, error) { + return p.parseShowFieldKeyCardinalityStatement() + }) + field.Handle(KEYS, func(p *Parser) (Statement, error) { + return p.parseShowFieldKeysStatement() + }) + }) + show.Group(GRANTS).Handle(FOR, func(p *Parser) (Statement, error) { + return p.parseGrantsForUserStatement() + }) + show.Group(MEASUREMENT).Handle(EXACT, func(p *Parser) (Statement, error) { + return p.parseShowMeasurementCardinalityStatement(true) + }) + show.Group(MEASUREMENT).Handle(CARDINALITY, func(p *Parser) (Statement, error) { + return p.parseShowMeasurementCardinalityStatement(false) + }) + show.Handle(MEASUREMENTS, func(p *Parser) (Statement, error) { + return p.parseShowMeasurementsStatement() + }) + show.Handle(QUERIES, func(p *Parser) (Statement, error) { + return p.parseShowQueriesStatement() + }) + show.Group(RETENTION).Handle(POLICIES, func(p *Parser) (Statement, error) { + return p.parseShowRetentionPoliciesStatement() + }) + show.Handle(SERIES, func(p *Parser) (Statement, error) { + return p.parseShowSeriesStatement() + }) + show.Group(SHARD).Handle(GROUPS, func(p *Parser) (Statement, error) { + return p.parseShowShardGroupsStatement() + }) + show.Handle(SHARDS, func(p *Parser) (Statement, error) { + return p.parseShowShardsStatement() + }) + show.Handle(STATS, func(p *Parser) (Statement, error) { + return p.parseShowStatsStatement() + }) + show.Handle(SUBSCRIPTIONS, func(p *Parser) (Statement, error) { + return p.parseShowSubscriptionsStatement() + }) + show.Group(TAG).With(func(tag *ParseTree) { + tag.Handle(KEY, func(p *Parser) (Statement, error) { + return p.parseShowTagKeyCardinalityStatement() + }) + tag.Handle(KEYS, func(p *Parser) (Statement, error) { + return p.parseShowTagKeysStatement() + }) + tag.Handle(VALUES, func(p *Parser) (Statement, error) { + return p.parseShowTagValuesStatement() + }) + }) + show.Handle(USERS, func(p *Parser) (Statement, error) { + return p.parseShowUsersStatement() + }) + }) + Language.Group(CREATE).With(func(create *ParseTree) { + create.Group(CONTINUOUS).Handle(QUERY, func(p *Parser) (Statement, error) { + return p.parseCreateContinuousQueryStatement() + }) + create.Handle(DATABASE, func(p *Parser) (Statement, error) { + return p.parseCreateDatabaseStatement() + }) + create.Handle(USER, func(p *Parser) (Statement, error) { + return p.parseCreateUserStatement() + }) + create.Group(RETENTION).Handle(POLICY, func(p *Parser) (Statement, error) { + return p.parseCreateRetentionPolicyStatement() + }) + create.Handle(SUBSCRIPTION, func(p *Parser) (Statement, error) { + return p.parseCreateSubscriptionStatement() + }) + }) + Language.Group(DROP).With(func(drop *ParseTree) { + drop.Group(CONTINUOUS).Handle(QUERY, func(p *Parser) (Statement, error) { + return p.parseDropContinuousQueryStatement() + }) + drop.Handle(DATABASE, func(p *Parser) (Statement, error) { + return p.parseDropDatabaseStatement() + }) + drop.Handle(MEASUREMENT, func(p *Parser) (Statement, error) { + return p.parseDropMeasurementStatement() + }) + drop.Group(RETENTION).Handle(POLICY, func(p *Parser) (Statement, error) { + return p.parseDropRetentionPolicyStatement() + }) + drop.Handle(SERIES, func(p *Parser) (Statement, error) { + return p.parseDropSeriesStatement() + }) + drop.Handle(SHARD, func(p *Parser) (Statement, error) { + return p.parseDropShardStatement() + }) + drop.Handle(SUBSCRIPTION, func(p *Parser) (Statement, error) { + return p.parseDropSubscriptionStatement() + }) + drop.Handle(USER, func(p *Parser) (Statement, error) { + return p.parseDropUserStatement() + }) + }) + Language.Handle(EXPLAIN, func(p *Parser) (Statement, error) { + return p.parseExplainStatement() + }) + Language.Handle(GRANT, func(p *Parser) (Statement, error) { + return p.parseGrantStatement() + }) + Language.Handle(REVOKE, func(p *Parser) (Statement, error) { + return p.parseRevokeStatement() + }) + Language.Group(ALTER, RETENTION).Handle(POLICY, func(p *Parser) (Statement, error) { + return p.parseAlterRetentionPolicyStatement() + }) + Language.Group(SET, PASSWORD).Handle(FOR, func(p *Parser) (Statement, error) { + return p.parseSetPasswordUserStatement() + }) + Language.Group(KILL).Handle(QUERY, func(p *Parser) (Statement, error) { + return p.parseKillQueryStatement() + }) +} diff --git a/vendor/github.com/influxdata/influxql/parser.go b/vendor/github.com/influxdata/influxql/parser.go new file mode 100644 index 0000000000..1ce0777321 --- /dev/null +++ b/vendor/github.com/influxdata/influxql/parser.go @@ -0,0 +1,3163 @@ +package influxql + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +const ( + // DateFormat represents the format for date literals. + DateFormat = "2006-01-02" + + // DateTimeFormat represents the format for date time literals. + DateTimeFormat = "2006-01-02 15:04:05.999999" +) + +// Parser represents an InfluxQL parser. +type Parser struct { + s *bufScanner + params map[string]Value +} + +// NewParser returns a new instance of Parser. +func NewParser(r io.Reader) *Parser { + return &Parser{s: newBufScanner(r)} +} + +// SetParams sets the parameters that will be used for any bound parameter substitutions. +func (p *Parser) SetParams(params map[string]interface{}) { + p.params = make(map[string]Value, len(params)) + for name, param := range params { + p.params[name] = BindValue(param) + } +} + +// ParseQuery parses a query string and returns its AST representation. +func ParseQuery(s string) (*Query, error) { return NewParser(strings.NewReader(s)).ParseQuery() } + +// ParseStatement parses a statement string and returns its AST representation. +func ParseStatement(s string) (Statement, error) { + return NewParser(strings.NewReader(s)).ParseStatement() +} + +// MustParseStatement parses a statement string and returns its AST. Panic on error. +func MustParseStatement(s string) Statement { + stmt, err := ParseStatement(s) + if err != nil { + panic(err.Error()) + } + return stmt +} + +// ParseExpr parses an expression string and returns its AST representation. +func ParseExpr(s string) (Expr, error) { return NewParser(strings.NewReader(s)).ParseExpr() } + +// MustParseExpr parses an expression string and returns its AST. Panic on error. +func MustParseExpr(s string) Expr { + expr, err := ParseExpr(s) + if err != nil { + panic(err.Error()) + } + return expr +} + +// ParseQuery parses an InfluxQL string and returns a Query AST object. +func (p *Parser) ParseQuery() (*Query, error) { + var statements Statements + semi := true + + for { + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok == EOF { + return &Query{Statements: statements}, nil + } else if tok == SEMICOLON { + semi = true + } else { + if !semi { + return nil, newParseError(tokstr(tok, lit), []string{";"}, pos) + } + p.Unscan() + s, err := p.ParseStatement() + if err != nil { + return nil, err + } + statements = append(statements, s) + semi = false + } + } +} + +// ParseStatement parses an InfluxQL string and returns a Statement AST object. +func (p *Parser) ParseStatement() (Statement, error) { + return Language.Parse(p) +} + +// parseSetPasswordUserStatement parses a string and returns a set statement. +// This function assumes the SET token has already been consumed. +func (p *Parser) parseSetPasswordUserStatement() (*SetPasswordUserStatement, error) { + stmt := &SetPasswordUserStatement{} + + // Parse username + ident, err := p.ParseIdent() + + if err != nil { + return nil, err + } + stmt.Name = ident + + // Consume the required = token. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != EQ { + return nil, newParseError(tokstr(tok, lit), []string{"="}, pos) + } + + // Parse new user's password + if ident, err = p.parseString(); err != nil { + return nil, err + } + stmt.Password = ident + + return stmt, nil +} + +// parseKillQueryStatement parses a string and returns a kill statement. +// This function assumes the KILL token has already been consumed. +func (p *Parser) parseKillQueryStatement() (*KillQueryStatement, error) { + qid, err := p.ParseUInt64() + if err != nil { + return nil, err + } + + var host string + if tok, _, _ := p.ScanIgnoreWhitespace(); tok == ON { + host, err = p.ParseIdent() + if err != nil { + return nil, err + } + } else { + p.Unscan() + } + return &KillQueryStatement{QueryID: qid, Host: host}, nil +} + +// parseCreateSubscriptionStatement parses a string and returns a CreateSubscriptionStatement. +// This function assumes the "CREATE SUBSCRIPTION" tokens have already been consumed. +func (p *Parser) parseCreateSubscriptionStatement() (*CreateSubscriptionStatement, error) { + stmt := &CreateSubscriptionStatement{} + + // Read the id of the subscription to create. + ident, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.Name = ident + + // Expect an "ON" keyword. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != ON { + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) + } + + // Read the name of the database. + if ident, err = p.ParseIdent(); err != nil { + return nil, err + } + stmt.Database = ident + + if tok, pos, lit := p.Scan(); tok != DOT { + return nil, newParseError(tokstr(tok, lit), []string{"."}, pos) + } + + // Read the name of the retention policy. + if ident, err = p.ParseIdent(); err != nil { + return nil, err + } + stmt.RetentionPolicy = ident + + // Expect a "DESTINATIONS" keyword. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != DESTINATIONS { + return nil, newParseError(tokstr(tok, lit), []string{"DESTINATIONS"}, pos) + } + + // Expect one of "ANY ALL" keywords. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok == ALL || tok == ANY { + stmt.Mode = tokens[tok] + } else { + return nil, newParseError(tokstr(tok, lit), []string{"ALL", "ANY"}, pos) + } + + // Read list of destinations. + var destinations []string + if destinations, err = p.parseStringList(); err != nil { + return nil, err + } + stmt.Destinations = destinations + + return stmt, nil +} + +// parseCreateRetentionPolicyStatement parses a string and returns a create retention policy statement. +// This function assumes the CREATE RETENTION POLICY tokens have already been consumed. +func (p *Parser) parseCreateRetentionPolicyStatement() (*CreateRetentionPolicyStatement, error) { + stmt := &CreateRetentionPolicyStatement{} + + // Parse the retention policy name. + ident, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.Name = ident + + // Consume the required ON token. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != ON { + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) + } + + // Parse the database name. + ident, err = p.ParseIdent() + if err != nil { + return nil, err + } + stmt.Database = ident + + // Parse required DURATION token. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != DURATION { + return nil, newParseError(tokstr(tok, lit), []string{"DURATION"}, pos) + } + + // Parse duration value + d, err := p.ParseDuration() + if err != nil { + return nil, err + } + stmt.Duration = d + + // Parse required REPLICATION token. + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != REPLICATION { + return nil, newParseError(tokstr(tok, lit), []string{"REPLICATION"}, pos) + } + + // Parse replication value. + n, err := p.ParseInt(1, math.MaxInt32) + if err != nil { + return nil, err + } + stmt.Replication = n + + // Parse optional SHARD token. + if tok, _, _ := p.ScanIgnoreWhitespace(); tok == SHARD { + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != DURATION { + return nil, newParseError(tokstr(tok, lit), []string{"DURATION"}, pos) + } + + // Check to see if they used the INF keyword + tok, pos, _ := p.ScanIgnoreWhitespace() + if tok == INF { + return nil, &ParseError{ + Message: "invalid duration INF for shard duration", + Pos: pos, + } + } + p.Unscan() + + d, err := p.ParseDuration() + if err != nil { + return nil, err + } + stmt.ShardGroupDuration = d + } else { + p.Unscan() + } + + // Parse optional DEFAULT token. + if tok, _, _ := p.ScanIgnoreWhitespace(); tok == DEFAULT { + stmt.Default = true + } else { + p.Unscan() + } + + return stmt, nil +} + +// parseAlterRetentionPolicyStatement parses a string and returns an alter retention policy statement. +// This function assumes the ALTER RETENTION POLICY tokens have already been consumed. +func (p *Parser) parseAlterRetentionPolicyStatement() (*AlterRetentionPolicyStatement, error) { + stmt := &AlterRetentionPolicyStatement{} + + // Parse the retention policy name. + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok == DEFAULT { + stmt.Name = "default" + } else if tok == IDENT { + stmt.Name = lit + } else { + return nil, newParseError(tokstr(tok, lit), []string{"identifier"}, pos) + } + + // Consume the required ON token. + if tok, pos, lit = p.ScanIgnoreWhitespace(); tok != ON { + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) + } + + // Parse the database name. + ident, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.Database = ident + + // Loop through option tokens (DURATION, REPLICATION, SHARD DURATION, DEFAULT, etc.). + found := make(map[Token]struct{}) +Loop: + for { + tok, pos, lit := p.ScanIgnoreWhitespace() + if _, ok := found[tok]; ok { + return nil, &ParseError{ + Message: fmt.Sprintf("found duplicate %s option", tok), + Pos: pos, + } + } + + switch tok { + case DURATION: + d, err := p.ParseDuration() + if err != nil { + return nil, err + } + stmt.Duration = &d + case REPLICATION: + n, err := p.ParseInt(1, math.MaxInt32) + if err != nil { + return nil, err + } + stmt.Replication = &n + case SHARD: + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok == DURATION { + // Check to see if they used the INF keyword + tok, pos, _ := p.ScanIgnoreWhitespace() + if tok == INF { + return nil, &ParseError{ + Message: "invalid duration INF for shard duration", + Pos: pos, + } + } + p.Unscan() + + d, err := p.ParseDuration() + if err != nil { + return nil, err + } + stmt.ShardGroupDuration = &d + } else { + return nil, newParseError(tokstr(tok, lit), []string{"DURATION"}, pos) + } + case DEFAULT: + stmt.Default = true + default: + if len(found) == 0 { + return nil, newParseError(tokstr(tok, lit), []string{"DURATION", "REPLICATION", "SHARD", "DEFAULT"}, pos) + } + p.Unscan() + break Loop + } + found[tok] = struct{}{} + } + + return stmt, nil +} + +// ParseInt parses a string representing a base 10 integer and returns the number. +// It returns an error if the parsed number is outside the range [min, max]. +func (p *Parser) ParseInt(min, max int) (int, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok != INTEGER { + return 0, newParseError(tokstr(tok, lit), []string{"integer"}, pos) + } + + // Convert string to int. + n, err := strconv.Atoi(lit) + if err != nil { + return 0, &ParseError{Message: err.Error(), Pos: pos} + } else if min > n || n > max { + return 0, &ParseError{ + Message: fmt.Sprintf("invalid value %d: must be %d <= n <= %d", n, min, max), + Pos: pos, + } + } + + return n, nil +} + +// ParseUInt64 parses a string and returns a 64-bit unsigned integer literal. +func (p *Parser) ParseUInt64() (uint64, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok != INTEGER { + return 0, newParseError(tokstr(tok, lit), []string{"integer"}, pos) + } + + // Convert string to unsigned 64-bit integer + n, err := strconv.ParseUint(lit, 10, 64) + if err != nil { + return 0, &ParseError{Message: err.Error(), Pos: pos} + } + + return uint64(n), nil +} + +// ParseDuration parses a string and returns a duration literal. +// This function assumes the DURATION token has already been consumed. +func (p *Parser) ParseDuration() (time.Duration, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok != DURATIONVAL && tok != INF { + return 0, newParseError(tokstr(tok, lit), []string{"duration"}, pos) + } + + if tok == INF { + return 0, nil + } + + d, err := ParseDuration(lit) + if err != nil { + return 0, &ParseError{Message: err.Error(), Pos: pos} + } + + return d, nil +} + +// ParseIdent parses an identifier. +func (p *Parser) ParseIdent() (string, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok != IDENT { + return "", newParseError(tokstr(tok, lit), []string{"identifier"}, pos) + } + return lit, nil +} + +// ParseIdentList parses a comma delimited list of identifiers. +func (p *Parser) ParseIdentList() ([]string, error) { + // Parse first (required) identifier. + ident, err := p.ParseIdent() + if err != nil { + return nil, err + } + idents := []string{ident} + + // Parse remaining (optional) identifiers. + for { + if tok, _, _ := p.ScanIgnoreWhitespace(); tok != COMMA { + p.Unscan() + return idents, nil + } + + if ident, err = p.ParseIdent(); err != nil { + return nil, err + } + + idents = append(idents, ident) + } +} + +// parseSegmentedIdents parses a segmented identifiers. +// e.g., "db"."rp".measurement or "db"..measurement +func (p *Parser) parseSegmentedIdents() ([]string, error) { + ident, err := p.ParseIdent() + if err != nil { + return nil, err + } + idents := []string{ident} + + // Parse remaining (optional) identifiers. + for { + if tok, _, _ := p.Scan(); tok != DOT { + // No more segments so we're done. + p.Unscan() + break + } + + if ch := p.peekRune(); ch == '/' { + // Next segment is a regex so we're done. + break + } else if ch == ':' { + // Next segment is context-specific so let caller handle it. + break + } else if ch == '.' { + // Add an empty identifier. + idents = append(idents, "") + continue + } + + // Parse the next identifier. + if ident, err = p.ParseIdent(); err != nil { + return nil, err + } + + idents = append(idents, ident) + } + + if len(idents) > 3 { + msg := fmt.Sprintf("too many segments in %s", QuoteIdent(idents...)) + return nil, &ParseError{Message: msg} + } + + return idents, nil +} + +// parseString parses a string. +func (p *Parser) parseString() (string, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok != STRING { + return "", newParseError(tokstr(tok, lit), []string{"string"}, pos) + } + return lit, nil +} + +// parseStringList parses a list of strings separated by commas. +func (p *Parser) parseStringList() ([]string, error) { + // Parse first (required) string. + str, err := p.parseString() + if err != nil { + return nil, err + } + strs := []string{str} + + // Parse remaining (optional) strings. + for { + if tok, _, _ := p.ScanIgnoreWhitespace(); tok != COMMA { + p.Unscan() + return strs, nil + } + + if str, err = p.parseString(); err != nil { + return nil, err + } + + strs = append(strs, str) + } +} + +// parseRevokeStatement parses a string and returns a revoke statement. +// This function assumes the REVOKE token has already been consumed. +func (p *Parser) parseRevokeStatement() (Statement, error) { + // Parse the privilege to be revoked. + priv, err := p.parsePrivilege() + if err != nil { + return nil, err + } + + // Check for ON or FROM clauses. + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok == ON { + stmt, err := p.parseRevokeOnStatement() + if err != nil { + return nil, err + } + stmt.Privilege = priv + return stmt, nil + } else if tok == FROM { + // Admin privilege is only revoked on ALL PRIVILEGES. + if priv != AllPrivileges { + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) + } + return p.parseRevokeAdminStatement() + } + + // Only ON or FROM clauses are allowed after privilege. + if priv == AllPrivileges { + return nil, newParseError(tokstr(tok, lit), []string{"ON", "FROM"}, pos) + } + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) +} + +// parseRevokeOnStatement parses a string and returns a revoke statement. +// This function assumes the [PRIVILEGE] ON tokens have already been consumed. +func (p *Parser) parseRevokeOnStatement() (*RevokeStatement, error) { + stmt := &RevokeStatement{} + + // Parse the name of the database. + lit, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.On = lit + + // Parse FROM clause. + tok, pos, lit := p.ScanIgnoreWhitespace() + + // Check for required FROM token. + if tok != FROM { + return nil, newParseError(tokstr(tok, lit), []string{"FROM"}, pos) + } + + // Parse the name of the user. + lit, err = p.ParseIdent() + if err != nil { + return nil, err + } + stmt.User = lit + + return stmt, nil +} + +// parseRevokeAdminStatement parses a string and returns a revoke admin statement. +// This function assumes the ALL [PRVILEGES] FROM token has already been consumed. +func (p *Parser) parseRevokeAdminStatement() (*RevokeAdminStatement, error) { + // Admin privilege is always false when revoke admin clause is called. + stmt := &RevokeAdminStatement{} + + // Parse the name of the user. + lit, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.User = lit + + return stmt, nil +} + +// parseGrantStatement parses a string and returns a grant statement. +// This function assumes the GRANT token has already been consumed. +func (p *Parser) parseGrantStatement() (Statement, error) { + // Parse the privilege to be granted. + priv, err := p.parsePrivilege() + if err != nil { + return nil, err + } + + // Check for ON or TO clauses. + tok, pos, lit := p.ScanIgnoreWhitespace() + if tok == ON { + stmt, err := p.parseGrantOnStatement() + if err != nil { + return nil, err + } + stmt.Privilege = priv + return stmt, nil + } else if tok == TO { + // Admin privilege is only granted on ALL PRIVILEGES. + if priv != AllPrivileges { + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) + } + return p.parseGrantAdminStatement() + } + + // Only ON or TO clauses are allowed after privilege. + if priv == AllPrivileges { + return nil, newParseError(tokstr(tok, lit), []string{"ON", "TO"}, pos) + } + return nil, newParseError(tokstr(tok, lit), []string{"ON"}, pos) +} + +// parseGrantOnStatement parses a string and returns a grant statement. +// This function assumes the [PRIVILEGE] ON tokens have already been consumed. +func (p *Parser) parseGrantOnStatement() (*GrantStatement, error) { + stmt := &GrantStatement{} + + // Parse the name of the database. + lit, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.On = lit + + // Parse TO clause. + tok, pos, lit := p.ScanIgnoreWhitespace() + + // Check for required TO token. + if tok != TO { + return nil, newParseError(tokstr(tok, lit), []string{"TO"}, pos) + } + + // Parse the name of the user. + lit, err = p.ParseIdent() + if err != nil { + return nil, err + } + stmt.User = lit + + return stmt, nil +} + +// parseGrantAdminStatement parses a string and returns a grant admin statement. +// This function assumes the ALL [PRVILEGES] TO tokens have already been consumed. +func (p *Parser) parseGrantAdminStatement() (*GrantAdminStatement, error) { + // Admin privilege is always true when grant admin clause is called. + stmt := &GrantAdminStatement{} + + // Parse the name of the user. + lit, err := p.ParseIdent() + if err != nil { + return nil, err + } + stmt.User = lit + + return stmt, nil +} + +// parsePrivilege parses a string and returns a Privilege. +func (p *Parser) parsePrivilege() (Privilege, error) { + tok, pos, lit := p.ScanIgnoreWhitespace() + switch tok { + case READ: + return ReadPrivilege, nil + case WRITE: + return WritePrivilege, nil + case ALL: + // Consume optional PRIVILEGES token + tok, pos, lit = p.ScanIgnoreWhitespace() + if tok != PRIVILEGES { + p.Unscan() + } + return AllPrivileges, nil + } + return 0, newParseError(tokstr(tok, lit), []string{"READ", "WRITE", "ALL [PRIVILEGES]"}, pos) +} + +// parseSelectStatement parses a select string and returns a Statement AST object. +// This function assumes the SELECT token has already been consumed. +func (p *Parser) parseSelectStatement(tr targetRequirement) (*SelectStatement, error) { + stmt := &SelectStatement{} + var err error + + // Parse fields: "FIELD+". + if stmt.Fields, err = p.parseFields(); err != nil { + return nil, err + } + + // Parse target: "INTO" + if stmt.Target, err = p.parseTarget(tr); err != nil { + return nil, err + } + + // Parse source: "FROM". + if tok, pos, lit := p.ScanIgnoreWhitespace(); tok != FROM { + return nil, newParseError(tokstr(tok, lit), []string{"FROM"}, pos) + } + if stmt.Sources, err = p.parseSources(true); err != nil { + return nil, err + } + + // Parse condition: "WHERE EXPR". + if stmt.Condition, err = p.parseCondition(); err != nil { + return nil, err + } + + // Parse dimensions: "GROUP BY DIMENSION+". + if stmt.Dimensions, err = p.parseDimensions(); err != nil { + return nil, err + } + + // Parse fill options: "fill(