From 724deb5f5eae29177187ab77b26fda17ed0d8cdb Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Wed, 26 Oct 2022 02:08:13 +0800 Subject: [PATCH] feature: google shell support query bigquery (#15213) Co-authored-by: Qiu Jian --- pkg/multicloud/google/bigquery.go | 166 ++++++++++++++++++++++++ pkg/multicloud/google/google.go | 3 + pkg/multicloud/google/shell/bigquery.go | 35 +++++ 3 files changed, 204 insertions(+) create mode 100644 pkg/multicloud/google/bigquery.go create mode 100644 pkg/multicloud/google/shell/bigquery.go diff --git a/pkg/multicloud/google/bigquery.go b/pkg/multicloud/google/bigquery.go new file mode 100644 index 0000000000..6ccadb9811 --- /dev/null +++ b/pkg/multicloud/google/bigquery.go @@ -0,0 +1,166 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +func (self *SGoogleClient) bigqueryPost(resource string, params map[string]string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + resource = fmt.Sprintf("projects/%s/%s", self.projectId, resource) + return jsonRequest(self.client, "POST", GOOGLE_BIGQUERY_DOMAIN, GOOGLE_BIGQUERY_API_VERSION, resource, params, body, self.debug) +} + +func (self *SGoogleClient) bigqueryList(resource string, params map[string]string) (jsonutils.JSONObject, error) { + return jsonRequest(self.client, "GET", GOOGLE_BIGQUERY_DOMAIN, GOOGLE_BIGQUERY_API_VERSION, resource, params, nil, self.debug) +} + +func (self *SGoogleClient) bigqueryGet(resource string) (jsonutils.JSONObject, error) { + return jsonRequest(self.client, "GET", GOOGLE_BIGQUERY_DOMAIN, GOOGLE_BIGQUERY_API_VERSION, resource, nil, nil, self.debug) +} + +func (region *SRegion) BigQuery(sql string) ([]jsonutils.JSONObject, error) { + req := struct { + Kind string `json:"kind"` + Query string `json:"query"` + MaxResults string `json:"maxResults"` + UseLegacySql bool `json:"useLegacySql"` + }{ + Kind: "query", + Query: sql, + MaxResults: "1", + UseLegacySql: false, + } + resp, err := region.client.bigqueryPost("queries", nil, jsonutils.Marshal(req)) + if err != nil { + return nil, errors.Wrap(err, "bigqueryPost") + } + result := SBigQueryResult{} + err = resp.Unmarshal(&result) + if err != nil { + return nil, errors.Wrap(err, "jsonutils.Unmarshal") + } + rows, err := result.GetRows() + if err != nil { + return nil, errors.Wrap(err, "GetRows") + } + return rows, nil +} + +type SBigQueryField struct { + Type string `json:"type"` + Name string `json:"name"` + Mode string `json:"mode"` + + Fields []SBigQueryField `json:"fields"` +} + +type SBigQuerySchema struct { + Fields []SBigQueryField `json:"fields"` +} + +type SBigQueryJobReference struct { +} + +type SBigQueryResult struct { + CacheHit bool `json:"cacheHit"` + JobComplete bool `json:"jobComplete"` + JobReference SBigQueryJobReference `json:"jobReference"` + Kind string `json:"kind"` + Rows []jsonutils.JSONObject `json:"rows"` + Schema SBigQuerySchema `json:"schema"` + TotalBytesProcessed int64 `json:"totalBytesProcessed"` + totalRows int64 `json:"totalRows"` +} + +func (res SBigQueryResult) GetRows() ([]jsonutils.JSONObject, error) { + rows := make([]jsonutils.JSONObject, 0) + for _, r := range res.Rows { + row, err := res.Schema.Parse(r) + if err != nil { + return nil, errors.Wrap(err, "Schema.Parse") + } + rows = append(rows, jsonutils.Marshal(row)) + } + return rows, nil +} + +func (schema SBigQuerySchema) Parse(r jsonutils.JSONObject) (map[string]jsonutils.JSONObject, error) { + f := SBigQueryField{ + Type: "RECORD", + Mode: "NULLABLE", + Name: "", + Fields: schema.Fields, + } + nr := jsonutils.NewDict() + nr.Add(r, "v") + return f.Parse(nr, "") +} + +func (f SBigQueryField) Parse(r jsonutils.JSONObject, prefix string) (map[string]jsonutils.JSONObject, error) { + if len(prefix) > 0 { + prefix = prefix + "." + f.Name + } else { + prefix = f.Name + } + ret := make(map[string]jsonutils.JSONObject) + switch f.Type { + case "RECORD": + if f.Mode == "REPEATED" { + items, err := r.GetArray("v") + if err != nil { + return nil, errors.Wrapf(err, "GetArray v %s", r) + } + nf := f + nf.Mode = "NULLABLE" + val := jsonutils.NewArray() + for _, item := range items { + obj, err := nf.Parse(item, "") + if err != nil { + return nil, errors.Wrap(err, "Parse items") + } + val.Add(jsonutils.Marshal(obj)) + } + ret[prefix] = val + } else { + items, err := r.GetArray("v", "f") + if err != nil { + return nil, errors.Wrap(err, "GetArray v.f") + } + if len(items) != len(f.Fields) { + return nil, errors.Wrap(errors.ErrServer, "inconsistent items and fields") + } + for i := range f.Fields { + res, err := f.Fields[i].Parse(items[i], prefix) + if err != nil { + return nil, errors.Wrapf(err, "Parse %s", prefix) + } + for k, v := range res { + ret[k] = v + } + } + } + default: + v, err := r.Get("v") + if err != nil { + return nil, errors.Wrap(err, "Get v") + } + ret[prefix] = v + } + return ret, nil +} diff --git a/pkg/multicloud/google/google.go b/pkg/multicloud/google/google.go index 8d1bd9fb52..fb40cacbe9 100644 --- a/pkg/multicloud/google/google.go +++ b/pkg/multicloud/google/google.go @@ -54,6 +54,8 @@ const ( GOOGLE_DBINSTANCE_API_VERSION = "v1beta4" GOOGLE_IAM_API_VERSION = "v1" + GOOGLE_BIGQUERY_API_VERSION = "v2" + GOOGLE_MANAGER_DOMAIN = "https://cloudresourcemanager.googleapis.com" GOOGLE_COMPUTE_DOMAIN = "https://www.googleapis.com/compute" GOOGLE_STORAGE_DOMAIN = "https://storage.googleapis.com/storage" @@ -63,6 +65,7 @@ const ( GOOGLE_MONITOR_DOMAIN = "https://monitoring.googleapis.com" GOOGLE_DBINSTANCE_DOMAIN = "https://www.googleapis.com/sql" GOOGLE_IAM_DOMAIN = "https://iam.googleapis.com" + GOOGLE_BIGQUERY_DOMAIN = "https://bigquery.googleapis.com/bigquery" MAX_RETRY = 3 ) diff --git a/pkg/multicloud/google/shell/bigquery.go b/pkg/multicloud/google/shell/bigquery.go new file mode 100644 index 0000000000..ce04a6d952 --- /dev/null +++ b/pkg/multicloud/google/shell/bigquery.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type BigQueryOptions struct { + SQL string `help:"sql to query"` + } + shellutils.R(&BigQueryOptions{}, "gcp-bigquery", "Exec bigquery", func(cli *google.SRegion, args *BigQueryOptions) error { + results, err := cli.BigQuery(args.SQL) + if err != nil { + return err + } + + printList(results, 0, 0, 0, nil) + return nil + }) +}