opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: convert clickhouse sqlstr from v1 to v2 (#14553)

Browse Source 
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2022-07-02 07:57:05 +08:00
committed by GitHub
parent 5b0338f543
commit 7ea8d33841
3 changed files with 239 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cloudcommon/clickhouse.go
+107
View File
@@ -0,0 +1,107 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudcommon
import (
"fmt"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/httperrors"
)
// convert clickhouse sqlstr v1 to v2
// v1: tcp://192.168.222.4:9000?database=yunionmeter&read_timeout=10&write_timeout=20
// v2: clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60
func clickhouseSqlStrV1ToV2(sqlstr string) (string, error) {
if strings.HasPrefix(sqlstr, "clickhouse://") {
// already v2 format
return sqlstr, nil
}
queryPos := strings.IndexByte(sqlstr, '?')
if queryPos <= 0 {
return "", errors.Wrap(httperrors.ErrInputParameter, "no query string")
}
hostPart := sqlstr[len("tcp://"):queryPos]
qs, err := jsonutils.ParseQueryString(sqlstr[queryPos+1:])
if err != nil {
return "", errors.Wrap(err, "ParseQueryString")
}
dbname, _ := qs.GetString("database")
if len(dbname) == 0 {
return "", errors.Wrap(httperrors.ErrInputParameter, "empty database")
}
return fmt.Sprintf("clickhouse://%s/%s?dial_timeout=200ms&max_execution_time=60", hostPart, dbname), nil
}
func clickhouseSqlStrV2ToV1(sqlstr string) (string, error) {
if strings.HasPrefix(sqlstr, "tcp://") {
// already v1 format
return sqlstr, nil
}
queryPos := strings.IndexByte(sqlstr, '?')
if queryPos <= 0 {
return "", errors.Wrap(httperrors.ErrInputParameter, "no query string")
}
hostPart := sqlstr[len("clickhouse://"):queryPos]
slashPos := strings.IndexByte(hostPart, '/')
if slashPos <= 0 {
return "", errors.Wrap(httperrors.ErrInputParameter, "no database part")
}
dbname := hostPart[slashPos+1:]
hostPart = hostPart[:slashPos]
return fmt.Sprintf("tcp://%s?database=%s&read_timeout=10&write_timeout=20", hostPart, dbname), nil
}
func validateClickhouseV2Str(sqlstr string) error {
if !strings.HasPrefix(sqlstr, "clickhouse://") {
return errors.Wrapf(httperrors.ErrInputParameter, "must start with clickhouse://")
}
qsPos := strings.IndexByte(sqlstr, '?')
if qsPos >= 0 {
sqlstr = sqlstr[:qsPos]
}
slashPos := strings.IndexByte(sqlstr, '/')
if slashPos <= 0 {
return errors.Wrapf(httperrors.ErrInputParameter, "missing db slash")
}
dbName := sqlstr[slashPos+1:]
if len(dbName) == 0 {
return errors.Wrapf(httperrors.ErrInputParameter, "empty database name")
}
return nil
}
func validateClickhouseV1Str(sqlstr string) error {
if !strings.HasPrefix(sqlstr, "tcp://") {
return errors.Wrapf(httperrors.ErrInputParameter, "must start with tcp://")
}
qsPos := strings.IndexByte(sqlstr, '?')
if qsPos <= 0 {
return errors.Wrapf(httperrors.ErrInputParameter, "mising query string")
}
qsPart := sqlstr[qsPos+1:]
qs, err := jsonutils.ParseQueryString(qsPart)
if err != nil {
return errors.Wrap(err, "ParseQueryString")
}
dbName, _ := qs.GetString("database")
if len(dbName) == 0 {
return errors.Wrapf(httperrors.ErrInputParameter, "empty database name")
}
return nil
}
pkg/cloudcommon/clickhouse_test.go
+119
View File
@@ -0,0 +1,119 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudcommon
import (
"testing"
)
func TestClickhouseSqlStrV1ToV2(t *testing.T) {
for _, c := range []struct {
in string
want string
}{
{
in: "tcp://192.168.222.4:9000?database=yunionmeter&read_timeout=10&write_timeout=20",
want: "clickhouse://192.168.222.4:9000/yunionmeter?dial_timeout=200ms&max_execution_time=60",
},
} {
got, err := clickhouseSqlStrV1ToV2(c.in)
if err != nil {
t.Errorf("%s", err)
} else if got != c.want {
t.Errorf("got %s want %s", got, c.want)
}
}
}
func TestClickhouseSqlStrV2ToV1(t *testing.T) {
for _, c := range []struct {
in string
want string
}{
{
in: "clickhouse://192.168.222.4:9000/yunionmeter?dial_timeout=200ms&max_execution_time=60",
want: "tcp://192.168.222.4:9000?database=yunionmeter&read_timeout=10&write_timeout=20",
},
} {
got, err := clickhouseSqlStrV2ToV1(c.in)
if err != nil {
t.Errorf("%s", err)
} else if got != c.want {
t.Errorf("got %s want %s", got, c.want)
}
}
}
func TestValidateClickhouseSqlstrV1(t *testing.T) {
for _, c := range []struct {
in string
valid bool
}{
{
in: "",
valid: false,
},
{
in: "tcp://192.168.222.4:9000?read_timeout=10&write_timeout=20",
valid: false,
},
{
in: "tcp://192.168.222.4:9000?database=yunionmeter&read_timeout=10&write_timeout=20",
valid: true,
},
{
in: "clickhouse://username:password@host1:9000,host2:9000?dial_timeout=200ms&max_execution_time=60",
valid: false,
},
{
in: "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60",
valid: false,
},
} {
err := validateClickhouseV1Str(c.in)
if err != nil && c.valid {
t.Errorf("%s", err)
}
}
}
func TestValidateClickhouseSqlstrV2(t *testing.T) {
for _, c := range []struct {
in string
valid bool
}{
{
in: "",
valid: false,
},
{
in: "tcp://192.168.222.4:9000?database=yunionmeter&read_timeout=10&write_timeout=20",
valid: false,
},
{
in: "clickhouse://username:password@host1:9000,host2:9000?dial_timeout=200ms&max_execution_time=60",
valid: false,
},
{
in: "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60",
valid: true,
},
} {
err := validateClickhouseV2Str(c.in)
if err != nil && c.valid {
t.Errorf("%s", err)
}
}
}
pkg/cloudcommon/database.go
+13 -1
View File
@@ -59,7 +59,8 @@ func InitDB(options *common_options.DBOptions) {
log.Fatalf("Invalid SqlConnection string: %s error: %v", options.SqlConnection, err)
}
backend := sqlchemy.MySQLBackend
if dialect == "sqlite3" {
switch dialect {
case "sqlite3":
backend = sqlchemy.SQLiteBackend
dialect = "sqlite3_with_extensions"
sql.Register(dialect,
@@ -69,6 +70,8 @@ func InitDB(options *common_options.DBOptions) {
},
},
)
case "clickhouse":
log.Fatalf("cannot use clickhouse as primary database")
}
log.Infof("database dialect: %s sqlStr: %s", dialect, sqlStr)
dbConn, err := sql.Open(dialect, sqlStr)
@@ -80,6 +83,15 @@ func InitDB(options *common_options.DBOptions) {
dialect, sqlStr, err = options.GetClickhouseConnStr()
if err == nil {
// connect to clickcloud
// force convert sqlstr from clickhouse v2 to v1
sqlStr, err = clickhouseSqlStrV2ToV1(sqlStr)
if err != nil {
log.Fatalf("fail to convert clickhouse sqlstr from v2 to v1: %s", err)
}
err = validateClickhouseV1Str(sqlStr)
if err != nil {
log.Fatalf("invalid clickhouse sqlstr: %s", err)
}
click, err := sql.Open(dialect, sqlStr)
if err != nil {
panic(err)