From df4321d9344c4043254546be7c018c96e0eb33fa Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Wed, 22 Jun 2022 06:35:33 +0800 Subject: [PATCH] fix: clickhouse set max idle connections (#14496) Co-authored-by: Qiu Jian --- go.mod | 2 +- go.sum | 4 +-- pkg/cloudcommon/database.go | 6 ---- pkg/cloudcommon/db/database.go | 31 ++++++++++++++++--- vendor/modules.txt | 2 +- .../backends/clickhouse/clickhouse.go | 18 ++++++++--- vendor/yunion.io/x/sqlchemy/sql.go | 8 +++-- 7 files changed, 49 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 210f59fda4..18eadd738e 100644 --- a/go.mod +++ b/go.mod @@ -156,7 +156,7 @@ require ( yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 yunion.io/x/pkg v1.0.0 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v1.1.2-0.20220611090616-7f6b1618bbc8 + yunion.io/x/sqlchemy v1.1.2-0.20220621143704-497d072a7a3b yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 ) diff --git a/go.sum b/go.sum index de91efea09..1e4e07501e 100644 --- a/go.sum +++ b/go.sum @@ -1139,7 +1139,7 @@ yunion.io/x/pkg v1.0.0/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= -yunion.io/x/sqlchemy v1.1.2-0.20220611090616-7f6b1618bbc8 h1:rmEVqdGJEApb1zNprsU409ZeGEDBlYnw8Peq95tnNm4= -yunion.io/x/sqlchemy v1.1.2-0.20220611090616-7f6b1618bbc8/go.mod h1:LdxyRXxK6LklfUVW1my2fNapK1kcYRZh8nNWnILwHyM= +yunion.io/x/sqlchemy v1.1.2-0.20220621143704-497d072a7a3b h1:eRzBAa4KZrpKe0T5gY8QTXOnoWu8iv5GlmW5wBei89o= +yunion.io/x/sqlchemy v1.1.2-0.20220621143704-497d072a7a3b/go.mod h1:LdxyRXxK6LklfUVW1my2fNapK1kcYRZh8nNWnILwHyM= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/cloudcommon/database.go b/pkg/cloudcommon/database.go index 6dca212ed7..0fe38943be 100644 --- a/pkg/cloudcommon/database.go +++ b/pkg/cloudcommon/database.go @@ -36,12 +36,6 @@ import ( common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) -const ( - MIN_DB_CONN_MAX = 5 - - ClickhouseDB = sqlchemy.DBName("clickhosue_db") -) - func InitDB(options *common_options.DBOptions) { if options.DebugSqlchemy { log.Warningf("debug Sqlchemy is turned on") diff --git a/pkg/cloudcommon/db/database.go b/pkg/cloudcommon/db/database.go index dd41f8207b..805d617839 100644 --- a/pkg/cloudcommon/db/database.go +++ b/pkg/cloudcommon/db/database.go @@ -16,13 +16,16 @@ package db import ( "context" + "database/sql" "fmt" "net/http" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" ) @@ -33,7 +36,20 @@ const ( ) func AppDBInit(app *appsrv.Application) { - dbConn := sqlchemy.GetDB() + dbConn := sqlchemy.GetDefaultDB() + if dbConn != nil { + setDbConnection(dbConn.DB()) + } + dbConn = sqlchemy.GetDBWithName(ClickhouseDB) + if dbConn != nil { + setDbConnection(dbConn.DB()) + } + + app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats") + app.AddDefaultHandler("GET", "/db_stats/", DBStatsHandler, "db_stats_with_name") +} + +func setDbConnection(dbConn *sql.DB) { if dbConn != nil { connMax := appsrv.GetDBConnectionCount() if connMax < MIN_DB_CONN_MAX { @@ -43,15 +59,20 @@ func AppDBInit(app *appsrv.Application) { dbConn.SetMaxIdleConns(connMax) dbConn.SetMaxOpenConns(connMax*2 + 1) } - - app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats") } func DBStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + var dbname sqlchemy.DBName + params := appctx.AppContextParams(ctx) + if dbn, ok := params[""]; ok && strings.HasPrefix(dbn, "click") { + dbname = ClickhouseDB + } else { + dbname = sqlchemy.DefaultDB + } result := jsonutils.NewDict() - dbConn := sqlchemy.GetDB() + dbConn := sqlchemy.GetDBWithName(dbname) if dbConn != nil { - stats := dbConn.Stats() + stats := dbConn.DB().Stats() result.Add(jsonutils.Marshal(&stats), "db_stats") } fmt.Fprintf(w, result.String()) diff --git a/vendor/modules.txt b/vendor/modules.txt index 144a88839c..0b4e284341 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1227,7 +1227,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.2-0.20220611090616-7f6b1618bbc8 +# yunion.io/x/sqlchemy v1.1.2-0.20220621143704-497d072a7a3b yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends yunion.io/x/sqlchemy/backends/clickhouse diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go index 669fe4b23e..318324be03 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go @@ -117,16 +117,24 @@ func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string if len(orderbys) == 0 { orderbys = primaries } - if len(orderbys) > 0 { - createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", ")) - } else { - createSql += fmt.Sprintf("\nORDER BY tuple()") - } if len(partitions) > 0 { createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", ")) } if len(primaries) > 0 { createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", ")) + newOrderBys := make([]string, len(primaries)) + copy(newOrderBys, primaries) + for _, f := range orderbys { + if !utils.IsInStringArray(f, newOrderBys) { + newOrderBys = append(newOrderBys, f) + } + } + orderbys = newOrderBys + } + if len(orderbys) > 0 { + createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", ")) + } else { + createSql += fmt.Sprintf("\nORDER BY tuple()") } if ttlCol != nil { ttlCount, ttlUnit := ttlCol.GetTTL() diff --git a/vendor/yunion.io/x/sqlchemy/sql.go b/vendor/yunion.io/x/sqlchemy/sql.go index 92057aac13..70bd176c33 100644 --- a/vendor/yunion.io/x/sqlchemy/sql.go +++ b/vendor/yunion.io/x/sqlchemy/sql.go @@ -16,7 +16,6 @@ package sqlchemy import ( "database/sql" - "fmt" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -80,7 +79,8 @@ func GetDBWithName(name DBName) *SDatabase { if db, ok := _db_tbl[name]; ok { return db } - panic(fmt.Sprintf("no such database %s", name)) + // panic(fmt.Sprintf("no such database %s", name)) + return nil } type sDBReferer struct { @@ -190,3 +190,7 @@ func (db *SDatabase) TxExec(sqlstr string, vars ...interface{}) (sql.Result, err } return results[0].Result, results[0].Error } + +func (db *SDatabase) DB() *sql.DB { + return db.db +}