diff --git a/cmd/climc/shell/misc/version.go b/cmd/climc/shell/misc/version.go index 1384ec263a..2fa6f686f2 100644 --- a/cmd/climc/shell/misc/version.go +++ b/cmd/climc/shell/misc/version.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" modules "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" ) @@ -32,7 +33,7 @@ func init() { SERVICE string `help:"Service type"` } R(&VersionOptions{}, "version-show", "query backend service for its version", func(s *mcclient.ClientSession, args *VersionOptions) error { - body, err := modules.GetVersion(s, args.SERVICE) + body, err := modulebase.GetVersion(s, args.SERVICE) if err != nil { return err } @@ -40,6 +41,36 @@ func init() { return nil }) + type StatsOptions struct { + SERVICE string `help:"Service type"` + } + R(&StatsOptions{}, "api-stats-show", "query backend service for its stats", func(s *mcclient.ClientSession, args *StatsOptions) error { + body, err := modulebase.GetStats(s, "stats", args.SERVICE) + if err != nil { + return err + } + printObject(body) + return nil + }) + R(&StatsOptions{}, "db-stats-show", "query backend service for its db stats", func(s *mcclient.ClientSession, args *StatsOptions) error { + body, err := modulebase.GetStats(s, "db_stats", args.SERVICE) + if err != nil { + return err + } + stats, _ := body.Get("db_stats") + printObject(stats) + return nil + }) + R(&StatsOptions{}, "worker-stats-show", "query backend service for its worker stats", func(s *mcclient.ClientSession, args *StatsOptions) error { + body, err := modulebase.GetStats(s, "worker_stats", args.SERVICE) + if err != nil { + return err + } + data, _ := body.GetArray("workers") + printList(&modulebase.ListResult{Data: data}, nil) + return nil + }) + type VersionListOptions struct { } R(&VersionListOptions{}, "version-list", "query all backend service version", func(s *mcclient.ClientSession, args *VersionListOptions) error { diff --git a/go.mod b/go.mod index f66ce7f7cf..eccae617d9 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Azure/go-autorest/autorest v0.9.6 github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/ClickHouse/clickhouse-go v1.4.7 // indirect github.com/DataDog/dd-trace-go v0.6.1 // indirect github.com/DataDog/zstd v1.3.4 // indirect github.com/Masterminds/goutils v1.1.0 // indirect @@ -154,8 +155,8 @@ require ( yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 - yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f + yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada + yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce ) diff --git a/go.sum b/go.sum index 821e747b29..e44352f7c9 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.4.7 h1:NNZQmlW8dVxGn19pF65BmWr0vq8Pj5Iy8ykyBKhFCPw= +github.com/ClickHouse/clickhouse-go v1.4.7/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/DataDog/dd-trace-go v0.6.1 h1:nsZ2lohbSw1CKtfNRu3wPh1jFirv6XSz8vqNpuIYWbM= github.com/DataDog/dd-trace-go v0.6.1/go.mod h1:SmQTTcC37XMyEm75HV0AWiZIYxDiaNhRi49zorIpW+o= github.com/DataDog/zstd v1.3.4 h1:LAGHkXuvC6yky+C2CUG2tD7w8QlrUwpue8XwIh0X4AY= @@ -145,6 +147,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= @@ -162,6 +166,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -261,6 +267,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -378,6 +385,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= @@ -423,6 +431,7 @@ github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae h1:XoMPFIGibcPKgLr github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 h1:ttJD8hTqvrPEUBoAG5hJKbDOJ84u7zmbnZsUL4V9430= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8/go.mod h1:VXFH11P7fHn2iPBsfSW1JacR59rttTcafJnwYcI/IdY= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libvirt/libvirt-go-xml v5.2.0+incompatible h1:ALyVpa0/lFfFaUpb5l0fNBohufmQG6jsOGRCl/nKEi8= github.com/libvirt/libvirt-go-xml v5.2.0+incompatible/go.mod h1:oBlgD3xOA01ihiK5stbhFzvieyW+jVS6kbbsMVF623A= github.com/ma314smith/signedxml v0.0.0-20200410192636-c342a2d0ae60 h1:q5rqPuvxdOzg1NC6sls3XSpRokQ5KAmLIYdLW1/kyOo= @@ -443,6 +452,7 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.7.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20181127064339-e4f871175a2f h1:4P7Ul+TAnk92vTeVkXs6VLjmf1EhrYtDRa03PCYY6VM= @@ -941,11 +951,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f h1:maply0f3AsvoXqkpTKkb6/tZFbRAT/rdkk1Cw3FkoB8= -yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= +yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg= +yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada h1:ZIhSSwFHfnzPMKSyiN0nkGwvC6RUeuCIAnWuTtDS900= -yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1 h1:WxvTLANfa01EFTORGY2GapbdjQkuLaQ/CwMDdBy632U= +yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/ansibleserver/service/service.go b/pkg/ansibleserver/service/service.go index 4c891dcafe..3e9950f8f7 100644 --- a/pkg/ansibleserver/service/service.go +++ b/pkg/ansibleserver/service/service.go @@ -17,9 +17,8 @@ package service import ( "os" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" "yunion.io/x/onecloud/pkg/ansibleserver/models" "yunion.io/x/onecloud/pkg/ansibleserver/options" @@ -43,9 +42,12 @@ func StartService() { models.InitPlaybookWorker() app := common_app.InitApp(baseOpts, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() common_app.ServeForever(app, baseOpts) diff --git a/pkg/appsrv/workers.go b/pkg/appsrv/workers.go index b8cef02f3e..197ea6a0b4 100644 --- a/pkg/appsrv/workers.go +++ b/pkg/appsrv/workers.go @@ -281,6 +281,8 @@ type SWorkerManagerStates struct { MaxWorkerCnt int ActiveWorkerCnt int DetachWorkerCnt int + DbWorker bool + AllowOverflow bool } func (s SWorkerManagerStates) IsBusy() bool { @@ -299,6 +301,8 @@ func (wm *SWorkerManager) getState() SWorkerManagerStates { state.MaxWorkerCnt = wm.workerCount state.ActiveWorkerCnt = wm.activeWorker.size() state.DetachWorkerCnt = wm.detachedWorker.size() + state.DbWorker = wm.dbWorker + state.AllowOverflow = wm.ignoreOverflow return state } diff --git a/pkg/cloudcommon/database.go b/pkg/cloudcommon/database.go index 9cee1045a2..d23ddeafa3 100644 --- a/pkg/cloudcommon/database.go +++ b/pkg/cloudcommon/database.go @@ -36,6 +36,8 @@ import ( const ( MIN_DB_CONN_MAX = 5 + + ClickhouseDB = sqlchemy.DBName("clickhosue_db") ) func InitDB(options *common_options.DBOptions) { @@ -56,11 +58,26 @@ func InitDB(options *common_options.DBOptions) { if err != nil { log.Fatalf("Invalid SqlConnection string: %s error: %v", options.SqlConnection, err) } + log.Infof("database dialect: %s sqlStr: %s", dialect, sqlStr) dbConn, err := sql.Open(dialect, sqlStr) if err != nil { panic(err) } - sqlchemy.SetDB(dbConn) + backend := sqlchemy.MySQLBackend + if dialect == "sqlite3" { + backend = sqlchemy.SQLiteBackend + } + sqlchemy.SetDBWithNameBackend(dbConn, sqlchemy.DefaultDB, backend) + + dialect, sqlStr, err = options.GetClickhouseConnStr() + if err == nil { + // connect to clickcloud + click, err := sql.Open(dialect, sqlStr) + if err != nil { + panic(err) + } + sqlchemy.SetDBWithNameBackend(click, ClickhouseDB, sqlchemy.ClickhouseBackend) + } switch options.LockmanMethod { case common_options.LockMethodInMemory, "": diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 119701880a..644c0b3d1e 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -145,7 +145,7 @@ func listItemsQueryByColumn(manager IModelManager, q *sqlchemy.SQuery, userCred } } else if len(arrV) > 1 { for i := range arrV { - arrV[i] = colSpec.ConvertFromString(arrV[i]) + arrV[i] = sqlchemy.GetStringValue(colSpec.ConvertFromString(arrV[i])) } q = q.In(fn, arrV) } else if len(arrV) == 1 { @@ -691,15 +691,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok if orderBy == nil { orderBy = []string{} } - if primaryCol != nil && primaryCol.IsNumeric() { - orderBy = append(orderBy, primaryCol.Name()) - } else if manager.TableSpec().ColumnSpec("created_at") != nil { - orderBy = append(orderBy, "created_at") - if manager.TableSpec().ColumnSpec("name") != nil { - orderBy = append(orderBy, "name") - } - if primaryCol != nil { + if !q.IsGroupBy() { + if primaryCol != nil && primaryCol.IsNumeric() { orderBy = append(orderBy, primaryCol.Name()) + } else if manager.TableSpec().ColumnSpec("created_at") != nil { + orderBy = append(orderBy, "created_at") + if manager.TableSpec().ColumnSpec("name") != nil { + orderBy = append(orderBy, "name") + } + if primaryCol != nil { + orderBy = append(orderBy, primaryCol.Name()) + } } } for _, orderByField := range orderBy { diff --git a/pkg/cloudcommon/db/i18n.go b/pkg/cloudcommon/db/i18n.go index 769897502d..a825c893a1 100644 --- a/pkg/cloudcommon/db/i18n.go +++ b/pkg/cloudcommon/db/i18n.go @@ -17,9 +17,8 @@ package db import ( "context" - "github.com/pkg/errors" - "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/compare" "yunion.io/x/onecloud/pkg/i18n" @@ -272,16 +271,20 @@ func (manager *SI18nManager) newFromI18n(ctx context.Context, userCred mcclient. } func (self *SI18n) updateFromI18n(ctx context.Context, userCred mcclient.TokenCredential, entry IModelI18nEntry) error { - _, err := Update(self, func() error { - self.KeyValue = entry.GetKeyValue() - self.Cn = entry.Lookup(i18n.I18N_TAG_CHINESE) - self.En = entry.Lookup(i18n.I18N_TAG_ENGLISH) - - return nil - }) - if err != nil { - log.Infof("updateFromI18n error %s", err) - return err + zh := entry.Lookup(i18n.I18N_TAG_CHINESE) + en := entry.Lookup(i18n.I18N_TAG_ENGLISH) + log.Debugf("updateFromI18n %#v %s %s", self, zh, en) + if self.Cn != zh || self.En != en { + _, err := Update(self, func() error { + self.KeyValue = entry.GetKeyValue() + self.Cn = zh + self.En = en + return nil + }) + if err != nil { + log.Infof("updateFromI18n error %s", err) + return err + } } return nil diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index a8447e5079..4913d1c051 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -53,11 +53,19 @@ type SModelBaseManager struct { } func NewModelBaseManager(model interface{}, tableName string, keyword string, keywordPlural string) SModelBaseManager { - return NewModelBaseManagerWithSplitable(model, tableName, keyword, keywordPlural, "", "", 0, 0) + return NewModelBaseManagerWithDBName(model, tableName, keyword, keywordPlural, sqlchemy.DefaultDB) +} + +func NewModelBaseManagerWithDBName(model interface{}, tableName string, keyword string, keywordPlural string, dbName sqlchemy.DBName) SModelBaseManager { + return NewModelBaseManagerWithSplitableDBName(model, tableName, keyword, keywordPlural, "", "", 0, 0, dbName) } func NewModelBaseManagerWithSplitable(model interface{}, tableName string, keyword string, keywordPlural string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) SModelBaseManager { - ts := newTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments) + return NewModelBaseManagerWithSplitableDBName(model, tableName, keyword, keywordPlural, indexField, dateField, maxDuration, maxSegments, sqlchemy.DefaultDB) +} + +func NewModelBaseManagerWithSplitableDBName(model interface{}, tableName string, keyword string, keywordPlural string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) SModelBaseManager { + ts := newTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments, dbName) modelMan := SModelBaseManager{tableSpec: ts, keyword: keyword, keywordPlural: keywordPlural} return modelMan } diff --git a/pkg/cloudcommon/db/models.go b/pkg/cloudcommon/db/models.go index 3960ca31cb..f1ed269ee9 100644 --- a/pkg/cloudcommon/db/models.go +++ b/pkg/cloudcommon/db/models.go @@ -138,8 +138,8 @@ func CheckSync(autoSync bool) bool { return inSync } -func EnsureAppInitSyncDB(app *appsrv.Application, opt *common_options.DBOptions, modelInitDBFunc func() error) { - cloudcommon.InitDB(opt) +func EnsureAppSyncDB(app *appsrv.Application, opt *common_options.DBOptions, modelInitDBFunc func() error) { + // cloudcommon.InitDB(opt) if !CheckSync(opt.AutoSyncTable) { log.Fatalf("database schema not in sync!") diff --git a/pkg/cloudcommon/db/opslog.go b/pkg/cloudcommon/db/opslog.go index 2be61a948f..bfec09aa49 100644 --- a/pkg/cloudcommon/db/opslog.go +++ b/pkg/cloudcommon/db/opslog.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/stringutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis" @@ -77,6 +78,7 @@ var _ IModelManager = (*SOpsLogManager)(nil) var _ IModel = (*SOpsLog)(nil) var opslogQueryWorkerMan *appsrv.SWorkerManager +var opslogWriteWorkerMan *appsrv.SWorkerManager func init() { OpsLog = &SOpsLogManager{NewModelBaseManagerWithSplitable( @@ -91,7 +93,8 @@ func init() { )} OpsLog.SetVirtualObject(OpsLog) - opslogQueryWorkerMan = appsrv.NewWorkerManager("opslog_query_worker", 2, 1024, true) + opslogQueryWorkerMan = appsrv.NewWorkerManager("opslog_query_worker", 2, 512, true) + opslogWriteWorkerMan = appsrv.NewWorkerManager("opslog_write_worker", 1, 2048, true) } func (manager *SOpsLogManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) { @@ -188,12 +191,20 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter } } - err := manager.TableSpec().Insert(context.Background(), opslog) + opslogWriteWorkerMan.Run(opslog, nil, nil) +} + +func (opslog *SOpsLog) Run() { + err := OpsLog.TableSpec().Insert(context.Background(), opslog) if err != nil { log.Errorf("fail to insert opslog: %s", err) } } +func (opslog *SOpsLog) Dump() string { + return fmt.Sprintf("[%s] %s %s", timeutils.CompactTime(opslog.OpsTime), opslog.Action, opslog.Notes) +} + func combineNotes(ctx context.Context, m2 IModel, notes jsonutils.JSONObject) *jsonutils.JSONDict { desc := m2.GetShortDesc(ctx) if notes != nil { diff --git a/pkg/cloudcommon/db/resourcebase_test.go b/pkg/cloudcommon/db/resourcebase_test.go index 83c0b4b001..7bac5523d9 100644 --- a/pkg/cloudcommon/db/resourcebase_test.go +++ b/pkg/cloudcommon/db/resourcebase_test.go @@ -18,6 +18,7 @@ import ( "testing" "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" @@ -32,6 +33,7 @@ func (tk *omniToken) IsAllow(scope rbacutils.TRbacScope, service string, resourc } func TestListFields(t *testing.T) { + sqlchemy.SetupMockDatabaseBackend() man := NewResourceBaseManager( &SResourceBase{}, "tbl", diff --git a/pkg/cloudcommon/db/tablespec.go b/pkg/cloudcommon/db/tablespec.go index 264c3f2f34..ff1d04101a 100644 --- a/pkg/cloudcommon/db/tablespec.go +++ b/pkg/cloudcommon/db/tablespec.go @@ -55,17 +55,19 @@ type sTableSpec struct { sqlchemy.ITableSpec } -func newTableSpec(model interface{}, tableName string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) ITableSpec { +func newTableSpec(model interface{}, tableName string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) ITableSpec { var itbl sqlchemy.ITableSpec if len(indexField) > 0 && len(dateField) > 0 { var err error - itbl, err = splitable.NewSplitTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments) + itbl, err = splitable.NewSplitTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments, dbName) if err != nil { log.Errorf("NewSplitTableSpec %s %s", tableName, err) return nil } else { log.Debugf("table %s maxDuration %d hour maxSegements %d", tableName, maxDuration/time.Hour, maxSegments) } + } else if len(dbName) > 0 { + itbl = sqlchemy.NewTableSpecFromStructWithDBName(model, tableName, dbName) } else { itbl = sqlchemy.NewTableSpecFromStruct(model, tableName) } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 8d37d262f0..3c30c390d3 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -592,12 +592,15 @@ func (self *STask) GetRequestContext() appctx.AppContextData { } func (self *STask) SaveRequestContext(data *appctx.AppContextData) { + jsonData := jsonutils.Marshal(data) + log.Debugf("SaveRequestContext %s param %s", jsonData, self.Params) _, err := db.Update(self, func() error { params := self.Params.CopyExcludes(REQUEST_CONTEXT_KEY) - params.Add(jsonutils.Marshal(data), REQUEST_CONTEXT_KEY) + params.Add(jsonData, REQUEST_CONTEXT_KEY) self.Params = params return nil }) + log.Debugf("Params: %s", self.Params) if err != nil { log.Errorf("save_request_context fail %s", err) } diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index 06f191b6ec..0a8654e57c 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "strings" "golang.org/x/net/http/httpproxy" "golang.org/x/text/language" @@ -38,6 +39,7 @@ import ( "yunion.io/x/structarg" "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/util/atexit" "yunion.io/x/onecloud/pkg/util/httputils" ) @@ -121,7 +123,7 @@ type CommonOptions struct { AdminDomain string `help:"Admin user domain" default:"Default"` AdminPassword string `help:"Admin password" alias:"admin-passwd"` AdminProject string `help:"Admin project" default:"system" alias:"admin-tenant-name"` - AdminProjectDomain string `help:"Domain of Admin project" default:"default"` + AdminProjectDomain string `help:"Domain of Admin project" default:"Default"` AuthTokenCacheSize uint32 `help:"Auth token Cache Size" default:"2048"` TenantCacheExpireSeconds int `help:"expire seconds of cached tenant/domain info. defailt 15 minutes" default:"900"` @@ -141,6 +143,8 @@ type HostCommonOptions struct { type DBOptions struct { SqlConnection string `help:"SQL connection string" alias:"connection"` + Clickhouse string `help:"Connection string for click house"` + AutoSyncTable bool `help:"Automatically synchronize table changes if differences are detected"` ExitAfterDBInit bool `help:"Exit program after db initialization" default:"false"` @@ -173,23 +177,23 @@ type EtcdOptions struct { EtcdKey string `help:"path to key file for connecting to etcd cluster"` } -func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) { +func (opt *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) { var ( cert tls.Certificate certLoaded bool capool *x509.CertPool ) - if this.EtcdCert != "" && this.EtcdKey != "" { + if opt.EtcdCert != "" && opt.EtcdKey != "" { var err error - cert, err = tls.LoadX509KeyPair(this.EtcdCert, this.EtcdKey) + cert, err = tls.LoadX509KeyPair(opt.EtcdCert, opt.EtcdKey) if err != nil { return nil, errors.Wrap(err, "load etcd cert and key") } certLoaded = true - this.EtcdUseTLS = true + opt.EtcdUseTLS = true } - if this.EtcdCacert != "" { - data, err := ioutil.ReadFile(this.EtcdCacert) + if opt.EtcdCacert != "" { + data, err := ioutil.ReadFile(opt.EtcdCacert) if err != nil { return nil, errors.Wrap(err, "read cacert file") } @@ -206,15 +210,15 @@ func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) { } capool.AddCert(cacert) } - this.EtcdUseTLS = true + opt.EtcdUseTLS = true } - if this.EtcdSkipTLSVerify { // it's false by default, true means user intends to use tls - this.EtcdUseTLS = true + if opt.EtcdSkipTLSVerify { // it's false by default, true means user intends to use tls + opt.EtcdUseTLS = true } - if this.EtcdUseTLS { + if opt.EtcdUseTLS { cfg := &tls.Config{ RootCAs: capool, - InsecureSkipVerify: this.EtcdSkipTLSVerify, + InsecureSkipVerify: opt.EtcdSkipTLSVerify, } if certLoaded { cfg.Certificates = []tls.Certificate{cert} @@ -224,8 +228,24 @@ func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) { return nil, nil } -func (this *DBOptions) GetDBConnection() (dialect, connstr string, err error) { - return utils.TransSQLAchemyURL(this.SqlConnection) +func (opt *DBOptions) GetDBConnection() (string, string, error) { + if strings.HasPrefix(opt.SqlConnection, "mysql") { + return utils.TransSQLAchemyURL(opt.SqlConnection) + } else { + pos := strings.Index(opt.SqlConnection, "://") + if pos > 0 { + return opt.SqlConnection[:pos], opt.SqlConnection[pos+3:], nil + } else { + return "", "", httperrors.ErrNotSupported + } + } +} + +func (opt *DBOptions) GetClickhouseConnStr() (string, string, error) { + if len(opt.Clickhouse) == 0 { + return "", "", errors.ErrNotFound + } + return "clickhouse", opt.Clickhouse, nil } func ParseOptions(optStruct interface{}, args []string, configFileName string, serviceType string) { diff --git a/pkg/cloudevent/service/service.go b/pkg/cloudevent/service/service.go index 6454ac9a10..02922db21e 100644 --- a/pkg/cloudevent/service/service.go +++ b/pkg/cloudevent/service/service.go @@ -18,9 +18,8 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/cloudevent" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -49,9 +48,10 @@ func StartService() { baseOpts := &opts.BaseOptions app := common_app.InitApp(baseOpts, false) + cloudcommon.InitDB(dbOpts) InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() if !opts.IsSlaveNode { diff --git a/pkg/cloudid/service/service.go b/pkg/cloudid/service/service.go index ba2f3b23f7..3b622322e9 100644 --- a/pkg/cloudid/service/service.go +++ b/pkg/cloudid/service/service.go @@ -19,6 +19,7 @@ import ( "time" "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/cloudid" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -47,9 +48,12 @@ func StartService() { common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange) app := common_app.InitApp(baseOpts, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() err := saml.InitSAML(app, api.SAML_IDP_PREFIX) diff --git a/pkg/cloudnet/service/service.go b/pkg/cloudnet/service/service.go index 63f9747fb0..3482c642f6 100644 --- a/pkg/cloudnet/service/service.go +++ b/pkg/cloudnet/service/service.go @@ -17,9 +17,8 @@ package service import ( "os" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" "yunion.io/x/onecloud/pkg/cloudcommon" common_app "yunion.io/x/onecloud/pkg/cloudcommon/app" @@ -42,9 +41,12 @@ func StartService() { baseOpts := &opts.BaseOptions app := common_app.InitApp(baseOpts, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() common_app.ServeForever(app, baseOpts) diff --git a/pkg/cloudproxy/service/service.go b/pkg/cloudproxy/service/service.go index 5d4cfc6eb6..2722a1efb9 100644 --- a/pkg/cloudproxy/service/service.go +++ b/pkg/cloudproxy/service/service.go @@ -15,7 +15,7 @@ package service import ( - _ "github.com/go-sql-driver/mysql" + _ "yunion.io/x/sqlchemy/backends" "yunion.io/x/onecloud/pkg/cloudcommon" common_app "yunion.io/x/onecloud/pkg/cloudcommon/app" @@ -32,9 +32,12 @@ func StartService() { ) app := common_app.InitApp(baseOpts, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() common_app.ServeForever(app, baseOpts) diff --git a/pkg/compute/models/billingresource.go b/pkg/compute/models/billingresource.go index 52300afa29..46912cbc24 100644 --- a/pkg/compute/models/billingresource.go +++ b/pkg/compute/models/billingresource.go @@ -254,7 +254,7 @@ func CheckBillingResourceExpireAt(ctx context.Context, userCred mcclient.TokenCr bq := BillingResourceCheckManager.Query("resource_id").Equals("resource_type", manager.Keyword()).Equals("advance_days", advanceDay).SubQuery() q = q.LeftJoin(bq, sqlchemy.Equals(q.Field("id"), bq.Field("resource_id"))) - q = q.IsNull("resource_id") + q = q.Filter(sqlchemy.IsNull(bq.Field("resource_id"))) vp := reflect.New(v.Type()) vp.Elem().Set(v) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index c06eead815..0ee6946321 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -1867,12 +1867,14 @@ func (account *SCloudaccount) markAllProvidersDicconnected(ctx context.Context, } func (account *SCloudaccount) markAccountConnected(ctx context.Context, userCred mcclient.TokenCredential) error { - _, err := db.UpdateWithLock(ctx, account, func() error { - account.ErrorCount = 0 - return nil - }) - if err != nil { - return err + if account.ErrorCount != 0 { + _, err := db.UpdateWithLock(ctx, account, func() error { + account.ErrorCount = 0 + return nil + }) + if err != nil { + return err + } } return account.SetStatus(userCred, api.CLOUD_PROVIDER_CONNECTED, "") } diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 66cf557c47..1de8745f8e 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -522,7 +522,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl return nil }) - if err != nil { + if err != nil && errors.Cause(err) != sqlchemy.ErrNoDataToUpdate { log.Errorf("syncWithCloudRegion %s", err) return err } diff --git a/pkg/compute/models/purge.go b/pkg/compute/models/purge.go index 938770acd3..e74971f0bd 100644 --- a/pkg/compute/models/purge.go +++ b/pkg/compute/models/purge.go @@ -390,14 +390,18 @@ func (lb *SLoadbalancer) purge(ctx context.Context, userCred mcclient.TokenCrede lb.DeletePreventionOff(lb, userCred) - _, err := db.UpdateWithLock(ctx, lb, func() error { - //避免 purge backendgroups 时循环依赖 - lb.BackendGroupId = "" - return nil - }) + var err error - if err != nil { - return fmt.Errorf("loadbalancer %s(%s): clear up backend group error: %v", lb.Name, lb.Id, err) + if lb.BackendGroupId != "" { + _, err = db.UpdateWithLock(ctx, lb, func() error { + //避免 purge backendgroups 时循环依赖 + lb.BackendGroupId = "" + return nil + }) + + if err != nil { + return fmt.Errorf("loadbalancer %s(%s): clear up backend group error: %v", lb.Name, lb.Id, err) + } } err = lb.detachAllNetworks(ctx, userCred) diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index d0bd8cd76e..68defac263 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -21,10 +21,9 @@ import ( "path/filepath" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" "yunion.io/x/pkg/errors" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -77,8 +76,11 @@ func StartService() { app := app_common.InitApp(baseOpts, true) + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() options.InitNameSyncResources() diff --git a/pkg/devtool/service/service.go b/pkg/devtool/service/service.go index 61cc5f9f8a..5e86609a04 100644 --- a/pkg/devtool/service/service.go +++ b/pkg/devtool/service/service.go @@ -17,9 +17,8 @@ package service import ( "os" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/devtool" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -51,8 +50,9 @@ func StartService() { } app := app_common.InitApp(&opts.BaseOptions, false) + cloudcommon.InitDB(dbOpts) InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) models.InitializeCronjobs() diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go index 8ffdda05c3..9e14c30305 100644 --- a/pkg/image/service/service.go +++ b/pkg/image/service/service.go @@ -21,9 +21,8 @@ import ( "strings" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/image" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -94,9 +93,12 @@ func StartService() { } app := app_common.InitApp(baseOpts, true) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange) diff --git a/pkg/keystone/models/domains.go b/pkg/keystone/models/domains.go index 2d2cd8d9d8..03c31e7681 100644 --- a/pkg/keystone/models/domains.go +++ b/pkg/keystone/models/domains.go @@ -142,6 +142,7 @@ func (manager *SDomainManager) FetchDomain(domainId string, domainName string) ( if len(domainId) == 0 && len(domainName) == 0 { domainId = api.DEFAULT_DOMAIN_ID } + log.Debugf("FetchDomain id %s name %s", domainId, domainName) if len(domainId) > 0 { return manager.FetchDomainById(domainId) } else { diff --git a/pkg/keystone/service/service.go b/pkg/keystone/service/service.go index 1a4a68a2ac..1a74bac44e 100644 --- a/pkg/keystone/service/service.go +++ b/pkg/keystone/service/service.go @@ -18,7 +18,7 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -72,8 +72,11 @@ func StartService() { app := app_common.InitApp(&opts.BaseOptions, true) + cloudcommon.InitDB(&opts.DBOptions) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, &opts.DBOptions, models.InitDB) + + db.EnsureAppSyncDB(app, &opts.DBOptions, models.InitDB) app_common.InitBaseAuth(&opts.BaseOptions) diff --git a/pkg/logger/service/service.go b/pkg/logger/service/service.go index 43de56d710..e48221ad21 100644 --- a/pkg/logger/service/service.go +++ b/pkg/logger/service/service.go @@ -17,9 +17,8 @@ package service import ( "os" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/logger" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -47,9 +46,12 @@ func StartService() { }) app := app_common.InitApp(baseOpts, true) + + cloudcommon.InitDB(dbOpts) + initHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() // models.StartNotifyToWebsocketWorker() diff --git a/pkg/mcclient/modulebase/stats.go b/pkg/mcclient/modulebase/stats.go new file mode 100644 index 0000000000..4c15c20127 --- /dev/null +++ b/pkg/mcclient/modulebase/stats.go @@ -0,0 +1,39 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modulebase + +import ( + "io/ioutil" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +// path: stats, db_stats, worker_stats +func GetStats(s *mcclient.ClientSession, path string, serviceType string) (jsonutils.JSONObject, error) { + man := NewBaseManager(serviceType, "", "", nil, nil) + resp, err := man.rawBaseUrlRequest(s, "GET", "/"+path, nil, nil) + if err != nil { + return nil, errors.Wrap(err, "man.rawBaseUrlRequest") + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "ioutil.ReadAll") + } + return jsonutils.Parse(body) +} diff --git a/pkg/monitor/service/service.go b/pkg/monitor/service/service.go index bbe94411a5..0600a29c6c 100644 --- a/pkg/monitor/service/service.go +++ b/pkg/monitor/service/service.go @@ -19,10 +19,10 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" "golang.org/x/sync/errgroup" "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" "yunion.io/x/onecloud/pkg/cloudcommon" common_app "yunion.io/x/onecloud/pkg/cloudcommon/app" @@ -56,9 +56,12 @@ func StartService() { baseOpts := &opts.BaseOptions app := common_app.InitApp(baseOpts, false) + + cloudcommon.InitDB(dbOpts) + InitHandlers(app) - db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) defer cloudcommon.CloseDB() go startServices() diff --git a/pkg/notify/service/service.go b/pkg/notify/service/service.go index 353953eab2..ee1b80e967 100644 --- a/pkg/notify/service/service.go +++ b/pkg/notify/service/service.go @@ -18,9 +18,8 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" api "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/cloudcommon" @@ -52,10 +51,13 @@ func StartService() { // init handler applicaion := app.InitApp(baseOpts, true) + + cloudcommon.InitDB(dbOpts) + InitHandlers(applicaion) // init database - db.EnsureAppInitSyncDB(applicaion, dbOpts, models.InitDB) + db.EnsureAppSyncDB(applicaion, dbOpts, models.InitDB) defer cloudcommon.CloseDB() err := models.ReceiverManager.StartWatchUserInKeystone() diff --git a/pkg/scheduledtask/models/scheduled_tasks.go b/pkg/scheduledtask/models/scheduled_tasks.go index 090f9fc91d..d81bd4a8c1 100644 --- a/pkg/scheduledtask/models/scheduled_tasks.go +++ b/pkg/scheduledtask/models/scheduled_tasks.go @@ -551,7 +551,7 @@ func (stm *SScheduledTaskManager) Timer(ctx context.Context, userCred mcclient.T if timerQueue == nil { timerQueue = make(chan struct{}, sop.Options.ScheduledTaskQueueSize) } - log.Infof("queueSize: %s", sop.Options.ScheduledTaskQueueSize) + log.Infof("queueSize: %d", sop.Options.ScheduledTaskQueueSize) // 60 is for fault tolerance interval := 60 + 30 timeScope := stm.timeScope(time.Now(), time.Duration(interval)*time.Second) diff --git a/pkg/scheduledtask/service/service.go b/pkg/scheduledtask/service/service.go index 0eb32724f3..000a9709fc 100644 --- a/pkg/scheduledtask/service/service.go +++ b/pkg/scheduledtask/service/service.go @@ -42,9 +42,12 @@ func StartService() { }) applicaion := app.InitApp(baseOpts, true) + + cloudcommon.InitDB(dbOpts) + InitHandlers(applicaion) - db.EnsureAppInitSyncDB(applicaion, dbOpts, nil) + db.EnsureAppSyncDB(applicaion, dbOpts, nil) defer cloudcommon.CloseDB() cron := cronman.InitCronJobManager(true, 4) diff --git a/pkg/util/splitable/insert.go b/pkg/util/splitable/insert.go index 87d2bd3c76..ab1651eb69 100644 --- a/pkg/util/splitable/insert.go +++ b/pkg/util/splitable/insert.go @@ -67,7 +67,7 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error { if lastMeta.StartDate.IsZero() { indexCol := t.tableSpec.ColumnSpec(t.indexField) _, err = t.metaSpec.Update(&lastMeta, func() error { - lastMeta.Start = indexCol.(*sqlchemy.SIntegerColumn).AutoIncrementOffset + lastMeta.Start = indexCol.AutoIncrementOffset() lastMeta.StartDate = lastDate return nil }) diff --git a/pkg/util/splitable/splitable.go b/pkg/util/splitable/splitable.go index 9bc902066c..f32c0db30f 100644 --- a/pkg/util/splitable/splitable.go +++ b/pkg/util/splitable/splitable.go @@ -47,6 +47,10 @@ func (t *SSplitTableSpec) Name() string { return t.tableName } +func (t *SSplitTableSpec) Database() *sqlchemy.SDatabase { + return t.metaSpec.Database() +} + func (t *SSplitTableSpec) Columns() []sqlchemy.IColumnSpec { return t.tableSpec.Columns() } @@ -116,29 +120,27 @@ func (t *SSplitTableSpec) Fetch(dt interface{}) error { return sql.ErrNoRows } -func NewSplitTableSpec(s interface{}, name string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) (*SSplitTableSpec, error) { - spec := sqlchemy.NewTableSpecFromStruct(s, name) - indexCol := spec.ColumnSpec(indexField) +func NewSplitTableSpec(s interface{}, name string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) (*SSplitTableSpec, error) { + spec := sqlchemy.NewTableSpecFromStructWithDBName(s, name, dbName) + /*indexCol := spec.ColumnSpec(indexField) if indexCol == nil { return nil, errors.Wrapf(errors.ErrNotFound, "indexField %s not found", indexField) } if !indexCol.IsPrimary() { return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not primary", indexField) } - if intCol, ok := indexCol.(*sqlchemy.SIntegerColumn); !ok { - return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not integer", indexField) - } else if !intCol.IsAutoIncrement { + if !indexCol.IsAutoIncrement() { return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not auto_increment", indexField) } dateCol := spec.ColumnSpec(dateField) if dateCol == nil { return nil, errors.Wrapf(errors.ErrNotFound, "dateField %s not found", dateField) } - if _, ok := dateCol.(*sqlchemy.SDateTimeColumn); !ok { + if !dateCol.IsDateTime() { return nil, errors.Wrapf(errors.ErrInvalidStatus, "dateField %s not datetime column", dateField) - } + }*/ - metaSpec := sqlchemy.NewTableSpecFromStruct(&STableMetadata{}, fmt.Sprintf("%s_metadata", name)) + metaSpec := sqlchemy.NewTableSpecFromStructWithDBName(&STableMetadata{}, fmt.Sprintf("%s_metadata", name), dbName) sts := &SSplitTableSpec{ indexField: indexField, diff --git a/pkg/util/splitable/sync.go b/pkg/util/splitable/sync.go index 815ae42faa..4b28884f2f 100644 --- a/pkg/util/splitable/sync.go +++ b/pkg/util/splitable/sync.go @@ -130,7 +130,7 @@ func (spec *SSplitTableSpec) SyncSQL() []string { now := time.Now() meta := STableMetadata{ Table: fmt.Sprintf("%s_%d", spec.tableName, now.Unix()), - Start: indexCol.(*sqlchemy.SIntegerColumn).AutoIncrementOffset, + Start: indexCol.AutoIncrementOffset(), } // insert the first meta sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now)) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/.gitignore b/vendor/github.com/ClickHouse/clickhouse-go/.gitignore new file mode 100644 index 0000000000..940583ffc4 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/.gitignore @@ -0,0 +1,28 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.out +*.exe +*.test +*.prof + +coverage.txt +.idea/** \ No newline at end of file diff --git a/vendor/github.com/ClickHouse/clickhouse-go/.travis.yml b/vendor/github.com/ClickHouse/clickhouse-go/.travis.yml new file mode 100644 index 0000000000..52b61d7cd0 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/.travis.yml @@ -0,0 +1,20 @@ +sudo: required +language: go +go: + - 1.15.x + - 1.16.x +go_import_path: github.com/ClickHouse/clickhouse-go +services: + - docker +install: + - export GO111MODULE="on" + - go mod vendor + +before_install: + - docker --version + - docker-compose --version + - docker-compose up -d +script: + - ./go.test.sh +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/CONTRIBUTING.md b/vendor/github.com/ClickHouse/clickhouse-go/CONTRIBUTING.md new file mode 100644 index 0000000000..d6b729c06f --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# Contributing notes + +## Local setup + +The easiest way to run tests is to use Docker Compose: + +``` +docker-compose up +make +``` diff --git a/vendor/github.com/ClickHouse/clickhouse-go/LICENSE b/vendor/github.com/ClickHouse/clickhouse-go/LICENSE new file mode 100644 index 0000000000..a860ed2071 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-2020 Kirill Shvakov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/ClickHouse/clickhouse-go/Makefile b/vendor/github.com/ClickHouse/clickhouse-go/Makefile new file mode 100644 index 0000000000..813b31965d --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/Makefile @@ -0,0 +1,8 @@ +test: + go install -race -v + go test -i -v + go test -race -timeout 30s -v . + +coverage: + go test -coverprofile=coverage.out -v . + go tool cover -html=coverage.out diff --git a/vendor/github.com/ClickHouse/clickhouse-go/README.md b/vendor/github.com/ClickHouse/clickhouse-go/README.md new file mode 100644 index 0000000000..67d6b4b090 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/README.md @@ -0,0 +1,302 @@ +# ClickHouse [![Build Status](https://travis-ci.org/ClickHouse/clickhouse-go.svg?branch=master)](https://travis-ci.org/ClickHouse/clickhouse-go) [![Go Report Card](https://goreportcard.com/badge/github.com/ClickHouse/clickhouse-go)](https://goreportcard.com/report/github.com/ClickHouse/clickhouse-go) [![codecov](https://codecov.io/gh/ClickHouse/clickhouse-go/branch/master/graph/badge.svg)](https://codecov.io/gh/ClickHouse/clickhouse-go) + +Golang SQL database driver for [Yandex ClickHouse](https://clickhouse.yandex/) + +## Key features + +* Uses native ClickHouse tcp client-server protocol +* Compatibility with `database/sql` +* Round Robin load-balancing +* Bulk write support : `begin->prepare->(in loop exec)->commit` +* LZ4 compression support (default to use pure go lz4, switch to use cgo lz4 by turn clz4 build tags on) +* External Tables support + +## DSN + +* username/password - auth credentials +* database - select the current default database +* read_timeout/write_timeout - timeout in second +* no_delay - disable/enable the Nagle Algorithm for tcp socket (default is 'true' - disable) +* alt_hosts - comma separated list of single address host for load-balancing +* connection_open_strategy - random/in_order (default random). + * random - choose random server from set + * in_order - first live server is choosen in specified order + * time_random - choose random(based on current time) server from set. This option differs from `random` in that randomness is based on current time rather than on amount of previous connections. +* block_size - maximum rows in block (default is 1000000). If the rows are larger then the data will be split into several blocks to send them to the server. If one block was sent to the server, the data will be persisted on the server disk, we can't rollback the transaction. So always keep in mind that the batch size no larger than the block_size if you want atomic batch insert. +* pool_size - maximum amount of preallocated byte chunks used in queries (default is 100). Decrease this if you experience memory problems at the expense of more GC pressure and vice versa. +* debug - enable debug output (boolean value) +* compress - enable lz4 compression (integer value, default is '0') + +SSL/TLS parameters: + +* secure - establish secure connection (default is false) +* skip_verify - skip certificate verification (default is false) +* tls_config - name of a TLS config with client certificates, registered using `clickhouse.RegisterTLSConfig()`; implies secure to be true, unless explicitly specified + +example: +``` +tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000 +``` + +## Supported data types + +* UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 +* Float32, Float64 +* String +* FixedString(N) +* Date +* DateTime +* IPv4 +* IPv6 +* Enum +* UUID +* Nullable(T) +* [Array(T)](https://clickhouse.yandex/reference_en.html#Array(T)) [godoc](https://godoc.org/github.com/ClickHouse/clickhouse-go#Array) +* Array(Nullable(T)) +* Tuple(...T) + +## TODO + +* Support other compression methods(zstd ...) + +## Install +``` +go get -u github.com/ClickHouse/clickhouse-go +``` + +## Example +```go +package main + +import ( + "database/sql" + "fmt" + "log" + "time" + + "github.com/ClickHouse/clickhouse-go" +) + +func main() { + connect, err := sql.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true") + if err != nil { + log.Fatal(err) + } + if err := connect.Ping(); err != nil { + if exception, ok := err.(*clickhouse.Exception); ok { + fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace) + } else { + fmt.Println(err) + } + return + } + + _, err = connect.Exec(` + CREATE TABLE IF NOT EXISTS example ( + country_code FixedString(2), + os_id UInt8, + browser_id UInt8, + categories Array(Int16), + action_day Date, + action_time DateTime + ) engine=Memory + `) + + if err != nil { + log.Fatal(err) + } + var ( + tx, _ = connect.Begin() + stmt, _ = tx.Prepare("INSERT INTO example (country_code, os_id, browser_id, categories, action_day, action_time) VALUES (?, ?, ?, ?, ?, ?)") + ) + defer stmt.Close() + + for i := 0; i < 100; i++ { + if _, err := stmt.Exec( + "RU", + 10+i, + 100+i, + clickhouse.Array([]int16{1, 2, 3}), + time.Now(), + time.Now(), + ); err != nil { + log.Fatal(err) + } + } + + if err := tx.Commit(); err != nil { + log.Fatal(err) + } + + rows, err := connect.Query("SELECT country_code, os_id, browser_id, categories, action_day, action_time FROM example") + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var ( + country string + os, browser uint8 + categories []int16 + actionDay, actionTime time.Time + ) + if err := rows.Scan(&country, &os, &browser, &categories, &actionDay, &actionTime); err != nil { + log.Fatal(err) + } + log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s", country, os, browser, categories, actionDay, actionTime) + } + + if err := rows.Err(); err != nil { + log.Fatal(err) + } + + if _, err := connect.Exec("DROP TABLE example"); err != nil { + log.Fatal(err) + } +} +``` + +Use [sqlx](https://github.com/jmoiron/sqlx) + +```go +package main + +import ( + "log" + "time" + + "github.com/jmoiron/sqlx" + _ "github.com/ClickHouse/clickhouse-go" +) + +func main() { + connect, err := sqlx.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true") + if err != nil { + log.Fatal(err) + } + var items []struct { + CountryCode string `db:"country_code"` + OsID uint8 `db:"os_id"` + BrowserID uint8 `db:"browser_id"` + Categories []int16 `db:"categories"` + ActionTime time.Time `db:"action_time"` + } + + if err := connect.Select(&items, "SELECT country_code, os_id, browser_id, categories, action_time FROM example"); err != nil { + log.Fatal(err) + } + + for _, item := range items { + log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_time: %s", item.CountryCode, item.OsID, item.BrowserID, item.Categories, item.ActionTime) + } +} +``` + +External tables support +```go +package main + +import ( + "database/sql" + "database/sql/driver" + "fmt" + "github.com/ClickHouse/clickhouse-go/lib/column" + "log" + "time" + + "github.com/ClickHouse/clickhouse-go" +) + +func main() { + connect, err := sql.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true") + if err != nil { + log.Fatal(err) + } + if err := connect.Ping(); err != nil { + if exception, ok := err.(*clickhouse.Exception); ok { + fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace) + } else { + fmt.Println(err) + } + return + } + + _, err = connect.Exec(` + CREATE TABLE IF NOT EXISTS example ( + country_code FixedString(2), + os_id UInt8, + browser_id UInt8, + categories Array(Int16), + action_day Date, + action_time DateTime + ) engine=Memory + `) + + if err != nil { + log.Fatal(err) + } + var ( + tx, _ = connect.Begin() + stmt, _ = tx.Prepare("INSERT INTO example (country_code, os_id, browser_id, categories, action_day, action_time) VALUES (?, ?, ?, ?, ?, ?)") + ) + defer stmt.Close() + + for i := 0; i < 100; i++ { + if _, err := stmt.Exec( + "RU", + 10+i, + 100+i, + clickhouse.Array([]int16{1, 2, 3}), + time.Now(), + time.Now(), + ); err != nil { + log.Fatal(err) + } + } + + if err := tx.Commit(); err != nil { + log.Fatal(err) + } + + col, err := column.Factory("country_code", "String", nil) + if err != nil { + log.Fatal(err) + } + countriesExternalTable := clickhouse.ExternalTable{ + Name: "countries", + Values: [][]driver.Value{ + {"RU"}, + }, + Columns: []column.Column{col}, + } + + rows, err := connect.Query("SELECT country_code, os_id, browser_id, categories, action_day, action_time "+ + "FROM example WHERE country_code IN ?", countriesExternalTable) + if err != nil { + log.Fatal(err) + } + defer rows.Close() + + for rows.Next() { + var ( + country string + os, browser uint8 + categories []int16 + actionDay, actionTime time.Time + ) + if err := rows.Scan(&country, &os, &browser, &categories, &actionDay, &actionTime); err != nil { + log.Fatal(err) + } + log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s", country, os, browser, categories, actionDay, actionTime) + } + + if err := rows.Err(); err != nil { + log.Fatal(err) + } + + if _, err := connect.Exec("DROP TABLE example"); err != nil { + log.Fatal(err) + } +} +``` \ No newline at end of file diff --git a/vendor/github.com/ClickHouse/clickhouse-go/array.go b/vendor/github.com/ClickHouse/clickhouse-go/array.go new file mode 100644 index 0000000000..997b8ab6a5 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/array.go @@ -0,0 +1,21 @@ +package clickhouse + +import ( + "time" +) + +func Array(v interface{}) interface{} { + return v +} + +func ArrayFixedString(len int, v interface{}) interface{} { + return v +} + +func ArrayDate(v []time.Time) interface{} { + return v +} + +func ArrayDateTime(v []time.Time) interface{} { + return v +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/bootstrap.go b/vendor/github.com/ClickHouse/clickhouse-go/bootstrap.go new file mode 100644 index 0000000000..f2d46540b3 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/bootstrap.go @@ -0,0 +1,245 @@ +package clickhouse + +import ( + "bufio" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "log" + "net/url" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +const ( + // DefaultDatabase when connecting to ClickHouse + DefaultDatabase = "default" + // DefaultUsername when connecting to ClickHouse + DefaultUsername = "default" + // DefaultConnTimeout when connecting to ClickHouse + DefaultConnTimeout = 5 * time.Second + // DefaultReadTimeout when reading query results + DefaultReadTimeout = time.Minute + // DefaultWriteTimeout when sending queries + DefaultWriteTimeout = time.Minute +) + +var ( + unixtime int64 + logOutput io.Writer = os.Stdout + hostname, _ = os.Hostname() + poolInit sync.Once +) + +func init() { + sql.Register("clickhouse", &bootstrap{}) + go func() { + for tick := time.Tick(time.Second); ; { + select { + case <-tick: + atomic.AddInt64(&unixtime, int64(time.Second)) + } + } + }() +} + +func now() time.Time { + return time.Unix(0, atomic.LoadInt64(&unixtime)) +} + +type bootstrap struct{} + +func (d *bootstrap) Open(dsn string) (driver.Conn, error) { + return Open(dsn) +} + +// SetLogOutput allows to change output of the default logger +func SetLogOutput(output io.Writer) { + logOutput = output +} + +// Open the connection +func Open(dsn string) (driver.Conn, error) { + clickhouse, err := open(dsn) + if err != nil { + return nil, err + } + + return clickhouse, err +} + +func open(dsn string) (*clickhouse, error) { + url, err := url.Parse(dsn) + if err != nil { + return nil, err + } + var ( + hosts = []string{url.Host} + query = url.Query() + secure = false + skipVerify = false + tlsConfigName = query.Get("tls_config") + noDelay = true + compress = false + database = query.Get("database") + username = query.Get("username") + password = query.Get("password") + blockSize = 1000000 + connTimeout = DefaultConnTimeout + readTimeout = DefaultReadTimeout + writeTimeout = DefaultWriteTimeout + connOpenStrategy = connOpenRandom + ) + if len(database) == 0 { + database = DefaultDatabase + } + if len(username) == 0 { + username = DefaultUsername + } + if v, err := strconv.ParseBool(query.Get("no_delay")); err == nil { + noDelay = v + } + tlsConfig := getTLSConfigClone(tlsConfigName) + if tlsConfigName != "" && tlsConfig == nil { + return nil, fmt.Errorf("invalid tls_config - no config registered under name %s", tlsConfigName) + } + secure = tlsConfig != nil + if v, err := strconv.ParseBool(query.Get("secure")); err == nil { + secure = v + } + if v, err := strconv.ParseBool(query.Get("skip_verify")); err == nil { + skipVerify = v + } + if duration, err := strconv.ParseFloat(query.Get("timeout"), 64); err == nil { + connTimeout = time.Duration(duration * float64(time.Second)) + } + if duration, err := strconv.ParseFloat(query.Get("read_timeout"), 64); err == nil { + readTimeout = time.Duration(duration * float64(time.Second)) + } + if duration, err := strconv.ParseFloat(query.Get("write_timeout"), 64); err == nil { + writeTimeout = time.Duration(duration * float64(time.Second)) + } + if size, err := strconv.ParseInt(query.Get("block_size"), 10, 64); err == nil { + blockSize = int(size) + } + if altHosts := strings.Split(query.Get("alt_hosts"), ","); len(altHosts) != 0 { + for _, host := range altHosts { + if len(host) != 0 { + hosts = append(hosts, host) + } + } + } + switch query.Get("connection_open_strategy") { + case "random": + connOpenStrategy = connOpenRandom + case "in_order": + connOpenStrategy = connOpenInOrder + case "time_random": + connOpenStrategy = connOpenTimeRandom + } + + settings, err := makeQuerySettings(query) + if err != nil { + return nil, err + } + + if v, err := strconv.ParseBool(query.Get("compress")); err == nil { + compress = v + } + + var ( + ch = clickhouse{ + logf: func(string, ...interface{}) {}, + settings: settings, + compress: compress, + blockSize: blockSize, + ServerInfo: data.ServerInfo{ + Timezone: time.Local, + }, + } + logger = log.New(logOutput, "[clickhouse]", 0) + ) + if debug, err := strconv.ParseBool(url.Query().Get("debug")); err == nil && debug { + ch.logf = logger.Printf + } + ch.logf("host(s)=%s, database=%s, username=%s", + strings.Join(hosts, ", "), + database, + username, + ) + options := connOptions{ + secure: secure, + tlsConfig: tlsConfig, + skipVerify: skipVerify, + hosts: hosts, + connTimeout: connTimeout, + readTimeout: readTimeout, + writeTimeout: writeTimeout, + noDelay: noDelay, + openStrategy: connOpenStrategy, + logf: ch.logf, + } + if ch.conn, err = dial(options); err != nil { + return nil, err + } + logger.SetPrefix(fmt.Sprintf("[clickhouse][connect=%d]", ch.conn.ident)) + ch.buffer = bufio.NewWriter(ch.conn) + + ch.decoder = binary.NewDecoderWithCompress(ch.conn) + ch.encoder = binary.NewEncoderWithCompress(ch.buffer) + + if err := ch.hello(database, username, password); err != nil { + ch.conn.Close() + return nil, err + } + return &ch, nil +} + +func (ch *clickhouse) hello(database, username, password string) error { + ch.logf("[hello] -> %s", ch.ClientInfo) + { + ch.encoder.Uvarint(protocol.ClientHello) + if err := ch.ClientInfo.Write(ch.encoder); err != nil { + return err + } + { + ch.encoder.String(database) + ch.encoder.String(username) + ch.encoder.String(password) + } + if err := ch.encoder.Flush(); err != nil { + return err + } + + } + { + packet, err := ch.decoder.Uvarint() + if err != nil { + return err + } + switch packet { + case protocol.ServerException: + return ch.exception() + case protocol.ServerHello: + if err := ch.ServerInfo.Read(ch.decoder); err != nil { + return err + } + case protocol.ServerEndOfStream: + ch.logf("[bootstrap] <- end of stream") + return nil + default: + return fmt.Errorf("[hello] unexpected packet [%d] from server", packet) + } + } + ch.logf("[hello] <- %s", ch.ServerInfo) + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse.go new file mode 100644 index 0000000000..f1d6d4e619 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse.go @@ -0,0 +1,341 @@ +package clickhouse + +import ( + "bufio" + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "net" + "reflect" + "regexp" + "sync" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" + "github.com/ClickHouse/clickhouse-go/lib/column" + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" + "github.com/ClickHouse/clickhouse-go/lib/types" +) + +type ( + Date = types.Date + DateTime = types.DateTime + UUID = types.UUID +) + +type ExternalTable struct { + Name string + Values [][]driver.Value + Columns []column.Column +} + +var ( + ErrInsertInNotBatchMode = errors.New("insert statement supported only in the batch mode (use begin/commit)") + ErrLimitDataRequestInTx = errors.New("data request has already been prepared in transaction") +) + +var ( + splitInsertRe = regexp.MustCompile(`(?i)\sVALUES\s*\(`) +) + +type logger func(format string, v ...interface{}) + +type clickhouse struct { + sync.Mutex + data.ServerInfo + data.ClientInfo + logf logger + conn *connect + block *data.Block + buffer *bufio.Writer + decoder *binary.Decoder + encoder *binary.Encoder + settings *querySettings + compress bool + blockSize int + inTransaction bool +} + +func (ch *clickhouse) Prepare(query string) (driver.Stmt, error) { + return ch.prepareContext(context.Background(), query) +} + +func (ch *clickhouse) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + return ch.prepareContext(ctx, query) +} + +func (ch *clickhouse) prepareContext(ctx context.Context, query string) (driver.Stmt, error) { + ch.logf("[prepare] %s", query) + switch { + case ch.conn.closed: + return nil, driver.ErrBadConn + case ch.block != nil: + return nil, ErrLimitDataRequestInTx + case isInsert(query): + if !ch.inTransaction { + return nil, ErrInsertInNotBatchMode + } + return ch.insert(ctx, query) + } + return &stmt{ + ch: ch, + query: query, + numInput: numInput(query), + }, nil +} + +func (ch *clickhouse) insert(ctx context.Context, query string) (_ driver.Stmt, err error) { + if err := ch.sendQuery(ctx, splitInsertRe.Split(query, -1)[0]+" VALUES ", nil); err != nil { + return nil, err + } + if ch.block, err = ch.readMeta(); err != nil { + return nil, err + } + return &stmt{ + ch: ch, + isInsert: true, + }, nil +} + +func (ch *clickhouse) Begin() (driver.Tx, error) { + return ch.beginTx(context.Background(), txOptions{}) +} + +func (ch *clickhouse) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + return ch.beginTx(ctx, txOptions{ + Isolation: int(opts.Isolation), + ReadOnly: opts.ReadOnly, + }) +} + +type txOptions struct { + Isolation int + ReadOnly bool +} + +func (ch *clickhouse) beginTx(ctx context.Context, opts txOptions) (*clickhouse, error) { + ch.logf("[begin] tx=%t, data=%t", ch.inTransaction, ch.block != nil) + switch { + case ch.inTransaction: + return nil, sql.ErrTxDone + case ch.conn.closed: + return nil, driver.ErrBadConn + } + if finish := ch.watchCancel(ctx); finish != nil { + defer finish() + } + ch.block = nil + ch.inTransaction = true + return ch, nil +} + +func (ch *clickhouse) Commit() error { + ch.logf("[commit] tx=%t, data=%t", ch.inTransaction, ch.block != nil) + defer func() { + if ch.block != nil { + ch.block.Reset() + ch.block = nil + } + ch.inTransaction = false + }() + switch { + case !ch.inTransaction: + return sql.ErrTxDone + case ch.conn.closed: + return driver.ErrBadConn + } + if ch.block != nil { + if err := ch.writeBlock(ch.block, ""); err != nil { + return err + } + // Send empty block as marker of end of data. + if err := ch.writeBlock(&data.Block{}, ""); err != nil { + return err + } + if err := ch.encoder.Flush(); err != nil { + return err + } + return ch.process() + } + return nil +} + +func (ch *clickhouse) Rollback() error { + ch.logf("[rollback] tx=%t, data=%t", ch.inTransaction, ch.block != nil) + if !ch.inTransaction { + return sql.ErrTxDone + } + if ch.block != nil { + ch.block.Reset() + } + ch.block = nil + ch.buffer = nil + ch.inTransaction = false + return ch.conn.Close() +} + +func (ch *clickhouse) CheckNamedValue(nv *driver.NamedValue) error { + switch nv.Value.(type) { + case ExternalTable, column.IP, column.UUID: + return nil + case nil, []byte, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64, string, time.Time: + return nil + } + switch v := nv.Value.(type) { + case + []int, []int8, []int16, []int32, []int64, + []uint, []uint8, []uint16, []uint32, []uint64, + []float32, []float64, + []string: + return nil + case net.IP, *net.IP: + return nil + case driver.Valuer: + value, err := v.Value() + if err != nil { + return err + } + nv.Value = value + default: + switch value := reflect.ValueOf(nv.Value); value.Kind() { + case reflect.Slice: + return nil + case reflect.Bool: + nv.Value = uint8(0) + if value.Bool() { + nv.Value = uint8(1) + } + case reflect.Int8: + nv.Value = int8(value.Int()) + case reflect.Int16: + nv.Value = int16(value.Int()) + case reflect.Int32: + nv.Value = int32(value.Int()) + case reflect.Int64: + nv.Value = value.Int() + case reflect.Uint8: + nv.Value = uint8(value.Uint()) + case reflect.Uint16: + nv.Value = uint16(value.Uint()) + case reflect.Uint32: + nv.Value = uint32(value.Uint()) + case reflect.Uint64: + nv.Value = uint64(value.Uint()) + case reflect.Float32: + nv.Value = float32(value.Float()) + case reflect.Float64: + nv.Value = float64(value.Float()) + case reflect.String: + nv.Value = value.String() + } + } + return nil +} + +func (ch *clickhouse) Close() error { + ch.block = nil + return ch.conn.Close() +} + +func (ch *clickhouse) process() error { + packet, err := ch.decoder.Uvarint() + if err != nil { + return err + } + for { + switch packet { + case protocol.ServerPong: + ch.logf("[process] <- pong") + return nil + case protocol.ServerException: + ch.logf("[process] <- exception") + return ch.exception() + case protocol.ServerProgress: + progress, err := ch.progress() + if err != nil { + return err + } + ch.logf("[process] <- progress: rows=%d, bytes=%d, total rows=%d", + progress.rows, + progress.bytes, + progress.totalRows, + ) + case protocol.ServerProfileInfo: + profileInfo, err := ch.profileInfo() + if err != nil { + return err + } + ch.logf("[process] <- profiling: rows=%d, bytes=%d, blocks=%d", profileInfo.rows, profileInfo.bytes, profileInfo.blocks) + case protocol.ServerData: + block, err := ch.readBlock() + if err != nil { + return err + } + ch.logf("[process] <- data: packet=%d, columns=%d, rows=%d", packet, block.NumColumns, block.NumRows) + case protocol.ServerEndOfStream: + ch.logf("[process] <- end of stream") + return nil + default: + ch.conn.Close() + return fmt.Errorf("[process] unexpected packet [%d] from server", packet) + } + if packet, err = ch.decoder.Uvarint(); err != nil { + return err + } + } +} + +func (ch *clickhouse) cancel() error { + ch.logf("[cancel request]") + // even if we fail to write the cancel, we still need to close + err := ch.encoder.Uvarint(protocol.ClientCancel) + if err == nil { + err = ch.encoder.Flush() + } + // return the close error if there was one, otherwise return the write error + if cerr := ch.conn.Close(); cerr != nil { + return cerr + } + return err +} + +func (ch *clickhouse) watchCancel(ctx context.Context) func() { + if done := ctx.Done(); done != nil { + finished := make(chan struct{}) + go func() { + select { + case <-done: + ch.cancel() + finished <- struct{}{} + ch.logf("[cancel] <- done") + case <-finished: + ch.logf("[cancel] <- finished") + } + }() + return func() { + select { + case <-finished: + case finished <- struct{}{}: + } + } + } + return func() {} +} + +func (ch *clickhouse) ExecContext(ctx context.Context, query string, + args []driver.NamedValue) (driver.Result, error) { + finish := ch.watchCancel(ctx) + defer finish() + stmt, err := ch.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + dargs := make([]driver.Value, len(args)) + for i, nv := range args { + dargs[i] = nv.Value + } + return stmt.Exec(dargs) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_exception.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_exception.go new file mode 100644 index 0000000000..2317d94377 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_exception.go @@ -0,0 +1,46 @@ +package clickhouse + +import ( + "fmt" + "strings" +) + +type Exception struct { + Code int32 + Name string + Message string + StackTrace string + nested error +} + +func (e *Exception) Error() string { + return fmt.Sprintf("code: %d, message: %s", e.Code, e.Message) +} + +func (ch *clickhouse) exception() error { + var ( + e Exception + err error + hasNested bool + ) + if e.Code, err = ch.decoder.Int32(); err != nil { + return err + } + if e.Name, err = ch.decoder.String(); err != nil { + return err + } + if e.Message, err = ch.decoder.String(); err != nil { + return err + } + e.Message = strings.TrimSpace(strings.TrimPrefix(e.Message, e.Name+":")) + if e.StackTrace, err = ch.decoder.String(); err != nil { + return err + } + if hasNested, err = ch.decoder.Bool(); err != nil { + return err + } + if hasNested { + e.nested = ch.exception() + } + return &e +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_ping.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_ping.go new file mode 100644 index 0000000000..9c6b50a2bf --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_ping.go @@ -0,0 +1,28 @@ +package clickhouse + +import ( + "context" + "database/sql/driver" + + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +func (ch *clickhouse) Ping(ctx context.Context) error { + return ch.ping(ctx) +} + +func (ch *clickhouse) ping(ctx context.Context) error { + if ch.conn.closed { + return driver.ErrBadConn + } + ch.logf("-> ping") + finish := ch.watchCancel(ctx) + defer finish() + if err := ch.encoder.Uvarint(protocol.ClientPing); err != nil { + return err + } + if err := ch.encoder.Flush(); err != nil { + return err + } + return ch.process() +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_profile_info.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_profile_info.go new file mode 100644 index 0000000000..97bc9c791c --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_profile_info.go @@ -0,0 +1,37 @@ +package clickhouse + +type profileInfo struct { + rows uint64 + bytes uint64 + blocks uint64 + appliedLimit bool + rowsBeforeLimit uint64 + calculatedRowsBeforeLimit bool +} + +func (ch *clickhouse) profileInfo() (*profileInfo, error) { + var ( + p profileInfo + err error + ) + if p.rows, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + if p.blocks, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + if p.bytes, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + + if p.appliedLimit, err = ch.decoder.Bool(); err != nil { + return nil, err + } + if p.rowsBeforeLimit, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + if p.calculatedRowsBeforeLimit, err = ch.decoder.Bool(); err != nil { + return nil, err + } + return &p, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_progress.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_progress.go new file mode 100644 index 0000000000..2ccc5fd942 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_progress.go @@ -0,0 +1,26 @@ +package clickhouse + +type progress struct { + rows uint64 + bytes uint64 + totalRows uint64 +} + +func (ch *clickhouse) progress() (*progress, error) { + var ( + p progress + err error + ) + if p.rows, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + if p.bytes, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + + if p.totalRows, err = ch.decoder.Uvarint(); err != nil { + return nil, err + } + + return &p, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_block.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_block.go new file mode 100644 index 0000000000..e9a3384dce --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_block.go @@ -0,0 +1,19 @@ +package clickhouse + +import ( + "github.com/ClickHouse/clickhouse-go/lib/data" +) + +func (ch *clickhouse) readBlock() (*data.Block, error) { + if _, err := ch.decoder.String(); err != nil { // temporary table + return nil, err + } + + ch.decoder.SelectCompress(ch.compress) + var block data.Block + if err := block.Read(&ch.ServerInfo, ch.decoder); err != nil { + return nil, err + } + ch.decoder.SelectCompress(false) + return &block, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_meta.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_meta.go new file mode 100644 index 0000000000..83e18dc146 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_meta.go @@ -0,0 +1,53 @@ +package clickhouse + +import ( + "fmt" + + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +func (ch *clickhouse) readMeta() (*data.Block, error) { + for { + packet, err := ch.decoder.Uvarint() + if err != nil { + return nil, err + } + + switch packet { + case protocol.ServerException: + ch.logf("[read meta] <- exception") + return nil, ch.exception() + case protocol.ServerProgress: + progress, err := ch.progress() + if err != nil { + return nil, err + } + ch.logf("[read meta] <- progress: rows=%d, bytes=%d, total rows=%d", + progress.rows, + progress.bytes, + progress.totalRows, + ) + case protocol.ServerProfileInfo: + profileInfo, err := ch.profileInfo() + if err != nil { + return nil, err + } + ch.logf("[read meta] <- profiling: rows=%d, bytes=%d, blocks=%d", profileInfo.rows, profileInfo.bytes, profileInfo.blocks) + case protocol.ServerData: + block, err := ch.readBlock() + if err != nil { + return nil, err + } + ch.logf("[read meta] <- data: packet=%d, columns=%d, rows=%d", packet, block.NumColumns, block.NumRows) + return block, nil + case protocol.ServerEndOfStream: + _, err := ch.readBlock() + ch.logf("[process] <- end of stream") + return nil, err + default: + ch.conn.Close() + return nil, fmt.Errorf("[read meta] unexpected packet [%d] from server", packet) + } + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_external_data.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_external_data.go new file mode 100644 index 0000000000..706a6c5856 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_external_data.go @@ -0,0 +1,35 @@ +package clickhouse + +import "github.com/ClickHouse/clickhouse-go/lib/data" + +func (ch *clickhouse) sendExternalTables(externalTables []ExternalTable) error { + ch.logf("[send external tables] count %d", len(externalTables)) + if externalTables == nil || len(externalTables) == 0 { + return nil + } + block := &data.Block{} + sentTables := make(map[string]bool, 0) + for _, externalTable := range externalTables { + if _, ok := sentTables[externalTable.Name]; ok { + continue + } + ch.logf("[send external table] name %s", externalTable.Name) + sentTables[externalTable.Name] = true + block.Columns = externalTable.Columns + block.NumColumns = uint64(len(externalTable.Columns)) + for _, row := range externalTable.Values { + err := block.AppendRow(row) + if err != nil { + return err + } + } + if err := ch.writeBlock(block, externalTable.Name); err != nil { + return err + } + if err := ch.encoder.Flush(); err != nil { + return err + } + block.Reset() + } + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_query.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_query.go new file mode 100644 index 0000000000..0717ed7797 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_query.go @@ -0,0 +1,71 @@ +package clickhouse + +import ( + "context" + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +func (ch *clickhouse) sendQuery(ctx context.Context, query string, externalTables []ExternalTable) error { + ch.logf("[send query] %s", query) + if err := ch.encoder.Uvarint(protocol.ClientQuery); err != nil { + return err + } + var queryID string + queryIDValue := ctx.Value(queryIDKey) + if queryIDValue != nil { + if queryIdStr, ok := queryIDValue.(string); ok { + queryID = queryIdStr + } + } + if err := ch.encoder.String(queryID); err != nil { + return err + } + { // client info + ch.encoder.Uvarint(1) + ch.encoder.String("") + ch.encoder.String("") + ch.encoder.String("[::ffff:127.0.0.1]:0") + ch.encoder.Uvarint(1) // iface type TCP + ch.encoder.String(hostname) + ch.encoder.String(hostname) + } + if err := ch.ClientInfo.Write(ch.encoder); err != nil { + return err + } + if ch.ServerInfo.Revision >= protocol.DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO { + ch.encoder.String("") + } + + // the settings are written as list of contiguous name-value pairs, finished with empty name + if !ch.settings.IsEmpty() { + ch.logf("[query settings] %s", ch.settings.settingsStr) + if err := ch.settings.Serialize(ch.encoder); err != nil { + return err + } + } + // empty string is a marker of the end of the settings + if err := ch.encoder.String(""); err != nil { + return err + } + if err := ch.encoder.Uvarint(protocol.StateComplete); err != nil { + return err + } + compress := protocol.CompressDisable + if ch.compress { + compress = protocol.CompressEnable + } + if err := ch.encoder.Uvarint(compress); err != nil { + return err + } + if err := ch.encoder.String(query); err != nil { + return err + } + if err := ch.sendExternalTables(externalTables); err != nil { + return err + } + if err := ch.writeBlock(&data.Block{}, ""); err != nil { + return err + } + return ch.encoder.Flush() +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_write_block.go b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_write_block.go new file mode 100644 index 0000000000..5a76f49799 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/clickhouse_write_block.go @@ -0,0 +1,40 @@ +package clickhouse + +import ( + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +func (ch *clickhouse) writeBlock(block *data.Block, tableName string) error { + ch.Lock() + defer ch.Unlock() + if err := ch.encoder.Uvarint(protocol.ClientData); err != nil { + return err + } + + if err := ch.encoder.String(tableName); err != nil { // temporary table + return err + } + + // implement CityHash v 1.0.2 and add LZ4 compression + /* + From Alexey Milovidov + Насколько я помню, сжимаются блоки с данными Native формата, а всё остальное (всякие номера пакетов и т. п.) передаётся без сжатия. + + Сжатые данные устроены так. Они представляют собой набор сжатых фреймов. + Каждый фрейм имеет следующий вид: + чексумма (16 байт), + идентификатор алгоритма сжатия (1 байт), + размер сжатых данных (4 байта, little endian, размер не включает в себя чексумму, но включает в себя остальные 9 байт заголовка), + размер несжатых данных (4 байта, little endian), затем сжатые данные. + Идентификатор алгоритма: 0x82 - lz4, 0x90 - zstd. + Чексумма - CityHash128 из CityHash версии 1.0.2, вычисленный от сжатых данных с учётом 9 байт заголовка. + + См. CompressedReadBufferBase, CompressedWriteBuffer, + utils/compressor, TCPHandler. + */ + ch.encoder.SelectCompress(ch.compress) + err := block.Write(&ch.ServerInfo, ch.encoder) + ch.encoder.SelectCompress(false) + return err +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/connect.go b/vendor/github.com/ClickHouse/clickhouse-go/connect.go new file mode 100644 index 0000000000..2fc6812832 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/connect.go @@ -0,0 +1,191 @@ +package clickhouse + +import ( + "bufio" + "crypto/tls" + "database/sql/driver" + "net" + "sync/atomic" + "time" +) + +var tick int32 + +type openStrategy int8 + +func (s openStrategy) String() string { + switch s { + case connOpenInOrder: + return "in_order" + case connOpenTimeRandom: + return "time_random" + } + return "random" +} + +const ( + connOpenRandom openStrategy = iota + 1 + connOpenInOrder + connOpenTimeRandom +) + +type connOptions struct { + secure, skipVerify bool + tlsConfig *tls.Config + hosts []string + connTimeout, readTimeout, writeTimeout time.Duration + noDelay bool + openStrategy openStrategy + logf func(string, ...interface{}) +} + +func dial(options connOptions) (*connect, error) { + var ( + err error + abs = func(v int) int { + if v < 0 { + return -1 * v + } + return v + } + conn net.Conn + ident = abs(int(atomic.AddInt32(&tick, 1))) + ) + tlsConfig := options.tlsConfig + if options.secure { + if tlsConfig == nil { + tlsConfig = &tls.Config{} + } + tlsConfig.InsecureSkipVerify = options.skipVerify + } + checkedHosts := make(map[int]struct{}, len(options.hosts)) + for i := range options.hosts { + var num int + switch options.openStrategy { + case connOpenInOrder: + num = i + case connOpenRandom: + num = (ident + i) % len(options.hosts) + case connOpenTimeRandom: + // select host based on milliseconds + num = int((time.Now().UnixNano()/1000)%1000) % len(options.hosts) + for _, ok := checkedHosts[num]; ok; _, ok = checkedHosts[num] { + num = int(time.Now().UnixNano()) % len(options.hosts) + } + checkedHosts[num] = struct{}{} + } + switch { + case options.secure: + conn, err = tls.DialWithDialer( + &net.Dialer{ + Timeout: options.connTimeout, + }, + "tcp", + options.hosts[num], + tlsConfig, + ) + default: + conn, err = net.DialTimeout("tcp", options.hosts[num], options.connTimeout) + } + if err == nil { + options.logf( + "[dial] secure=%t, skip_verify=%t, strategy=%s, ident=%d, server=%d -> %s", + options.secure, + options.skipVerify, + options.openStrategy, + ident, + num, + conn.RemoteAddr(), + ) + if tcp, ok := conn.(*net.TCPConn); ok { + err = tcp.SetNoDelay(options.noDelay) // Disable or enable the Nagle Algorithm for this tcp socket + if err != nil { + return nil, err + } + } + return &connect{ + Conn: conn, + logf: options.logf, + ident: ident, + buffer: bufio.NewReader(conn), + readTimeout: options.readTimeout, + writeTimeout: options.writeTimeout, + }, nil + } else { + options.logf( + "[dial err] secure=%t, skip_verify=%t, strategy=%s, ident=%d, addr=%s\n%#v", + options.secure, + options.skipVerify, + options.openStrategy, + ident, + options.hosts[num], + err, + ) + } + } + return nil, err +} + +type connect struct { + net.Conn + logf func(string, ...interface{}) + ident int + buffer *bufio.Reader + closed bool + readTimeout time.Duration + writeTimeout time.Duration + lastReadDeadlineTime time.Time + lastWriteDeadlineTime time.Time +} + +func (conn *connect) Read(b []byte) (int, error) { + var ( + n int + err error + total int + dstLen = len(b) + ) + if currentTime := now(); conn.readTimeout != 0 && currentTime.Sub(conn.lastReadDeadlineTime) > (conn.readTimeout>>2) { + conn.SetReadDeadline(time.Now().Add(conn.readTimeout)) + conn.lastReadDeadlineTime = currentTime + } + for total < dstLen { + if n, err = conn.buffer.Read(b[total:]); err != nil { + conn.logf("[connect] read error: %v", err) + conn.Close() + return n, driver.ErrBadConn + } + total += n + } + return total, nil +} + +func (conn *connect) Write(b []byte) (int, error) { + var ( + n int + err error + total int + srcLen = len(b) + ) + if currentTime := now(); conn.writeTimeout != 0 && currentTime.Sub(conn.lastWriteDeadlineTime) > (conn.writeTimeout>>2) { + conn.SetWriteDeadline(time.Now().Add(conn.writeTimeout)) + conn.lastWriteDeadlineTime = currentTime + } + for total < srcLen { + if n, err = conn.Conn.Write(b[total:]); err != nil { + conn.logf("[connect] write error: %v", err) + conn.Close() + return n, driver.ErrBadConn + } + total += n + } + return n, nil +} + +func (conn *connect) Close() error { + if !conn.closed { + conn.closed = true + return conn.Conn.Close() + } + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/docker-compose.yml b/vendor/github.com/ClickHouse/clickhouse-go/docker-compose.yml new file mode 100644 index 0000000000..2d03214ba1 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/docker-compose.yml @@ -0,0 +1,9 @@ +--- +version: '3' +services: + clickhouse: + image: yandex/clickhouse-server + ports: + - 127.0.0.1:8123:8123 + - 127.0.0.1:9000:9000 + - 127.0.0.1:9009:9009 diff --git a/vendor/github.com/ClickHouse/clickhouse-go/go.mod b/vendor/github.com/ClickHouse/clickhouse-go/go.mod new file mode 100644 index 0000000000..33116921c5 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/go.mod @@ -0,0 +1,11 @@ +module github.com/ClickHouse/clickhouse-go + +go 1.12 + +require ( + github.com/bkaradzic/go-lz4 v1.0.0 + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 + github.com/jmoiron/sqlx v1.2.0 + github.com/pierrec/lz4 v2.0.5+incompatible + github.com/stretchr/testify v1.3.0 +) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/go.sum b/vendor/github.com/ClickHouse/clickhouse-go/go.sum new file mode 100644 index 0000000000..c3d6a6f69b --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/go.sum @@ -0,0 +1,18 @@ +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/vendor/github.com/ClickHouse/clickhouse-go/go.test.sh b/vendor/github.com/ClickHouse/clickhouse-go/go.test.sh new file mode 100644 index 0000000000..70ffd08c2a --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/go.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor | grep -v examples); do + go test -race -coverprofile=profile.out -covermode=atomic $d + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done \ No newline at end of file diff --git a/vendor/github.com/ClickHouse/clickhouse-go/helpers.go b/vendor/github.com/ClickHouse/clickhouse-go/helpers.go new file mode 100644 index 0000000000..ec5ad7bf7a --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/helpers.go @@ -0,0 +1,146 @@ +package clickhouse + +import ( + "bytes" + "database/sql/driver" + "fmt" + "math" + "reflect" + "regexp" + "strings" + "time" +) + +func numInput(query string) int { + + var ( + count int + args = make(map[string]struct{}) + reader = bytes.NewReader([]byte(query)) + quote, gravis bool + escape bool + keyword bool + inBetween bool + like = newMatcher("like") + limit = newMatcher("limit") + offset = newMatcher("offset") + between = newMatcher("between") + in = newMatcher("in") + and = newMatcher("and") + from = newMatcher("from") + join = newMatcher("join") + ) + for { + if char, _, err := reader.ReadRune(); err == nil { + if escape { + escape = false + continue + } + switch char { + case '\\': + if gravis || quote { + escape = true + } + case '\'': + if !gravis { + quote = !quote + } + case '`': + if !quote { + gravis = !gravis + } + } + if quote || gravis { + continue + } + switch { + case char == '?' && keyword: + count++ + case char == '@': + if param := paramParser(reader); len(param) != 0 { + if _, found := args[param]; !found { + args[param] = struct{}{} + count++ + } + } + case + char == '=', + char == '<', + char == '>', + char == '(', + char == ',', + char == '[', + char == '%': + keyword = true + default: + if limit.matchRune(char) || offset.matchRune(char) || like.matchRune(char) || + in.matchRune(char) || from.matchRune(char) || join.matchRune(char) { + keyword = true + } else if between.matchRune(char) { + keyword = true + inBetween = true + } else if inBetween && and.matchRune(char) { + keyword = true + inBetween = false + } else { + keyword = keyword && (char == ' ' || char == '\t' || char == '\n') + } + } + } else { + break + } + } + return count +} + +func paramParser(reader *bytes.Reader) string { + var name bytes.Buffer + for { + if char, _, err := reader.ReadRune(); err == nil { + if char == '_' || char >= '0' && char <= '9' || 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' { + name.WriteRune(char) + } else { + reader.UnreadRune() + break + } + } else { + break + } + } + return name.String() +} + +var selectRe = regexp.MustCompile(`\s+SELECT\s+`) + +func isInsert(query string) bool { + if f := strings.Fields(query); len(f) > 2 { + return strings.EqualFold("INSERT", f[0]) && strings.EqualFold("INTO", f[1]) && !selectRe.MatchString(strings.ToUpper(query)) + } + return false +} + +func quote(v driver.Value) string { + switch v := reflect.ValueOf(v); v.Kind() { + case reflect.Slice: + values := make([]string, 0, v.Len()) + for i := 0; i < v.Len(); i++ { + values = append(values, quote(v.Index(i).Interface())) + } + return strings.Join(values, ", ") + } + switch v := v.(type) { + case string: + return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'" + case time.Time: + return formatTime(v) + } + return fmt.Sprint(v) +} + +func formatTime(value time.Time) string { + // toDate() overflows after 65535 days, but toDateTime() only overflows when time.Time overflows (after 9223372036854775807 seconds) + if days := value.Unix() / 24 / 3600; days <= math.MaxUint16 && (value.Hour()+value.Minute()+value.Second()+value.Nanosecond()) == 0 { + return fmt.Sprintf("toDate(%d)", days) + } + return fmt.Sprintf("toDateTime(%d)", value.Unix()) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader.go new file mode 100644 index 0000000000..be428b3648 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader.go @@ -0,0 +1,107 @@ +// +build !clz4 + +package binary + +import ( + "encoding/binary" + "fmt" + "io" + + "github.com/ClickHouse/clickhouse-go/lib/lz4" +) + +type compressReader struct { + reader io.Reader + // data uncompressed + data []byte + // data position + pos int + // data compressed + zdata []byte + // lz4 headers + header []byte +} + +// NewCompressReader wrap the io.Reader +func NewCompressReader(r io.Reader) *compressReader { + p := &compressReader{ + reader: r, + header: make([]byte, HeaderSize), + } + p.data = make([]byte, BlockMaxSize, BlockMaxSize) + + zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize + p.zdata = make([]byte, zlen, zlen) + + p.pos = len(p.data) + return p +} + +func (cr *compressReader) Read(buf []byte) (n int, err error) { + var bytesRead = 0 + n = len(buf) + + if cr.pos < len(cr.data) { + copyedSize := copy(buf, cr.data[cr.pos:]) + + bytesRead += copyedSize + cr.pos += copyedSize + } + + for bytesRead < n { + if err = cr.readCompressedData(); err != nil { + return bytesRead, err + } + copyedSize := copy(buf[bytesRead:], cr.data) + + bytesRead += copyedSize + cr.pos = copyedSize + } + return n, nil +} + +func (cr *compressReader) readCompressedData() (err error) { + cr.pos = 0 + var n int + n, err = cr.reader.Read(cr.header) + if err != nil { + return + } + if n != len(cr.header) { + return fmt.Errorf("Lz4 decompression header EOF") + } + + compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9 + decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:])) + + if compressedSize > cap(cr.zdata) { + cr.zdata = make([]byte, compressedSize) + } + if decompressedSize > cap(cr.data) { + cr.data = make([]byte, decompressedSize) + } + + cr.zdata = cr.zdata[:compressedSize] + cr.data = cr.data[:decompressedSize] + + // @TODO checksum + if cr.header[16] == LZ4 { + n, err = cr.reader.Read(cr.zdata) + if err != nil { + return + } + + if n != len(cr.zdata) { + return fmt.Errorf("Decompress read size not match") + } + + _, err = lz4.Decode(cr.data, cr.zdata) + if err != nil { + return + } + } else { + return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16]) + } + + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader_clz4.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader_clz4.go new file mode 100644 index 0000000000..2845b3b8be --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader_clz4.go @@ -0,0 +1,107 @@ +// +build clz4 + +package binary + +import ( + "encoding/binary" + "fmt" + "io" + + lz4 "github.com/cloudflare/golz4" +) + +type compressReader struct { + reader io.Reader + // data uncompressed + data []byte + // data position + pos int + // data compressed + zdata []byte + // lz4 headers + header []byte +} + +// NewCompressReader wrap the io.Reader +func NewCompressReader(r io.Reader) *compressReader { + p := &compressReader{ + reader: r, + header: make([]byte, HeaderSize), + } + p.data = make([]byte, BlockMaxSize, BlockMaxSize) + + zlen := lz4.CompressBound(p.data) + HeaderSize + p.zdata = make([]byte, zlen, zlen) + + p.pos = len(p.data) + return p +} + +func (cr *compressReader) Read(buf []byte) (n int, err error) { + var bytesRead = 0 + n = len(buf) + + if cr.pos < len(cr.data) { + copyedSize := copy(buf, cr.data[cr.pos:]) + + bytesRead += copyedSize + cr.pos += copyedSize + } + + for bytesRead < n { + if err = cr.readCompressedData(); err != nil { + return bytesRead, err + } + copyedSize := copy(buf[bytesRead:], cr.data) + + bytesRead += copyedSize + cr.pos = copyedSize + } + return n, nil +} + +func (cr *compressReader) readCompressedData() (err error) { + cr.pos = 0 + var n int + n, err = cr.reader.Read(cr.header) + if err != nil { + return + } + if n != len(cr.header) { + return fmt.Errorf("Lz4 decompression header EOF") + } + + compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9 + decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:])) + + if compressedSize > cap(cr.zdata) { + cr.zdata = make([]byte, compressedSize) + } + if decompressedSize > cap(cr.data) { + cr.data = make([]byte, decompressedSize) + } + + cr.zdata = cr.zdata[:compressedSize] + cr.data = cr.data[:decompressedSize] + + // @TODO checksum + if cr.header[16] == LZ4 { + n, err = cr.reader.Read(cr.zdata) + if err != nil { + return + } + + if n != len(cr.zdata) { + return fmt.Errorf("Decompress read size not match") + } + + err = lz4.Uncompress(cr.zdata, cr.data) + if err != nil { + return + } + } else { + return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16]) + } + + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_settings.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_settings.go new file mode 100644 index 0000000000..ca4559976a --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_settings.go @@ -0,0 +1,21 @@ +package binary + +type CompressionMethodByte byte + +const ( + NONE CompressionMethodByte = 0x02 + LZ4 = 0x82 + ZSTD = 0x90 +) + +const ( + // ChecksumSize is 128bits for cityhash102 checksum + ChecksumSize = 16 + // CompressHeader magic + compressed_size + uncompressed_size + CompressHeaderSize = 1 + 4 + 4 + + // HeaderSize + HeaderSize = ChecksumSize + CompressHeaderSize + // BlockMaxSize 1MB + BlockMaxSize = 1 << 20 +) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer.go new file mode 100644 index 0000000000..e4a73cf018 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer.go @@ -0,0 +1,79 @@ +// +build !clz4 + +package binary + +import ( + "encoding/binary" + "io" + + "github.com/ClickHouse/clickhouse-go/lib/cityhash102" + "github.com/ClickHouse/clickhouse-go/lib/lz4" +) + +type compressWriter struct { + writer io.Writer + // data uncompressed + data []byte + // data position + pos int + // data compressed + zdata []byte +} + +// NewCompressWriter wrap the io.Writer +func NewCompressWriter(w io.Writer) *compressWriter { + p := &compressWriter{writer: w} + p.data = make([]byte, BlockMaxSize, BlockMaxSize) + + zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize + p.zdata = make([]byte, zlen, zlen) + return p +} + +func (cw *compressWriter) Write(buf []byte) (int, error) { + var n int + for len(buf) > 0 { + // Accumulate the data to be compressed. + m := copy(cw.data[cw.pos:], buf) + cw.pos += m + buf = buf[m:] + + if cw.pos == len(cw.data) { + err := cw.Flush() + if err != nil { + return n, err + } + } + n += m + } + return n, nil +} + +func (cw *compressWriter) Flush() (err error) { + if cw.pos == 0 { + return + } + + // write the headers + compressedSize, err := lz4.Encode(cw.zdata[HeaderSize:], cw.data[:cw.pos]) + if err != nil { + return err + } + compressedSize += CompressHeaderSize + // fill the header, compressed_size_32 + uncompressed_size_32 + cw.zdata[16] = LZ4 + binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize)) + binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos)) + + // fill the checksum + checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize)) + binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64()) + binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64()) + + cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize]) + if w, ok := cw.writer.(WriteFlusher); ok { + err = w.Flush() + } + cw.pos = 0 + return +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer_clz4.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer_clz4.go new file mode 100644 index 0000000000..1302e16113 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer_clz4.go @@ -0,0 +1,78 @@ +// +build clz4 + +package binary + +import ( + "encoding/binary" + "io" + + lz4 "github.com/cloudflare/golz4" + "github.com/ClickHouse/clickhouse-go/lib/cityhash102" +) + +type compressWriter struct { + writer io.Writer + // data uncompressed + data []byte + // data position + pos int + // data compressed + zdata []byte +} + +// NewCompressWriter wrap the io.Writer +func NewCompressWriter(w io.Writer) *compressWriter { + p := &compressWriter{writer: w} + p.data = make([]byte, BlockMaxSize, BlockMaxSize) + + zlen := lz4.CompressBound(p.data) + HeaderSize + p.zdata = make([]byte, zlen, zlen) + return p +} + +func (cw *compressWriter) Write(buf []byte) (int, error) { + var n int + for len(buf) > 0 { + // Accumulate the data to be compressed. + m := copy(cw.data[cw.pos:], buf) + cw.pos += m + buf = buf[m:] + + if cw.pos == len(cw.data) { + err := cw.Flush() + if err != nil { + return n, err + } + } + n += m + } + return n, nil +} + +func (cw *compressWriter) Flush() (err error) { + if cw.pos == 0 { + return + } + // write the headers + compressedSize, err := lz4.Compress(cw.data[:cw.pos], cw.zdata[HeaderSize:]) + if err != nil { + return err + } + compressedSize += CompressHeaderSize + // fill the header, compressed_size_32 + uncompressed_size_32 + cw.zdata[16] = LZ4 + binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize)) + binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos)) + + // fill the checksum + checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize)) + binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64()) + binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64()) + + cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize]) + if w, ok := cw.writer.(WriteFlusher); ok { + err = w.Flush() + } + cw.pos = 0 + return +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/decoder.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/decoder.go new file mode 100644 index 0000000000..1bf213be74 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/decoder.go @@ -0,0 +1,177 @@ +package binary + +import ( + "encoding/binary" + "io" + "math" +) + +func NewDecoder(input io.Reader) *Decoder { + return &Decoder{ + input: input, + } +} + +func NewDecoderWithCompress(input io.Reader) *Decoder { + return &Decoder{ + input: input, + compressInput: NewCompressReader(input), + } +} + +type Decoder struct { + compress bool + input io.Reader + compressInput io.Reader + scratch [binary.MaxVarintLen64]byte +} + +func (decoder *Decoder) SelectCompress(compress bool) { + decoder.compress = compress +} + +func (decoder *Decoder) Get() io.Reader { + if decoder.compress && decoder.compressInput != nil { + return decoder.compressInput + } + return decoder.input +} + +func (decoder *Decoder) Bool() (bool, error) { + v, err := decoder.ReadByte() + if err != nil { + return false, err + } + return v == 1, nil +} + +func (decoder *Decoder) Uvarint() (uint64, error) { + return binary.ReadUvarint(decoder) +} + +func (decoder *Decoder) Int8() (int8, error) { + v, err := decoder.ReadByte() + if err != nil { + return 0, err + } + return int8(v), nil +} + +func (decoder *Decoder) Int16() (int16, error) { + v, err := decoder.UInt16() + if err != nil { + return 0, err + } + return int16(v), nil +} + +func (decoder *Decoder) Int32() (int32, error) { + v, err := decoder.UInt32() + if err != nil { + return 0, err + } + return int32(v), nil +} + +func (decoder *Decoder) Int64() (int64, error) { + v, err := decoder.UInt64() + if err != nil { + return 0, err + } + return int64(v), nil +} + +func (decoder *Decoder) UInt8() (uint8, error) { + v, err := decoder.ReadByte() + if err != nil { + return 0, err + } + return uint8(v), nil +} + +func (decoder *Decoder) UInt16() (uint16, error) { + if _, err := decoder.Get().Read(decoder.scratch[:2]); err != nil { + return 0, err + } + return uint16(decoder.scratch[0]) | uint16(decoder.scratch[1])<<8, nil +} + +func (decoder *Decoder) UInt32() (uint32, error) { + if _, err := decoder.Get().Read(decoder.scratch[:4]); err != nil { + return 0, err + } + return uint32(decoder.scratch[0]) | + uint32(decoder.scratch[1])<<8 | + uint32(decoder.scratch[2])<<16 | + uint32(decoder.scratch[3])<<24, nil +} + +func (decoder *Decoder) UInt64() (uint64, error) { + if _, err := decoder.Get().Read(decoder.scratch[:8]); err != nil { + return 0, err + } + return uint64(decoder.scratch[0]) | + uint64(decoder.scratch[1])<<8 | + uint64(decoder.scratch[2])<<16 | + uint64(decoder.scratch[3])<<24 | + uint64(decoder.scratch[4])<<32 | + uint64(decoder.scratch[5])<<40 | + uint64(decoder.scratch[6])<<48 | + uint64(decoder.scratch[7])<<56, nil +} + +func (decoder *Decoder) Float32() (float32, error) { + v, err := decoder.UInt32() + if err != nil { + return 0, err + } + return math.Float32frombits(v), nil +} + +func (decoder *Decoder) Float64() (float64, error) { + v, err := decoder.UInt64() + if err != nil { + return 0, err + } + return math.Float64frombits(v), nil +} + +func (decoder *Decoder) Fixed(ln int) ([]byte, error) { + if reader, ok := decoder.Get().(FixedReader); ok { + return reader.Fixed(ln) + } + buf := make([]byte, ln) + if _, err := decoder.Get().Read(buf); err != nil { + return nil, err + } + return buf, nil +} + +func (decoder *Decoder) String() (string, error) { + strlen, err := decoder.Uvarint() + if err != nil { + return "", err + } + str, err := decoder.Fixed(int(strlen)) + if err != nil { + return "", err + } + return string(str), nil +} + +func (decoder *Decoder) Decimal128() ([]byte, error) { + bytes := make([]byte, 16) + _, err := decoder.Get().Read(bytes) + return bytes, err +} + +func (decoder *Decoder) ReadByte() (byte, error) { + if _, err := decoder.Get().Read(decoder.scratch[:1]); err != nil { + return 0x0, err + } + return decoder.scratch[0], nil +} + +type FixedReader interface { + Fixed(ln int) ([]byte, error) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/encoder.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/encoder.go new file mode 100644 index 0000000000..d1502a3613 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/binary/encoder.go @@ -0,0 +1,364 @@ +package binary + +import ( + "encoding/binary" + "io" + "math" + "reflect" + "unsafe" +) + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + output: w, + } +} + +func NewEncoderWithCompress(w io.Writer) *Encoder { + return &Encoder{ + output: w, + compressOutput: NewCompressWriter(w), + } +} + +type Encoder struct { + compress bool + output io.Writer + compressOutput io.Writer + scratch [binary.MaxVarintLen64]byte +} + +func (enc *Encoder) SelectCompress(compress bool) { + if enc.compressOutput == nil { + return + } + if enc.compress && !compress { + enc.Flush() + } + enc.compress = compress +} + +func (enc *Encoder) Get() io.Writer { + if enc.compress && enc.compressOutput != nil { + return enc.compressOutput + } + return enc.output +} + +func (enc *Encoder) Nullable(isNull bool) error { + nullablePrefix := uint8(0) + if !isNull { + nullablePrefix = uint8(1) + } + if _, err := enc.Get().Write([]byte{nullablePrefix}); err != nil { + return err + } + return nil +} + +func (enc *Encoder) Uvarint(v uint64) error { + ln := binary.PutUvarint(enc.scratch[:binary.MaxVarintLen64], v) + if _, err := enc.Get().Write(enc.scratch[0:ln]); err != nil { + return err + } + return nil +} + +func (enc *Encoder) UvarintNullable(v *uint64) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Uvarint(0) + } + return enc.Uvarint(*v) +} + +func (enc *Encoder) Bool(v bool) error { + if v { + return enc.UInt8(1) + } + return enc.UInt8(0) +} + +func (enc *Encoder) BoolNullable(v *bool) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Bool(false) + } + return enc.Bool(*v) +} + +func (enc *Encoder) Int8(v int8) error { + return enc.UInt8(uint8(v)) +} + +func (enc *Encoder) Int8Nullable(v *int8) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Int8(0) + } + return enc.Int8(*v) +} + +func (enc *Encoder) Int16(v int16) error { + return enc.UInt16(uint16(v)) +} + +func (enc *Encoder) Int16Nullable(v *int16) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Int16(0) + } + return enc.Int16(*v) +} + +func (enc *Encoder) Int32(v int32) error { + return enc.UInt32(uint32(v)) +} + +func (enc *Encoder) Int32Nullable(v *int32) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Int32(0) + } + return enc.Int32(*v) +} + +func (enc *Encoder) Int64(v int64) error { + return enc.UInt64(uint64(v)) +} + +func (enc *Encoder) Int64Nullable(v *int64) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Int64(0) + } + return enc.Int64(*v) +} + +func (enc *Encoder) UInt8(v uint8) error { + enc.scratch[0] = v + if _, err := enc.Get().Write(enc.scratch[:1]); err != nil { + return err + } + return nil +} + +func (enc *Encoder) UInt8Nullable(v *uint8) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.UInt8(0) + } + return enc.UInt8(*v) +} + +func (enc *Encoder) UInt16(v uint16) error { + enc.scratch[0] = byte(v) + enc.scratch[1] = byte(v >> 8) + if _, err := enc.Get().Write(enc.scratch[:2]); err != nil { + return err + } + return nil +} + +func (enc *Encoder) UInt16Nullable(v *uint16) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.UInt16(0) + } + return enc.UInt16(*v) +} + +func (enc *Encoder) UInt32(v uint32) error { + enc.scratch[0] = byte(v) + enc.scratch[1] = byte(v >> 8) + enc.scratch[2] = byte(v >> 16) + enc.scratch[3] = byte(v >> 24) + if _, err := enc.Get().Write(enc.scratch[:4]); err != nil { + return err + } + return nil +} + +func (enc *Encoder) UInt32Nullable(v *uint32) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.UInt32(0) + } + return enc.UInt32(*v) +} + +func (enc *Encoder) UInt64(v uint64) error { + enc.scratch[0] = byte(v) + enc.scratch[1] = byte(v >> 8) + enc.scratch[2] = byte(v >> 16) + enc.scratch[3] = byte(v >> 24) + enc.scratch[4] = byte(v >> 32) + enc.scratch[5] = byte(v >> 40) + enc.scratch[6] = byte(v >> 48) + enc.scratch[7] = byte(v >> 56) + if _, err := enc.Get().Write(enc.scratch[:8]); err != nil { + return err + } + return nil +} + +func (enc *Encoder) UInt64Nullable(v *uint64) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.UInt64(0) + } + return enc.UInt64(*v) +} + +func (enc *Encoder) Float32(v float32) error { + return enc.UInt32(math.Float32bits(v)) +} + +func (enc *Encoder) Float32Nullable(v *float32) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Float32(0) + } + return enc.Float32(*v) +} + +func (enc *Encoder) Float64(v float64) error { + return enc.UInt64(math.Float64bits(v)) +} + +func (enc *Encoder) Float64Nullable(v *float64) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.Float64(0) + } + return enc.Float64(*v) +} + +func (enc *Encoder) String(v string) error { + str := Str2Bytes(v) + if err := enc.Uvarint(uint64(len(str))); err != nil { + return err + } + if _, err := enc.Get().Write(str); err != nil { + return err + } + return nil +} + +func (enc *Encoder) StringNullable(v *string) error { + isNil := v == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.String("") + } + return enc.String(*v) +} + +func (enc *Encoder) RawString(str []byte) error { + if err := enc.Uvarint(uint64(len(str))); err != nil { + return err + } + if _, err := enc.Get().Write(str); err != nil { + return err + } + return nil +} + +func (enc *Encoder) RawStringNullable(str *[]byte) error { + isNil := str == nil + if err := enc.Nullable(isNil); err != nil { + return err + } + if isNil { + return enc.RawString([]byte{}) + } + return enc.RawString(*str) +} + +func (enc *Encoder) Decimal128(bytes []byte) error { + _, err := enc.Get().Write(bytes) + return err +} + +func (enc *Encoder) Write(b []byte) (int, error) { + return enc.Get().Write(b) +} + +func (enc *Encoder) Flush() error { + if w, ok := enc.Get().(WriteFlusher); ok { + return w.Flush() + } + return nil +} + +type WriteFlusher interface { + Flush() error +} + +func Str2Bytes(str string) []byte { + // Copied from https://github.com/m3db/m3/blob/master/src/x/unsafe/string.go#L62 + if len(str) == 0 { + return nil + } + + // We need to declare a real byte slice so internally the compiler + // knows to use an unsafe.Pointer to keep track of the underlying memory so that + // once the slice's array pointer is updated with the pointer to the string's + // underlying bytes, the compiler won't prematurely GC the memory when the string + // goes out of scope. + var b []byte + byteHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + + // This makes sure that even if GC relocates the string's underlying + // memory after this assignment, the corresponding unsafe.Pointer in the internal + // slice struct will be updated accordingly to reflect the memory relocation. + byteHeader.Data = (*reflect.StringHeader)(unsafe.Pointer(&str)).Data + + // It is important that we access str after we assign the Data + // pointer of the string header to the Data pointer of the slice header to + // make sure the string (and the underlying bytes backing the string) don't get + // GC'ed before the assignment happens. + l := len(str) + byteHeader.Len = l + byteHeader.Cap = l + + return b +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/city64.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/city64.go new file mode 100644 index 0000000000..c410c44dca --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/city64.go @@ -0,0 +1,45 @@ +package cityhash102 + +import ( + "encoding/binary" + "hash" +) + +type City64 struct { + s []byte +} + +var _ hash.Hash64 = (*City64)(nil) +var _ hash.Hash = (*City64)(nil) + +func New64() hash.Hash64 { + return &City64{} +} + +func (this *City64) Sum(b []byte) []byte { + b2 := make([]byte, 8) + binary.BigEndian.PutUint64(b2, this.Sum64()) + b = append(b, b2...) + return b +} + +func (this *City64) Sum64() uint64 { + return CityHash64(this.s, uint32(len(this.s))) +} + +func (this *City64) Reset() { + this.s = this.s[0:0] +} + +func (this *City64) BlockSize() int { + return 1 +} + +func (this *City64) Write(s []byte) (n int, err error) { + this.s = append(this.s, s...) + return len(s), nil +} + +func (this *City64) Size() int { + return 8 +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/cityhash.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/cityhash.go new file mode 100644 index 0000000000..dcc666b671 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/cityhash.go @@ -0,0 +1,383 @@ +/* + * Go implementation of Google city hash (MIT license) + * https://code.google.com/p/cityhash/ + * + * MIT License http://www.opensource.org/licenses/mit-license.php + * + * I don't even want to pretend to understand the details of city hash. + * I am only reproducing the logic in Go as faithfully as I can. + * + */ + +package cityhash102 + +import ( + "encoding/binary" +) + +const ( + k0 uint64 = 0xc3a5c85c97cb3127 + k1 uint64 = 0xb492b66fbe98f273 + k2 uint64 = 0x9ae16a3b2f90404f + k3 uint64 = 0xc949d7c7509e6557 + + kMul uint64 = 0x9ddfea08eb382d69 +) + +func fetch64(p []byte) uint64 { + return binary.LittleEndian.Uint64(p) + //return uint64InExpectedOrder(unalignedLoad64(p)) +} + +func fetch32(p []byte) uint32 { + return binary.LittleEndian.Uint32(p) + //return uint32InExpectedOrder(unalignedLoad32(p)) +} + +func rotate64(val uint64, shift uint32) uint64 { + if shift != 0 { + return ((val >> shift) | (val << (64 - shift))) + } + + return val +} + +func rotate32(val uint32, shift uint32) uint32 { + if shift != 0 { + return ((val >> shift) | (val << (32 - shift))) + } + + return val +} + +func swap64(a, b *uint64) { + *a, *b = *b, *a +} + +func swap32(a, b *uint32) { + *a, *b = *b, *a +} + +func permute3(a, b, c *uint32) { + swap32(a, b) + swap32(a, c) +} + +func rotate64ByAtLeast1(val uint64, shift uint32) uint64 { + return (val >> shift) | (val << (64 - shift)) +} + +func shiftMix(val uint64) uint64 { + return val ^ (val >> 47) +} + +type Uint128 [2]uint64 + +func (this *Uint128) setLower64(l uint64) { + this[0] = l +} + +func (this *Uint128) setHigher64(h uint64) { + this[1] = h +} + +func (this Uint128) Lower64() uint64 { + return this[0] +} + +func (this Uint128) Higher64() uint64 { + return this[1] +} + +func (this Uint128) Bytes() []byte { + b := make([]byte, 16) + binary.LittleEndian.PutUint64(b, this[0]) + binary.LittleEndian.PutUint64(b[8:], this[1]) + return b +} + +func hash128to64(x Uint128) uint64 { + // Murmur-inspired hashing. + var a = (x.Lower64() ^ x.Higher64()) * kMul + a ^= (a >> 47) + var b = (x.Higher64() ^ a) * kMul + b ^= (b >> 47) + b *= kMul + return b +} + +func hashLen16(u, v uint64) uint64 { + return hash128to64(Uint128{u, v}) +} + +func hashLen16_3(u, v, mul uint64) uint64 { + // Murmur-inspired hashing. + var a = (u ^ v) * mul + a ^= (a >> 47) + var b = (v ^ a) * mul + b ^= (b >> 47) + b *= mul + return b +} + +func hashLen0to16(s []byte, length uint32) uint64 { + if length > 8 { + var a = fetch64(s) + var b = fetch64(s[length-8:]) + + return hashLen16(a, rotate64ByAtLeast1(b+uint64(length), length)) ^ b + } + + if length >= 4 { + var a = fetch32(s) + return hashLen16(uint64(length)+(uint64(a)<<3), uint64(fetch32(s[length-4:]))) + } + + if length > 0 { + var a uint8 = uint8(s[0]) + var b uint8 = uint8(s[length>>1]) + var c uint8 = uint8(s[length-1]) + + var y uint32 = uint32(a) + (uint32(b) << 8) + var z uint32 = length + (uint32(c) << 2) + + return shiftMix(uint64(y)*k2^uint64(z)*k3) * k2 + } + + return k2 +} + +// This probably works well for 16-byte strings as well, but it may be overkill +func hashLen17to32(s []byte, length uint32) uint64 { + var a = fetch64(s) * k1 + var b = fetch64(s[8:]) + var c = fetch64(s[length-8:]) * k2 + var d = fetch64(s[length-16:]) * k0 + + return hashLen16(rotate64(a-b, 43)+rotate64(c, 30)+d, + a+rotate64(b^k3, 20)-c+uint64(length)) +} + +func weakHashLen32WithSeeds(w, x, y, z, a, b uint64) Uint128 { + a += w + b = rotate64(b+a+z, 21) + var c uint64 = a + a += x + a += y + b += rotate64(a, 44) + return Uint128{a + z, b + c} +} + +func weakHashLen32WithSeeds_3(s []byte, a, b uint64) Uint128 { + return weakHashLen32WithSeeds(fetch64(s), fetch64(s[8:]), fetch64(s[16:]), fetch64(s[24:]), a, b) +} + +func hashLen33to64(s []byte, length uint32) uint64 { + var z uint64 = fetch64(s[24:]) + var a uint64 = fetch64(s) + (uint64(length)+fetch64(s[length-16:]))*k0 + var b uint64 = rotate64(a+z, 52) + var c uint64 = rotate64(a, 37) + + a += fetch64(s[8:]) + c += rotate64(a, 7) + a += fetch64(s[16:]) + + var vf uint64 = a + z + var vs = b + rotate64(a, 31) + c + + a = fetch64(s[16:]) + fetch64(s[length-32:]) + z = fetch64(s[length-8:]) + b = rotate64(a+z, 52) + c = rotate64(a, 37) + a += fetch64(s[length-24:]) + c += rotate64(a, 7) + a += fetch64(s[length-16:]) + + wf := a + z + ws := b + rotate64(a, 31) + c + r := shiftMix((vf+ws)*k2 + (wf+vs)*k0) + return shiftMix(r*k0+vs) * k2 +} + +func CityHash64(s []byte, length uint32) uint64 { + if length <= 32 { + if length <= 16 { + return hashLen0to16(s, length) + } else { + return hashLen17to32(s, length) + } + } else if length <= 64 { + return hashLen33to64(s, length) + } + + var x uint64 = fetch64(s) + var y uint64 = fetch64(s[length-16:]) ^ k1 + var z uint64 = fetch64(s[length-56:]) ^ k0 + + var v Uint128 = weakHashLen32WithSeeds_3(s[length-64:], uint64(length), y) + var w Uint128 = weakHashLen32WithSeeds_3(s[length-32:], uint64(length)*k1, k0) + + z += shiftMix(v.Higher64()) * k1 + x = rotate64(z+x, 39) * k1 + y = rotate64(y, 33) * k1 + + length = (length - 1) & ^uint32(63) + for { + x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1 + y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 + + x ^= w.Higher64() + y ^= v.Lower64() + + z = rotate64(z^w.Lower64(), 33) + v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) + w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y) + + swap64(&z, &x) + s = s[64:] + length -= 64 + + if length == 0 { + break + } + } + + return hashLen16(hashLen16(v.Lower64(), w.Lower64())+shiftMix(y)*k1+z, hashLen16(v.Higher64(), w.Higher64())+x) +} + +func CityHash64WithSeed(s []byte, length uint32, seed uint64) uint64 { + return CityHash64WithSeeds(s, length, k2, seed) +} + +func CityHash64WithSeeds(s []byte, length uint32, seed0, seed1 uint64) uint64 { + return hashLen16(CityHash64(s, length)-seed0, seed1) +} + +func cityMurmur(s []byte, length uint32, seed Uint128) Uint128 { + var a uint64 = seed.Lower64() + var b uint64 = seed.Higher64() + var c uint64 = 0 + var d uint64 = 0 + var l int32 = int32(length) - 16 + + if l <= 0 { // len <= 16 + a = shiftMix(a*k1) * k1 + c = b*k1 + hashLen0to16(s, length) + + if length >= 8 { + d = shiftMix(a + fetch64(s)) + } else { + d = shiftMix(a + c) + } + + } else { // len > 16 + c = hashLen16(fetch64(s[length-8:])+k1, a) + d = hashLen16(b+uint64(length), c+fetch64(s[length-16:])) + a += d + + for { + a ^= shiftMix(fetch64(s)*k1) * k1 + a *= k1 + b ^= a + c ^= shiftMix(fetch64(s[8:])*k1) * k1 + c *= k1 + d ^= c + s = s[16:] + l -= 16 + + if l <= 0 { + break + } + } + } + a = hashLen16(a, c) + b = hashLen16(d, b) + return Uint128{a ^ b, hashLen16(b, a)} +} + +func CityHash128WithSeed(s []byte, length uint32, seed Uint128) Uint128 { + if length < 128 { + return cityMurmur(s, length, seed) + } + + // We expect length >= 128 to be the common case. Keep 56 bytes of state: + // v, w, x, y, and z. + var v, w Uint128 + var x uint64 = seed.Lower64() + var y uint64 = seed.Higher64() + var z uint64 = uint64(length) * k1 + + var pos uint32 + var t = s + + v.setLower64(rotate64(y^k1, 49)*k1 + fetch64(s)) + v.setHigher64(rotate64(v.Lower64(), 42)*k1 + fetch64(s[8:])) + w.setLower64(rotate64(y+z, 35)*k1 + x) + w.setHigher64(rotate64(x+fetch64(s[88:]), 53) * k1) + + // This is the same inner loop as CityHash64(), manually unrolled. + for { + x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1 + y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 + + x ^= w.Higher64() + y ^= v.Lower64() + z = rotate64(z^w.Lower64(), 33) + v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) + w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y) + swap64(&z, &x) + s = s[64:] + pos += 64 + + x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1 + y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1 + x ^= w.Higher64() + y ^= v.Lower64() + z = rotate64(z^w.Lower64(), 33) + v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64()) + w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y) + swap64(&z, &x) + s = s[64:] + pos += 64 + length -= 128 + + if length < 128 { + break + } + } + + y += rotate64(w.Lower64(), 37)*k0 + z + x += rotate64(v.Lower64()+z, 49) * k0 + + // If 0 < length < 128, hash up to 4 chunks of 32 bytes each from the end of s. + var tailDone uint32 + for tailDone = 0; tailDone < length; { + tailDone += 32 + y = rotate64(y-x, 42)*k0 + v.Higher64() + + //TODO why not use origin_len ? + w.setLower64(w.Lower64() + fetch64(t[pos+length-tailDone+16:])) + x = rotate64(x, 49)*k0 + w.Lower64() + w.setLower64(w.Lower64() + v.Lower64()) + v = weakHashLen32WithSeeds_3(t[pos+length-tailDone:], v.Lower64(), v.Higher64()) + } + // At this point our 48 bytes of state should contain more than + // enough information for a strong 128-bit hash. We use two + // different 48-byte-to-8-byte hashes to get a 16-byte final result. + x = hashLen16(x, v.Lower64()) + y = hashLen16(y, w.Lower64()) + + return Uint128{hashLen16(x+v.Higher64(), w.Higher64()) + y, + hashLen16(x+w.Higher64(), y+v.Higher64())} +} + +func CityHash128(s []byte, length uint32) (result Uint128) { + if length >= 16 { + result = CityHash128WithSeed(s[16:length], length-16, Uint128{fetch64(s) ^ k3, fetch64(s[8:])}) + } else if length >= 8 { + result = CityHash128WithSeed(nil, 0, Uint128{fetch64(s) ^ (uint64(length) * k0), fetch64(s[length-8:]) ^ k1}) + } else { + result = CityHash128WithSeed(s, length, Uint128{k0, k1}) + } + return +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/doc.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/doc.go new file mode 100644 index 0000000000..01ee5f8545 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/doc.go @@ -0,0 +1,5 @@ +/** COPY from https://github.com/zentures/cityhash/ + +NOTE: The code is modified to be compatible with CityHash128 used in ClickHouse +*/ +package cityhash102 diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/array.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/array.go new file mode 100644 index 0000000000..a9e052a224 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/array.go @@ -0,0 +1,270 @@ +package column + +import ( + "errors" + "fmt" + "net" + "reflect" + "strings" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type columnDecoder func() (interface{}, error) + +var unsupportedArrayTypeErrTemp = "unsupported Array type '%s'" + +// If you add Nullable type, that can be used in Array(Nullable(T)) add this type to ../codegen/nullable_appender/main.go in structure values.Types. +// Run code generation. +//go:generate go run ../codegen/nullable_appender -package $GOPACKAGE -file nullable_appender.go +type Array struct { + base + depth int + column Column + nullable bool +} + +func (array *Array) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + return nil, fmt.Errorf("do not use Read method for Array(T) column") +} + +func (array *Array) WriteNull(nulls, encoder *binary.Encoder, v interface{}) error { + if array.nullable { + column, ok := array.column.(*Nullable) + if !ok { + return fmt.Errorf("cannot convert to nullable type") + } + return column.WriteNull(nulls, encoder, v) + } + return fmt.Errorf("write null to not nullable array") +} + +func (array *Array) Write(encoder *binary.Encoder, v interface{}) error { + return array.column.Write(encoder, v) +} + +func (array *Array) ReadArray(decoder *binary.Decoder, rows int) (_ []interface{}, err error) { + var ( + offsets = make([][]uint64, array.depth) + values = make([]interface{}, rows) + ) + + // Read offsets + lastOffset := uint64(rows) + for i := 0; i < array.depth; i++ { + offset := make([]uint64, lastOffset) + for j := uint64(0); j < lastOffset; j++ { + if offset[j], err = decoder.UInt64(); err != nil { + return nil, err + } + } + offsets[i] = offset + lastOffset = 0 + if len(offset) > 0 { + lastOffset = offset[len(offset)-1] + } + } + + var cd columnDecoder + + switch column := array.column.(type) { + case *Nullable: + nullRows, err := column.ReadNull(decoder, int(lastOffset)) + if err != nil { + return nil, err + } + cd = func(rows []interface{}) columnDecoder { + i := 0 + return func() (interface{}, error) { + if i > len(rows) { + return nil, errors.New("not enough rows to return while parsing Null column") + } + ret := rows[i] + i++ + return ret, nil + } + }(nullRows) + case *Tuple: + tupleRows, err := column.ReadTuple(decoder, int(lastOffset)) + if err != nil { + return nil, err + } + // closure to return fully assembled tuple values as if they + // were decoded one at a time + cd = func(rows []interface{}) columnDecoder { + i := 0 + return func() (interface{}, error) { + if i > len(rows) { + return nil, errors.New("not enough rows to return while parsing Tuple column") + } + ret := rows[i] + i++ + return ret, nil + } + }(tupleRows) + default: + cd = func(decoder *binary.Decoder) columnDecoder { + return func() (interface{}, error) { return array.column.Read(decoder, array.nullable) } + }(decoder) + } + + // Read values + for i := 0; i < rows; i++ { + if values[i], err = array.read(cd, offsets, uint64(i), 0); err != nil { + return nil, err + } + } + return values, nil +} + +func (array *Array) read(readColumn columnDecoder, offsets [][]uint64, index uint64, level int) (interface{}, error) { + end := offsets[level][index] + start := uint64(0) + if index > 0 { + start = offsets[level][index-1] + } + + scanT := array.column.ScanType() + slice := reflect.MakeSlice(array.arrayType(level), 0, int(end-start)) + for i := start; i < end; i++ { + var ( + value interface{} + err error + ) + if level == array.depth-1 { + value, err = readColumn() + } else { + value, err = array.read(readColumn, offsets, i, level+1) + } + if err != nil { + return nil, err + } + if array.nullable && level == array.depth-1 { + f, ok := nullableAppender[scanT.String()] + if !ok { + return nil, fmt.Errorf(unsupportedArrayTypeErrTemp, scanT.String()) + } + + cSlice, err := f(value, slice) + if err != nil { + return nil, err + } + + slice = cSlice + } else { + slice = reflect.Append(slice, reflect.ValueOf(value)) + } + + } + return slice.Interface(), nil +} + +func (array *Array) arrayType(level int) reflect.Type { + t := array.column.ScanType() + for i := 0; i < array.depth-level; i++ { + t = reflect.SliceOf(t) + } + return t +} + +func (array *Array) Depth() int { + return array.depth +} + +func parseArray(name, chType string, timezone *time.Location) (*Array, error) { + if len(chType) < 11 { + return nil, fmt.Errorf("invalid Array column type: %s", chType) + } + var ( + depth int + columnType = chType + ) + +loop: + for _, str := range strings.Split(chType, "Array(") { + switch { + case len(str) == 0: + depth++ + default: + chType = str[:len(str)-depth] + break loop + } + } + column, err := Factory(name, chType, timezone) + if err != nil { + return nil, fmt.Errorf("Array(T): %v", err) + } + + var scanType interface{} + switch t := column.ScanType(); t { + case arrayBaseTypes[int8(0)]: + scanType = []int8{} + case arrayBaseTypes[int16(0)]: + scanType = []int16{} + case arrayBaseTypes[int32(0)]: + scanType = []int32{} + case arrayBaseTypes[int64(0)]: + scanType = []int64{} + case arrayBaseTypes[uint8(0)]: + scanType = []uint8{} + case arrayBaseTypes[uint16(0)]: + scanType = []uint16{} + case arrayBaseTypes[uint32(0)]: + scanType = []uint32{} + case arrayBaseTypes[uint64(0)]: + scanType = []uint64{} + case arrayBaseTypes[float32(0)]: + scanType = []float32{} + case arrayBaseTypes[float64(0)]: + scanType = []float64{} + case arrayBaseTypes[string("")]: + scanType = []string{} + case arrayBaseTypes[time.Time{}]: + scanType = []time.Time{} + case arrayBaseTypes[IPv4{}], arrayBaseTypes[IPv6{}]: + scanType = []net.IP{} + case reflect.ValueOf([]interface{}{}).Type(): + scanType = [][]interface{}{} + + //nullable + case arrayBaseTypes[ptrInt8T]: + scanType = []*int8{} + case arrayBaseTypes[ptrInt16T]: + scanType = []*int16{} + case arrayBaseTypes[ptrInt32T]: + scanType = []*int32{} + case arrayBaseTypes[ptrInt64T]: + scanType = []*int64{} + case arrayBaseTypes[ptrUInt8T]: + scanType = []*uint8{} + case arrayBaseTypes[ptrUInt16T]: + scanType = []*uint16{} + case arrayBaseTypes[ptrUInt32T]: + scanType = []*uint32{} + case arrayBaseTypes[ptrUInt64T]: + scanType = []*uint64{} + case arrayBaseTypes[ptrFloat32]: + scanType = []*float32{} + case arrayBaseTypes[ptrFloat64]: + scanType = []*float64{} + case arrayBaseTypes[ptrString]: + scanType = []*string{} + case arrayBaseTypes[ptrTime]: + scanType = []*time.Time{} + case arrayBaseTypes[ptrIPv4], arrayBaseTypes[ptrIPv6]: + scanType = []*net.IP{} + default: + return nil, fmt.Errorf(unsupportedArrayTypeErrTemp, column.ScanType().Name()) + } + return &Array{ + base: base{ + name: name, + chType: columnType, + valueOf: reflect.ValueOf(scanType), + }, + depth: depth, + column: column, + nullable: strings.HasPrefix(column.CHType(), "Nullable"), + }, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/column.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/column.go new file mode 100644 index 0000000000..10a3dc6a3e --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/column.go @@ -0,0 +1,200 @@ +package column + +import ( + "fmt" + "reflect" + "strings" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Column interface { + Name() string + CHType() string + ScanType() reflect.Type + Read(*binary.Decoder, bool) (interface{}, error) + Write(*binary.Encoder, interface{}) error + defaultValue() interface{} + Depth() int +} + +func Factory(name, chType string, timezone *time.Location) (Column, error) { + switch chType { + case "Int8": + return &Int8{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[int8(0)], + }, + }, nil + case "Int16": + return &Int16{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[int16(0)], + }, + }, nil + case "Int32": + return &Int32{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[int32(0)], + }, + }, nil + case "Int64": + return &Int64{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[int64(0)], + }, + }, nil + case "UInt8": + return &UInt8{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[uint8(0)], + }, + }, nil + case "UInt16": + return &UInt16{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[uint16(0)], + }, + }, nil + case "UInt32": + return &UInt32{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[uint32(0)], + }, + }, nil + case "UInt64": + return &UInt64{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[uint64(0)], + }, + }, nil + case "Float32": + return &Float32{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[float32(0)], + }, + }, nil + case "Float64": + return &Float64{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[float64(0)], + }, + }, nil + case "String": + return &String{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[string("")], + }, + }, nil + case "UUID": + return &UUID{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[string("")], + }, + }, nil + case "Date": + _, offset := time.Unix(0, 0).In(timezone).Zone() + return &Date{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[time.Time{}], + }, + Timezone: timezone, + offset: int64(offset), + }, nil + case "IPv4": + return &IPv4{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[IPv4{}], + }, + }, nil + case "IPv6": + return &IPv6{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[IPv6{}], + }, + }, nil + } + switch { + case strings.HasPrefix(chType, "DateTime") && !strings.HasPrefix(chType, "DateTime64"): + return &DateTime{ + base: base{ + name: name, + chType: "DateTime", + valueOf: columnBaseTypes[time.Time{}], + }, + Timezone: timezone, + }, nil + case strings.HasPrefix(chType, "DateTime64"): + return &DateTime64{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[time.Time{}], + }, + Timezone: timezone, + }, nil + case strings.HasPrefix(chType, "Array"): + return parseArray(name, chType, timezone) + case strings.HasPrefix(chType, "Nullable"): + return parseNullable(name, chType, timezone) + case strings.HasPrefix(chType, "FixedString"): + return parseFixedString(name, chType) + case strings.HasPrefix(chType, "Enum8"), strings.HasPrefix(chType, "Enum16"): + return parseEnum(name, chType) + case strings.HasPrefix(chType, "Decimal"): + return parseDecimal(name, chType) + case strings.HasPrefix(chType, "SimpleAggregateFunction"): + if nestedType, err := getNestedType(chType, "SimpleAggregateFunction"); err != nil { + return nil, err + } else { + return Factory(name, nestedType, timezone) + } + case strings.HasPrefix(chType, "Tuple"): + return parseTuple(name, chType, timezone) + } + return nil, fmt.Errorf("column: unhandled type %v", chType) +} + +func getNestedType(chType string, wrapType string) (string, error) { + prefixLen := len(wrapType) + 1 + suffixLen := 1 + + if len(chType) > prefixLen+suffixLen { + nested := strings.Split(chType[prefixLen:len(chType)-suffixLen], ",") + if len(nested) == 2 { + return strings.TrimSpace(nested[1]), nil + } + } + return "", fmt.Errorf("column: invalid %s type (%s)", wrapType, chType) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/common.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/common.go new file mode 100644 index 0000000000..0583eeea91 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/common.go @@ -0,0 +1,115 @@ +package column + +import ( + "fmt" + "net" + "reflect" + "time" +) + +type ErrUnexpectedType struct { + Column Column + T interface{} +} + +func (err *ErrUnexpectedType) Error() string { + return fmt.Sprintf("%s: unexpected type %T", err.Column, err.T) +} + +var columnBaseTypes = map[interface{}]reflect.Value{ + int8(0): reflect.ValueOf(int8(0)), + int16(0): reflect.ValueOf(int16(0)), + int32(0): reflect.ValueOf(int32(0)), + int64(0): reflect.ValueOf(int64(0)), + uint8(0): reflect.ValueOf(uint8(0)), + uint16(0): reflect.ValueOf(uint16(0)), + uint32(0): reflect.ValueOf(uint32(0)), + uint64(0): reflect.ValueOf(uint64(0)), + float32(0): reflect.ValueOf(float32(0)), + float64(0): reflect.ValueOf(float64(0)), + string(""): reflect.ValueOf(string("")), + time.Time{}: reflect.ValueOf(time.Time{}), + IPv4{}: reflect.ValueOf(net.IPv4zero), + IPv6{}: reflect.ValueOf(net.IPv6unspecified), +} + +type ptrTo uint8 + +const ( + ptrInt8T ptrTo = iota + ptrInt16T + ptrInt32T + ptrInt64T + ptrUInt8T + ptrUInt16T + ptrUInt32T + ptrUInt64T + ptrFloat32 + ptrFloat64 + ptrString + ptrTime + ptrIPv4 + ptrIPv6 +) + +var arrayBaseTypes = map[interface{}]reflect.Type{ + int8(0): reflect.ValueOf(int8(0)).Type(), + int16(0): reflect.ValueOf(int16(0)).Type(), + int32(0): reflect.ValueOf(int32(0)).Type(), + int64(0): reflect.ValueOf(int64(0)).Type(), + uint8(0): reflect.ValueOf(uint8(0)).Type(), + uint16(0): reflect.ValueOf(uint16(0)).Type(), + uint32(0): reflect.ValueOf(uint32(0)).Type(), + uint64(0): reflect.ValueOf(uint64(0)).Type(), + float32(0): reflect.ValueOf(float32(0)).Type(), + float64(0): reflect.ValueOf(float64(0)).Type(), + string(""): reflect.ValueOf(string("")).Type(), + time.Time{}: reflect.ValueOf(time.Time{}).Type(), + IPv4{}: reflect.ValueOf(net.IPv4zero).Type(), + IPv6{}: reflect.ValueOf(net.IPv6unspecified).Type(), + + // nullable + ptrInt8T: reflect.PtrTo(reflect.ValueOf(int8(0)).Type()), + ptrInt16T: reflect.PtrTo(reflect.ValueOf(int16(0)).Type()), + ptrInt32T: reflect.PtrTo(reflect.ValueOf(int32(0)).Type()), + ptrInt64T: reflect.PtrTo(reflect.ValueOf(int64(0)).Type()), + ptrUInt8T: reflect.PtrTo(reflect.ValueOf(uint8(0)).Type()), + ptrUInt16T: reflect.PtrTo(reflect.ValueOf(uint16(0)).Type()), + ptrUInt32T: reflect.PtrTo(reflect.ValueOf(uint32(0)).Type()), + ptrUInt64T: reflect.PtrTo(reflect.ValueOf(uint64(0)).Type()), + ptrFloat32: reflect.PtrTo(reflect.ValueOf(float32(0)).Type()), + ptrFloat64: reflect.PtrTo(reflect.ValueOf(float64(0)).Type()), + ptrString: reflect.PtrTo(reflect.ValueOf(string("")).Type()), + ptrTime: reflect.PtrTo(reflect.ValueOf(time.Time{}).Type()), + ptrIPv4: reflect.PtrTo(reflect.ValueOf(net.IPv4zero).Type()), + ptrIPv6: reflect.PtrTo(reflect.ValueOf(net.IPv6unspecified).Type()), +} + +type base struct { + name, chType string + valueOf reflect.Value +} + +func (base *base) Name() string { + return base.name +} + +func (base *base) CHType() string { + return base.chType +} + +func (base *base) ScanType() reflect.Type { + return base.valueOf.Type() +} + +func (base *base) defaultValue() interface{} { + return base.valueOf.Interface() +} + +func (base *base) String() string { + return fmt.Sprintf("%s (%s)", base.name, base.chType) +} + +func (base *base) Depth() int { + return 0 +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/date.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/date.go new file mode 100644 index 0000000000..6dacb45c2d --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/date.go @@ -0,0 +1,84 @@ +package column + +import ( + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Date struct { + base + Timezone *time.Location + offset int64 +} + +func (dt *Date) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + sec, err := decoder.Int16() + if err != nil { + return nil, err + } + return time.Unix(int64(sec)*24*3600-dt.offset, 0).In(dt.Timezone), nil +} + +func (dt *Date) Write(encoder *binary.Encoder, v interface{}) error { + var timestamp int64 + switch value := v.(type) { + case time.Time: + _, offset := value.Zone() + timestamp = value.Unix() + int64(offset) + case int16: + return encoder.Int16(value) + case int32: + timestamp = int64(value) + dt.offset + case uint32: + timestamp = int64(value) + dt.offset + case uint64: + timestamp = int64(value) + dt.offset + case int64: + timestamp = value + dt.offset + case string: + var err error + timestamp, err = dt.parse(value) + if err != nil { + return err + } + + // this relies on Nullable never sending nil values through + case *time.Time: + _, offset := value.Zone() + timestamp = (*value).Unix() + int64(offset) + case *int16: + return encoder.Int16(*value) + case *int32: + timestamp = int64(*value) + dt.offset + case *int64: + timestamp = *value + dt.offset + case *string: + var err error + timestamp, err = dt.parse(*value) + if err != nil { + return err + } + + default: + return &ErrUnexpectedType{ + T: v, + Column: dt, + } + } + + return encoder.Int16(int16(timestamp / 24 / 3600)) +} + +func (dt *Date) parse(value string) (int64, error) { + tv, err := time.Parse("2006-01-02", value) + if err != nil { + return 0, err + } + return time.Date( + time.Time(tv).Year(), + time.Time(tv).Month(), + time.Time(tv).Day(), + 0, 0, 0, 0, time.UTC, + ).Unix(), nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime.go new file mode 100644 index 0000000000..3252c83530 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime.go @@ -0,0 +1,87 @@ +package column + +import ( + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type DateTime struct { + base + Timezone *time.Location +} + +func (dt *DateTime) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + sec, err := decoder.Int32() + if err != nil { + return nil, err + } + return time.Unix(int64(sec), 0).In(dt.Timezone), nil +} + +func (dt *DateTime) Write(encoder *binary.Encoder, v interface{}) error { + var timestamp int64 + switch value := v.(type) { + case time.Time: + if !value.IsZero() { + timestamp = value.Unix() + } + case int16: + timestamp = int64(value) + case int32: + timestamp = int64(value) + case uint32: + timestamp = int64(value) + case uint64: + timestamp = int64(value) + case int64: + timestamp = value + case string: + var err error + timestamp, err = dt.parse(value) + if err != nil { + return err + } + + case *time.Time: + if value != nil && !(*value).IsZero() { + timestamp = (*value).Unix() + } + case *int16: + timestamp = int64(*value) + case *int32: + timestamp = int64(*value) + case *int64: + timestamp = *value + case *string: + var err error + timestamp, err = dt.parse(*value) + if err != nil { + return err + } + + default: + return &ErrUnexpectedType{ + T: v, + Column: dt, + } + } + + return encoder.Int32(int32(timestamp)) +} + +func (dt *DateTime) parse(value string) (int64, error) { + tv, err := time.Parse("2006-01-02 15:04:05", value) + if err != nil { + return 0, err + } + return time.Date( + time.Time(tv).Year(), + time.Time(tv).Month(), + time.Time(tv).Day(), + time.Time(tv).Hour(), + time.Time(tv).Minute(), + time.Time(tv).Second(), + 0, time.Local, //use local timzone when insert into clickhouse + ).Unix(), nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime64.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime64.go new file mode 100644 index 0000000000..20657f970a --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime64.go @@ -0,0 +1,100 @@ +package column + +import ( + "math" + "strconv" + "strings" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type DateTime64 struct { + base + Timezone *time.Location +} + +func (dt *DateTime64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + value, err := decoder.Int64() + if err != nil { + return nil, err + } + + precision, err := dt.getPrecision() + if err != nil { + return nil, err + } + + var nano int64 + if precision < 19 { + nano = value * int64(math.Pow10(9-precision)) + } + + sec := nano / int64(10e8) + nsec := nano - sec*10e8 + + return time.Unix(sec, nsec).In(dt.Timezone), nil +} + +func (dt *DateTime64) Write(encoder *binary.Encoder, v interface{}) error { + var timestamp int64 + switch value := v.(type) { + case time.Time: + if !value.IsZero() { + timestamp = value.UnixNano() + } + case uint64: + timestamp = int64(value) + case int64: + timestamp = value + case string: + var err error + timestamp, err = dt.parse(value) + if err != nil { + return err + } + case *time.Time: + if value != nil && !(*value).IsZero() { + timestamp = (*value).UnixNano() + } + case *int64: + timestamp = *value + case *string: + var err error + timestamp, err = dt.parse(*value) + if err != nil { + return err + } + default: + return &ErrUnexpectedType{ + T: v, + Column: dt, + } + } + + precision, err := dt.getPrecision() + if err != nil { + return err + } + + timestamp = timestamp / int64(math.Pow10(9-precision)) + + return encoder.Int64(timestamp) +} + +func (dt *DateTime64) parse(value string) (int64, error) { + tv, err := time.Parse("2006-01-02 15:04:05.999", value) + if err != nil { + return 0, err + } + return tv.UnixNano(), nil +} + +func (dt *DateTime64) getPrecision() (int, error) { + dtParams := dt.base.chType[11 : len(dt.base.chType)-1] + precision, err := strconv.Atoi(strings.Split(dtParams, ",")[0]) + if err != nil { + return 0, err + } + return precision, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/decimal.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/decimal.go new file mode 100644 index 0000000000..464208eee8 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/decimal.go @@ -0,0 +1,377 @@ +package column + +import ( + b "encoding/binary" + "errors" + "fmt" + "math" + "reflect" + "strconv" + "strings" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +// Table of powers of 10 for fast casting from floating types to decimal type +// representations. +var factors10 = []float64{ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, + 1e14, 1e15, 1e16, 1e17, 1e18, +} + +// Decimal represents Decimal(P, S) ClickHouse. Decimal is represented as +// integral. Also floating-point types are supported for query parameters. +// +// Since there is no support for int128 in Golang, decimals with precision 19 +// through 38 are represented as 16 little-endian bytes. +type Decimal struct { + base + nobits int // its domain is {32, 64} + precision int + scale int +} + +func (d *Decimal) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + switch d.nobits { + case 32: + return decoder.Int32() + case 64: + return decoder.Int64() + case 128: + return decoder.Decimal128() + default: + return nil, errors.New("unachievable execution path") + } +} + +func (d *Decimal) Write(encoder *binary.Encoder, v interface{}) error { + switch d.nobits { + case 32: + return d.write32(encoder, v) + case 64: + return d.write64(encoder, v) + case 128: + return d.write128(encoder, v) + default: + return errors.New("unachievable execution path") + } +} + +func (d *Decimal) float2int32(floating float64) int32 { + fixed := int32(floating * factors10[d.scale]) + return fixed +} + +func (d *Decimal) float2int64(floating float64) int64 { + fixed := int64(floating * factors10[d.scale]) + return fixed +} + +func (d *Decimal) write32(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int8: + return encoder.Int32(int32(v)) + case int16: + return encoder.Int32(int32(v)) + case int32: + return encoder.Int32(int32(v)) + case int64: + if v > math.MaxInt32 || v < math.MinInt32 { + return errors.New("overflow when narrowing type conversion from int64 to int32") + } + return encoder.Int32(int32(v)) + + case uint8: + return encoder.Int32(int32(v)) + case uint16: + return encoder.Int32(int32(v)) + case uint32: + if v > math.MaxInt32 { + return errors.New("overflow when narrowing type conversion from uint32 to int32") + } + return encoder.Int32(int32(v)) + case uint64: + if v > math.MaxInt32 { + return errors.New("overflow when narrowing type conversion from uint64 to int32") + } + return encoder.Int32(int32(v)) + + case float32: + fixed := d.float2int32(float64(v)) + return encoder.Int32(fixed) + case float64: + fixed := d.float2int32(float64(v)) + return encoder.Int32(fixed) + + // this relies on Nullable never sending nil values through + case *int8: + return encoder.Int32(int32(*v)) + case *int16: + return encoder.Int32(int32(*v)) + case *int32: + return encoder.Int32(int32(*v)) + case *int64: + if *v > math.MaxInt32 || *v < math.MinInt32 { + return errors.New("overflow when narrowing type conversion from int64 to int32") + } + return encoder.Int32(int32(*v)) + + case *uint8: + return encoder.Int32(int32(*v)) + case *uint16: + return encoder.Int32(int32(*v)) + case *uint32: + if *v > math.MaxInt32 { + return errors.New("overflow when narrowing type conversion from uint34 to int32") + } + return encoder.Int32(int32(*v)) + case *uint64: + if *v > math.MaxInt32 { + return errors.New("overflow when narrowing type conversion from uint64 to int32") + } + return encoder.Int32(int32(*v)) + + case *float32: + fixed := d.float2int32(float64(*v)) + return encoder.Int32(fixed) + case *float64: + fixed := d.float2int32(float64(*v)) + return encoder.Int32(fixed) + } + + return &ErrUnexpectedType{ + T: v, + Column: d, + } +} + +func (d *Decimal) write64(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int: + return encoder.Int64(int64(v)) + case int8: + return encoder.Int64(int64(v)) + case int16: + return encoder.Int64(int64(v)) + case int32: + return encoder.Int64(int64(v)) + case int64: + return encoder.Int64(int64(v)) + + case uint8: + return encoder.Int64(int64(v)) + case uint16: + return encoder.Int64(int64(v)) + case uint32: + return encoder.Int64(int64(v)) + case uint64: + if v > math.MaxInt64 { + return errors.New("overflow when narrowing type conversion from uint64 to int64") + } + return encoder.Int64(int64(v)) + + case float32: + fixed := d.float2int64(float64(v)) + return encoder.Int64(fixed) + case float64: + fixed := d.float2int64(float64(v)) + return encoder.Int64(fixed) + + // this relies on Nullable never sending nil values through + case *int: + return encoder.Int64(int64(*v)) + case *int8: + return encoder.Int64(int64(*v)) + case *int16: + return encoder.Int64(int64(*v)) + case *int32: + return encoder.Int64(int64(*v)) + case *int64: + return encoder.Int64(int64(*v)) + + case *uint8: + return encoder.Int64(int64(*v)) + case *uint16: + return encoder.Int64(int64(*v)) + case *uint32: + return encoder.Int64(int64(*v)) + case *uint64: + if *v > math.MaxInt64 { + return errors.New("overflow when narrowing type conversion from uint64 to int64") + } + return encoder.Int64(int64(*v)) + + case *float32: + fixed := d.float2int64(float64(*v)) + return encoder.Int64(fixed) + case *float64: + fixed := d.float2int64(float64(*v)) + return encoder.Int64(fixed) + } + + return &ErrUnexpectedType{ + T: v, + Column: d, + } +} + +// Turns an int64 into 16 little-endian bytes. +func int64ToDecimal128(v int64) []byte { + bytes := make([]byte, 16) + b.LittleEndian.PutUint64(bytes[:8], uint64(v)) + sign := 0 + if v < 0 { + sign = -1 + } + b.LittleEndian.PutUint64(bytes[8:], uint64(sign)) + return bytes +} + +// Turns a uint64 into 16 little-endian bytes. +func uint64ToDecimal128(v uint64) []byte { + bytes := make([]byte, 16) + b.LittleEndian.PutUint64(bytes[:8], uint64(v)) + return bytes +} + +func (d *Decimal) write128(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int: + return encoder.Decimal128(int64ToDecimal128(int64(v))) + case int8: + return encoder.Decimal128(int64ToDecimal128(int64(v))) + case int16: + return encoder.Decimal128(int64ToDecimal128(int64(v))) + case int32: + return encoder.Decimal128(int64ToDecimal128(int64(v))) + case int64: + return encoder.Decimal128(int64ToDecimal128(v)) + + case uint8: + return encoder.Decimal128(uint64ToDecimal128(uint64(v))) + case uint16: + return encoder.Decimal128(uint64ToDecimal128(uint64(v))) + case uint32: + return encoder.Decimal128(uint64ToDecimal128(uint64(v))) + case uint64: + return encoder.Decimal128(uint64ToDecimal128(v)) + + case float32: + fixed := d.float2int64(float64(v)) + return encoder.Decimal128(int64ToDecimal128(fixed)) + case float64: + fixed := d.float2int64(float64(v)) + return encoder.Decimal128(int64ToDecimal128(fixed)) + + case []byte: + if len(v) != 16 { + return errors.New("expected 16 bytes") + } + return encoder.Decimal128(v) + + // this relies on Nullable never sending nil values through + case *int: + return encoder.Decimal128(int64ToDecimal128(int64(*v))) + case *int8: + return encoder.Decimal128(int64ToDecimal128(int64(*v))) + case *int16: + return encoder.Decimal128(int64ToDecimal128(int64(*v))) + case *int32: + return encoder.Decimal128(int64ToDecimal128(int64(*v))) + case *int64: + return encoder.Decimal128(int64ToDecimal128(*v)) + + case *uint8: + return encoder.Decimal128(uint64ToDecimal128(uint64(*v))) + case *uint16: + return encoder.Decimal128(uint64ToDecimal128(uint64(*v))) + case *uint32: + return encoder.Decimal128(uint64ToDecimal128(uint64(*v))) + case *uint64: + return encoder.Decimal128(uint64ToDecimal128(*v)) + + case *float32: + fixed := d.float2int64(float64(*v)) + return encoder.Decimal128(int64ToDecimal128(fixed)) + case *float64: + fixed := d.float2int64(float64(*v)) + return encoder.Decimal128(int64ToDecimal128(fixed)) + + case *[]byte: + if len(*v) != 16 { + return errors.New("expected 16 bytes") + } + return encoder.Decimal128(*v) + } + + return &ErrUnexpectedType{ + T: v, + Column: d, + } +} + +func parseDecimal(name, chType string) (Column, error) { + switch { + case len(chType) < 12: + fallthrough + case !strings.HasPrefix(chType, "Decimal"): + fallthrough + case chType[7] != '(': + fallthrough + case chType[len(chType)-1] != ')': + return nil, fmt.Errorf("invalid Decimal format: '%s'", chType) + } + + var params = strings.Split(chType[8:len(chType)-1], ",") + + if len(params) != 2 { + return nil, fmt.Errorf("invalid Decimal format: '%s'", chType) + } + + params[0] = strings.TrimSpace(params[0]) + params[1] = strings.TrimSpace(params[1]) + + var err error + var decimal = &Decimal{ + base: base{ + name: name, + chType: chType, + }, + } + + if decimal.precision, err = strconv.Atoi(params[0]); err != nil { + return nil, fmt.Errorf("'%s' is not Decimal type: %s", chType, err) + } else if decimal.precision < 1 { + return nil, errors.New("wrong precision of Decimal type") + } + + if decimal.scale, err = strconv.Atoi(params[1]); err != nil { + return nil, fmt.Errorf("'%s' is not Decimal type: %s", chType, err) + } else if decimal.scale < 0 || decimal.scale > decimal.precision { + return nil, errors.New("wrong scale of Decimal type") + } + + switch { + case decimal.precision <= 9: + decimal.nobits = 32 + decimal.valueOf = columnBaseTypes[int32(0)] + case decimal.precision <= 18: + decimal.nobits = 64 + decimal.valueOf = columnBaseTypes[int64(0)] + case decimal.precision <= 38: + decimal.nobits = 128 + decimal.valueOf = reflect.ValueOf([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}) + default: + return nil, errors.New("precision of Decimal exceeds max bound") + } + + return decimal, nil +} + +func (d *Decimal) GetPrecision() int { + return d.precision +} + +func (d *Decimal) GetScale() int { + return d.scale +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/enum.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/enum.go new file mode 100644 index 0000000000..59a8d7073c --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/enum.go @@ -0,0 +1,175 @@ +package column + +import ( + "fmt" + "strconv" + "strings" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Enum struct { + iv map[string]interface{} + vi map[interface{}]string + base + baseType interface{} +} + +func (enum *Enum) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + var ( + err error + ident interface{} + ) + switch enum.baseType.(type) { + case int16: + if ident, err = decoder.Int16(); err != nil { + return nil, err + } + default: + if ident, err = decoder.Int8(); err != nil { + return nil, err + } + } + if ident, found := enum.vi[ident]; found || isNull { + return ident, nil + } + return nil, fmt.Errorf("invalid Enum value: %v", ident) +} + +func (enum *Enum) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case string: + return enum.encodeFromString(v, encoder) + case uint8: + if _, ok := enum.baseType.(int8); ok { + return encoder.Int8(int8(v)) + } + case int8: + if _, ok := enum.baseType.(int8); ok { + return encoder.Int8(v) + } + case uint16: + if _, ok := enum.baseType.(int16); ok { + return encoder.Int16(int16(v)) + } + case int16: + if _, ok := enum.baseType.(int16); ok { + return encoder.Int16(v) + } + case int64: + switch enum.baseType.(type) { + case int8: + return encoder.Int8(int8(v)) + case int16: + return encoder.Int16(int16(v)) + } + // nullable enums + case *string: + return enum.encodeFromString(*v, encoder) + case *uint8: + if _, ok := enum.baseType.(int8); ok { + return encoder.Int8(int8(*v)) + } + case *int8: + if _, ok := enum.baseType.(int8); ok { + return encoder.Int8(*v) + } + case *uint16: + if _, ok := enum.baseType.(int16); ok { + return encoder.Int16(int16(*v)) + } + case *int16: + if _, ok := enum.baseType.(int16); ok { + return encoder.Int16(*v) + } + case *int64: + switch enum.baseType.(type) { + case int8: + return encoder.Int8(int8(*v)) + case int16: + return encoder.Int16(int16(*v)) + } + } + return &ErrUnexpectedType{ + T: v, + Column: enum, + } +} + +func (enum *Enum) encodeFromString(v string, encoder *binary.Encoder) error { + ident, found := enum.iv[v] + if !found { + return fmt.Errorf("invalid Enum ident: %s", v) + } + switch ident := ident.(type) { + case int8: + return encoder.Int8(ident) + case int16: + return encoder.Int16(ident) + default: + return &ErrUnexpectedType{ + T: ident, + Column: enum, + } + } +} + +func (enum *Enum) defaultValue() interface{} { + return enum.baseType +} + +func parseEnum(name, chType string) (*Enum, error) { + var ( + data string + isEnum16 bool + ) + if len(chType) < 8 { + return nil, fmt.Errorf("invalid Enum format: %s", chType) + } + switch { + case strings.HasPrefix(chType, "Enum8"): + data = chType[6:] + case strings.HasPrefix(chType, "Enum16"): + data = chType[7:] + isEnum16 = true + default: + return nil, fmt.Errorf("'%s' is not Enum type", chType) + } + enum := Enum{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[string("")], + }, + iv: make(map[string]interface{}), + vi: make(map[interface{}]string), + } + for _, block := range strings.Split(data[:len(data)-1], ",") { + parts := strings.Split(block, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid Enum format: %s", chType) + } + var ( + ident = strings.TrimSpace(parts[0]) + value, err = strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 16) + ) + if err != nil { + return nil, fmt.Errorf("invalid Enum value: %v", chType) + } + { + var ( + ident = ident[1 : len(ident)-1] + value interface{} = int16(value) + ) + if !isEnum16 { + value = int8(value.(int16)) + } + if enum.baseType == nil { + enum.baseType = value + } + enum.iv[ident] = value + enum.vi[value] = ident + } + } + return &enum, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/fixed_string.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/fixed_string.go new file mode 100644 index 0000000000..75135f7b23 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/fixed_string.go @@ -0,0 +1,71 @@ +package column + +import ( + "encoding" + "fmt" + "reflect" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type FixedString struct { + base + len int + scanType reflect.Type +} + +func (str *FixedString) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Fixed(str.len) + if err != nil { + return "", err + } + return string(v), nil +} + +func (str *FixedString) Write(encoder *binary.Encoder, v interface{}) error { + var fixedString []byte + switch v := v.(type) { + case string: + fixedString = binary.Str2Bytes(v) + case []byte: + fixedString = v + case encoding.BinaryMarshaler: + bytes, err := v.MarshalBinary() + if err != nil { + return err + } + fixedString = bytes + default: + return &ErrUnexpectedType{ + T: v, + Column: str, + } + } + switch { + case len(fixedString) > str.len: + return fmt.Errorf("too large value '%s' (expected %d, got %d)", fixedString, str.len, len(fixedString)) + case len(fixedString) < str.len: + tmp := make([]byte, str.len) + copy(tmp, fixedString) + fixedString = tmp + } + if _, err := encoder.Write(fixedString); err != nil { + return err + } + return nil +} + +func parseFixedString(name, chType string) (*FixedString, error) { + var strLen int + if _, err := fmt.Sscanf(chType, "FixedString(%d)", &strLen); err != nil { + return nil, err + } + return &FixedString{ + base: base{ + name: name, + chType: chType, + valueOf: columnBaseTypes[string("")], + }, + len: strLen, + }, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float32.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float32.go new file mode 100644 index 0000000000..477a6a1a39 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float32.go @@ -0,0 +1,35 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Float32 struct{ base } + +func (Float32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Float32() + if err != nil { + return float32(0), err + } + return v, nil +} + +func (float *Float32) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case float32: + return encoder.Float32(v) + case float64: + return encoder.Float32(float32(v)) + + // this relies on Nullable never sending nil values through + case *float32: + return encoder.Float32(*v) + case *float64: + return encoder.Float32(float32(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: float, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float64.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float64.go new file mode 100644 index 0000000000..2f8454004b --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/float64.go @@ -0,0 +1,35 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Float64 struct{ base } + +func (Float64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Float64() + if err != nil { + return float64(0), err + } + return v, nil +} + +func (float *Float64) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case float32: + return encoder.Float64(float64(v)) + case float64: + return encoder.Float64(v) + + // this relies on Nullable never sending nil values through + case *float32: + return encoder.Float64(float64(*v)) + case *float64: + return encoder.Float64(*v) + } + + return &ErrUnexpectedType{ + T: v, + Column: float, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int16.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int16.go new file mode 100644 index 0000000000..0b2d167dad --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int16.go @@ -0,0 +1,39 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Int16 struct{ base } + +func (Int16) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Int16() + if err != nil { + return int16(0), err + } + return v, nil +} + +func (i *Int16) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int16: + return encoder.Int16(v) + case int64: + return encoder.Int16(int16(v)) + case int: + return encoder.Int16(int16(v)) + + // this relies on Nullable never sending nil values through + case *int16: + return encoder.Int16(*v) + case *int64: + return encoder.Int16(int16(*v)) + case *int: + return encoder.Int16(int16(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: i, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int32.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int32.go new file mode 100644 index 0000000000..34328e3043 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int32.go @@ -0,0 +1,39 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Int32 struct{ base } + +func (Int32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Int32() + if err != nil { + return int32(0), err + } + return v, nil +} + +func (i *Int32) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int32: + return encoder.Int32(v) + case int64: + return encoder.Int32(int32(v)) + case int: + return encoder.Int32(int32(v)) + + // this relies on Nullable never sending nil values through + case *int32: + return encoder.Int32(*v) + case *int64: + return encoder.Int32(int32(*v)) + case *int: + return encoder.Int32(int32(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: i, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int64.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int64.go new file mode 100644 index 0000000000..fa627f0cb8 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int64.go @@ -0,0 +1,40 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Int64 struct{ base } + +func (Int64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Int64() + if err != nil { + return int64(0), err + } + return v, nil +} + +func (i *Int64) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int: + return encoder.Int64(int64(v)) + case int64: + return encoder.Int64(v) + case []byte: + if _, err := encoder.Write(v); err != nil { + return err + } + return nil + + // this relies on Nullable never sending nil values through + case *int: + return encoder.Int64(int64(*v)) + case *int64: + return encoder.Int64(*v) + } + + return &ErrUnexpectedType{ + T: v, + Column: i, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int8.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int8.go new file mode 100644 index 0000000000..aa36bee2bf --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/int8.go @@ -0,0 +1,49 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Int8 struct{ base } + +func (Int8) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Int8() + if err != nil { + return int8(0), err + } + return v, nil +} + +func (i *Int8) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case int8: + return encoder.Int8(v) + case int64: + return encoder.Int8(int8(v)) + case int: + return encoder.Int8(int8(v)) + case bool: + if v { + return encoder.Int8(int8(1)) + } + return encoder.Int8(int8(0)) + + // this relies on Nullable never sending nil values through + case *int8: + return encoder.Int8(*v) + case *int64: + return encoder.Int8(int8(*v)) + case *int: + return encoder.Int8(int8(*v)) + case *bool: + if *v { + return encoder.Int8(int8(1)) + } + return encoder.Int8(int8(0)) + } + + return &ErrUnexpectedType{ + T: v, + Column: i, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ip.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ip.go new file mode 100644 index 0000000000..03056d7c54 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ip.go @@ -0,0 +1,84 @@ +/* +IP type supporting for clickhouse as FixedString(16) +*/ + +package column + +import ( + "database/sql/driver" + "errors" + "net" + "strings" +) + +var ( + errInvalidScanType = errors.New("Invalid scan types") + errInvalidScanValue = errors.New("Invalid scan value") +) + +// IP column type +type IP net.IP + +// Value implements the driver.Valuer interface, json field interface +// Alignment on the right side +func (ip IP) Value() (driver.Value, error) { + return ip.MarshalBinary() +} + +func (ip IP) MarshalBinary() ([]byte, error) { + if len(ip) < 16 { + var ( + buff = make([]byte, 16) + j = 0 + ) + for i := 16 - len(ip); i < 16; i++ { + buff[i] = ip[j] + j++ + } + for i := 0; i < 16-len(ip); i++ { + buff[i] = '\x00' + } + if len(ip) == 4 { + buff[11] = '\xff' + buff[10] = '\xff' + } + return buff, nil + } + return []byte(ip), nil +} + +// Scan implements the driver.Valuer interface, json field interface +func (ip *IP) Scan(value interface{}) (err error) { + switch v := value.(type) { + case []byte: + if len(v) == 4 || len(v) == 16 { + *ip = IP(v) + } else { + err = errInvalidScanValue + } + case string: + if v == "" { + err = errInvalidScanValue + return + } + if (len(v) == 4 || len(v) == 16) && !strings.Contains(v, ".") && !strings.Contains(v, ":"){ + *ip = IP([]byte(v)) + return + } + if strings.Contains(v, ":") { + *ip = IP(net.ParseIP(v)) + return + } + *ip = IP(net.ParseIP(v).To4()) + case net.IP: + *ip = IP(v) + default: + err = errInvalidScanType + } + return +} + +// String implements the fmt.Stringer interface +func (ip IP) String() string { + return net.IP(ip).String() +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv4.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv4.go new file mode 100644 index 0000000000..eaac52c6da --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv4.go @@ -0,0 +1,54 @@ +package column + +import ( + "net" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type IPv4 struct { + base +} + +func (*IPv4) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Fixed(4) + if err != nil { + return nil, err + } + return net.IPv4(v[3], v[2], v[1], v[0]), nil +} + +func (ip *IPv4) Write(encoder *binary.Encoder, v interface{}) error { + var netIP net.IP + switch v.(type) { + case string: + netIP = net.ParseIP(v.(string)) + case net.IP: + netIP = v.(net.IP) + case *net.IP: + netIP = *(v.(*net.IP)) + default: + return &ErrUnexpectedType{ + T: v, + Column: ip, + } + } + + if netIP == nil { + return &ErrUnexpectedType{ + T: v, + Column: ip, + } + } + ip4 := netIP.To4() + if ip4 == nil { + return &ErrUnexpectedType{ + T: v, + Column: ip, + } + } + if _, err := encoder.Write([]byte{ip4[3], ip4[2], ip4[1], ip4[0]}); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv6.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv6.go new file mode 100644 index 0000000000..0fa9f8a9f8 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv6.go @@ -0,0 +1,47 @@ +package column + +import ( + "net" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type IPv6 struct { + base +} + +func (*IPv6) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.Fixed(16) + if err != nil { + return nil, err + } + return net.IP(v), nil +} + +func (ip *IPv6) Write(encoder *binary.Encoder, v interface{}) error { + var netIP net.IP + switch v.(type) { + case string: + netIP = net.ParseIP(v.(string)) + case net.IP: + netIP = v.(net.IP) + case *net.IP: + netIP = *(v.(*net.IP)) + default: + return &ErrUnexpectedType{ + T: v, + Column: ip, + } + } + + if netIP == nil { + return &ErrUnexpectedType{ + T: v, + Column: ip, + } + } + if _, err := encoder.Write([]byte(netIP.To16())); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable.go new file mode 100644 index 0000000000..9dd88b546a --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable.go @@ -0,0 +1,96 @@ +package column + +import ( + "fmt" + "reflect" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type Nullable struct { + base + column Column +} + +func (null *Nullable) ScanType() reflect.Type { + return reflect.PtrTo(null.column.ScanType()) +} + +func (null *Nullable) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + return null.column.Read(decoder, isNull) +} + +func (null *Nullable) Write(encoder *binary.Encoder, v interface{}) error { + return nil +} + +func (null *Nullable) ReadNull(decoder *binary.Decoder, rows int) (_ []interface{}, err error) { + var ( + isNull byte + value interface{} + nulls = make([]byte, rows) + values = make([]interface{}, rows) + ) + for i := 0; i < rows; i++ { + if isNull, err = decoder.ReadByte(); err != nil { + return nil, err + } + nulls[i] = isNull + } + for i, isNull := range nulls { + switch value, err = null.column.Read(decoder, isNull != 0); true { + case err != nil: + return nil, err + case isNull == 0: + values[i] = value + default: + values[i] = nil + } + } + return values, nil +} +func (null *Nullable) WriteNull(nulls, encoder *binary.Encoder, v interface{}) error { + if isNil(v) { + if _, err := nulls.Write([]byte{1}); err != nil { + return err + } + return null.column.Write(encoder, null.column.defaultValue()) + } + if _, err := nulls.Write([]byte{0}); err != nil { + return err + } + return null.column.Write(encoder, v) +} + +func parseNullable(name, chType string, timezone *time.Location) (*Nullable, error) { + if len(chType) < 14 { + return nil, fmt.Errorf("invalid Nullable column type: %s", chType) + } + column, err := Factory(name, chType[9:][:len(chType)-10], timezone) + if err != nil { + return nil, fmt.Errorf("Nullable(T): %v", err) + } + return &Nullable{ + base: base{ + name: name, + chType: chType, + }, + column: column, + }, nil +} + +func (null *Nullable) GetColumn() Column { + return null.column +} + +func isNil(v interface{}) bool { + if v == nil { + return true + } + switch val := reflect.ValueOf(v); val.Type().Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Ptr, reflect.Slice: + return val.IsNil() + } + return false +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable_appender.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable_appender.go new file mode 100644 index 0000000000..a768026211 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable_appender.go @@ -0,0 +1,173 @@ + +// DANGER! This code was autogenerated from template by clickhouse-go/lib/codegen/nullable_appender. +// You shouldn't change it manually. +// For more info check clickhouse-go/lib/codegen/nullable_appender/main.go + +package column + +import ( + "fmt" + "net" + "reflect" + "time" +) + +var nullableAppender = map[string]func(v interface{}, slice reflect.Value) (reflect.Value, error){ + + "*int8": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(int8) + if !ok { + return slice, fmt.Errorf("cannot assert to type int8") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *int8 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*int16": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(int16) + if !ok { + return slice, fmt.Errorf("cannot assert to type int16") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *int16 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*int32": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(int32) + if !ok { + return slice, fmt.Errorf("cannot assert to type int32") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *int32 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*int64": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(int64) + if !ok { + return slice, fmt.Errorf("cannot assert to type int64") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *int64 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*uint8": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(uint8) + if !ok { + return slice, fmt.Errorf("cannot assert to type uint8") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *uint8 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*uint16": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(uint16) + if !ok { + return slice, fmt.Errorf("cannot assert to type uint16") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *uint16 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*uint32": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(uint32) + if !ok { + return slice, fmt.Errorf("cannot assert to type uint32") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *uint32 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*uint64": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(uint64) + if !ok { + return slice, fmt.Errorf("cannot assert to type uint64") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *uint64 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*float32": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(float32) + if !ok { + return slice, fmt.Errorf("cannot assert to type float32") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *float32 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*float64": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(float64) + if !ok { + return slice, fmt.Errorf("cannot assert to type float64") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *float64 + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*string": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(string) + if !ok { + return slice, fmt.Errorf("cannot assert to type string") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *string + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*time.Time": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(time.Time) + if !ok { + return slice, fmt.Errorf("cannot assert to type time.Time") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *time.Time + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + + "*net.IP": func(v interface{}, slice reflect.Value) (reflect.Value, error) { + if v != nil { + v, ok := v.(net.IP) + if !ok { + return slice, fmt.Errorf("cannot assert to type net.IP") + } + return reflect.Append(slice, reflect.ValueOf(&v)), nil + } + var vNil *net.IP + return reflect.Append(slice, reflect.ValueOf(vNil)), nil + }, + +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/string.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/string.go new file mode 100644 index 0000000000..a190660fa5 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/string.go @@ -0,0 +1,35 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type String struct{ base } + +func (String) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.String() + if err != nil { + return "", err + } + return v, nil +} + +func (str *String) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case string: + return encoder.String(v) + case []byte: + return encoder.RawString(v) + + // this relies on Nullable never sending nil values through + case *string: + return encoder.String(*v) + case *[]byte: + return encoder.RawString(*v) + } + + return &ErrUnexpectedType{ + T: v, + Column: str, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/tuple.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/tuple.go new file mode 100644 index 0000000000..00578a6e2c --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/tuple.go @@ -0,0 +1,110 @@ +package column + +import ( + "fmt" + "github.com/ClickHouse/clickhouse-go/lib/binary" + "reflect" + "strconv" + "time" +) + +type Tuple struct { + base + columns []Column +} + +func (tuple *Tuple) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + return nil, fmt.Errorf("do not use Read method for Tuple(T) column") +} + +func (tuple *Tuple) ReadTuple(decoder *binary.Decoder, rows int) ([]interface{}, error) { + var values = make([][]interface{}, rows) + + for _, c := range tuple.columns { + + switch column := c.(type) { + case *Array: + cols, err := column.ReadArray(decoder, rows) + if err != nil { + return nil, err + } + for i := 0; i < rows; i++ { + values[i] = append(values[i], cols[i]) + } + + case *Nullable: + cols, err := column.ReadNull(decoder, rows) + if err != nil { + return nil, err + } + for i := 0; i < rows; i++ { + values[i] = append(values[i], cols[i]) + } + + case *Tuple: + cols, err := column.ReadTuple(decoder, rows) + if err != nil { + return nil, err + } + for i := 0; i < rows; i++ { + values[i] = append(values[i], cols[i]) + } + + default: + for i := 0; i < rows; i++ { + value, err := c.Read(decoder, false) + if err != nil { + return nil, err + } + values[i] = append(values[i], value) + } + } + } + + var ret = make([]interface{}, rows) + for i := range values { + ret[i] = values[i] + } + + return ret, nil +} + +func (tuple *Tuple) Write(encoder *binary.Encoder, v interface{}) (err error) { + return fmt.Errorf("unsupported Tuple(T) type [%T]", v) +} + +func parseTuple(name, chType string, timezone *time.Location) (Column, error) { + var columnType = chType + + chType = chType[6 : len(chType)-1] + var types []string + var last, diff int + for i, b := range chType + "," { + if b == '(' { + diff++ + } else if b == ')' { + diff-- + } else if b == ',' && diff == 0 { + types = append(types, chType[last:i]) + last = i + 2 + } + } + + var columns = make([]Column, 0, len(types)) + for i, chType := range types { + column, err := Factory(name+"."+strconv.Itoa(i+1), chType, timezone) + if err != nil { + return nil, fmt.Errorf("%s: %v", chType, err) + } + columns = append(columns, column) + } + + return &Tuple{ + base: base{ + name: name, + chType: columnType, + valueOf: reflect.ValueOf([]interface{}{}), + }, + columns: columns, + }, nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint16.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint16.go new file mode 100644 index 0000000000..cf761efbcc --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint16.go @@ -0,0 +1,43 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type UInt16 struct{ base } + +func (UInt16) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.UInt16() + if err != nil { + return uint16(0), err + } + return v, nil +} + +func (u *UInt16) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case uint16: + return encoder.UInt16(v) + case int64: + return encoder.UInt16(uint16(v)) + case uint64: + return encoder.UInt16(uint16(v)) + case int: + return encoder.UInt16(uint16(v)) + + // this relies on Nullable never sending nil values through + case *uint16: + return encoder.UInt16(*v) + case *int64: + return encoder.UInt16(uint16(*v)) + case *uint64: + return encoder.UInt16(uint16(*v)) + case *int: + return encoder.UInt16(uint16(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: u, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint32.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint32.go new file mode 100644 index 0000000000..f5f72f5f16 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint32.go @@ -0,0 +1,43 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type UInt32 struct{ base } + +func (UInt32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.UInt32() + if err != nil { + return uint32(0), err + } + return v, nil +} + +func (u *UInt32) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case uint32: + return encoder.UInt32(v) + case uint64: + return encoder.UInt32(uint32(v)) + case int64: + return encoder.UInt32(uint32(v)) + case int: + return encoder.UInt32(uint32(v)) + + // this relies on Nullable never sending nil values through + case *uint64: + return encoder.UInt32(uint32(*v)) + case *uint32: + return encoder.UInt32(*v) + case *int64: + return encoder.UInt32(uint32(*v)) + case *int: + return encoder.UInt32(uint32(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: u, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint64.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint64.go new file mode 100644 index 0000000000..09261e0106 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint64.go @@ -0,0 +1,44 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type UInt64 struct{ base } + +func (UInt64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.UInt64() + if err != nil { + return uint64(0), err + } + return v, nil +} + +func (u *UInt64) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case []byte: + if _, err := encoder.Write(v); err != nil { + return err + } + return nil + case uint64: + return encoder.UInt64(v) + case int64: + return encoder.UInt64(uint64(v)) + case int: + return encoder.UInt64(uint64(v)) + + // this relies on Nullable never sending nil values through + case *uint64: + return encoder.UInt64(*v) + case *int64: + return encoder.UInt64(uint64(*v)) + case *int: + return encoder.UInt64(uint64(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: u, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint8.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint8.go new file mode 100644 index 0000000000..8af493e7a0 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint8.go @@ -0,0 +1,47 @@ +package column + +import ( + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type UInt8 struct{ base } + +func (UInt8) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + v, err := decoder.UInt8() + if err != nil { + return uint8(0), err + } + return v, nil +} + +func (u *UInt8) Write(encoder *binary.Encoder, v interface{}) error { + switch v := v.(type) { + case bool: + return encoder.Bool(v) + case uint8: + return encoder.UInt8(v) + case int64: + return encoder.UInt8(uint8(v)) + case uint64: + return encoder.UInt8(uint8(v)) + case int: + return encoder.UInt8(uint8(v)) + + // this relies on Nullable never sending nil values through + case *bool: + return encoder.Bool(*v) + case *uint8: + return encoder.UInt8(*v) + case *int64: + return encoder.UInt8(uint8(*v)) + case *uint64: + return encoder.UInt8(uint8(*v)) + case *int: + return encoder.UInt8(uint8(*v)) + } + + return &ErrUnexpectedType{ + T: v, + Column: u, + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uuid.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uuid.go new file mode 100644 index 0000000000..f778acc923 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/column/uuid.go @@ -0,0 +1,139 @@ +package column + +import ( + "encoding/hex" + "errors" + "fmt" + "reflect" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +const ( + UUIDLen = 16 + NullUUID = "00000000-0000-0000-0000-000000000000" +) + +var ErrInvalidUUIDFormat = errors.New("invalid UUID format") + +type UUID struct { + base + scanType reflect.Type +} + +func (*UUID) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) { + src, err := decoder.Fixed(UUIDLen) + if err != nil { + return "", err + } + + src = swap(src) + + var uuid [36]byte + { + hex.Encode(uuid[:], src[:4]) + uuid[8] = '-' + hex.Encode(uuid[9:13], src[4:6]) + uuid[13] = '-' + hex.Encode(uuid[14:18], src[6:8]) + uuid[18] = '-' + hex.Encode(uuid[19:23], src[8:10]) + uuid[23] = '-' + hex.Encode(uuid[24:], src[10:]) + } + return string(uuid[:]), nil +} + +func (u *UUID) Write(encoder *binary.Encoder, v interface{}) (err error) { + var uuid []byte + switch v := v.(type) { + case string: + if uuid, err = uuid2bytes(v); err != nil { + return err + } + case []byte: + if len(v) != UUIDLen { + return fmt.Errorf("invalid raw UUID len '%s' (expected %d, got %d)", uuid, UUIDLen, len(uuid)) + } + uuid = make([]byte, 16) + copy(uuid, v) + default: + return &ErrUnexpectedType{ + T: v, + Column: u, + } + } + + uuid = swap(uuid) + + if _, err := encoder.Write(uuid); err != nil { + return err + } + return nil +} + +func swap(src []byte) []byte { + _ = src[15] + src[0], src[7] = src[7], src[0] + src[1], src[6] = src[6], src[1] + src[2], src[5] = src[5], src[2] + src[3], src[4] = src[4], src[3] + src[8], src[15] = src[15], src[8] + src[9], src[14] = src[14], src[9] + src[10], src[13] = src[13], src[10] + src[11], src[12] = src[12], src[11] + return src +} + +func uuid2bytes(str string) ([]byte, error) { + var uuid [16]byte + strLength := len(str) + if strLength == 0 { + str = NullUUID + } else if strLength != 36 { + return nil, ErrInvalidUUIDFormat + } + if str[8] != '-' || str[13] != '-' || str[18] != '-' || str[23] != '-' { + return nil, ErrInvalidUUIDFormat + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, 14, 16, + 19, 21, 24, 26, + 28, 30, 32, 34, + } { + if v, ok := xtob(str[x], str[x+1]); !ok { + return nil, ErrInvalidUUIDFormat + } else { + uuid[i] = v + } + } + return uuid[:], nil +} + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts hex characters x1 and x2 into a byte. +func xtob(x1, x2 byte) (byte, bool) { + b1 := xvalues[x1] + b2 := xvalues[x2] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block.go new file mode 100644 index 0000000000..a3417821c6 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block.go @@ -0,0 +1,323 @@ +package data + +import ( + "bytes" + "database/sql/driver" + "fmt" + "io" + "reflect" + "strings" + + "github.com/ClickHouse/clickhouse-go/lib/binary" + "github.com/ClickHouse/clickhouse-go/lib/column" +) + +type offset [][]int + +type Block struct { + Values [][]interface{} + Columns []column.Column + NumRows uint64 + NumColumns uint64 + offsets []offset + buffers []*buffer + info blockInfo +} + +func (block *Block) Copy() *Block { + return &Block{ + Columns: block.Columns, + NumColumns: block.NumColumns, + info: block.info, + } +} + +func (block *Block) ColumnNames() []string { + names := make([]string, 0, len(block.Columns)) + for _, column := range block.Columns { + names = append(names, column.Name()) + } + return names +} + +func (block *Block) Read(serverInfo *ServerInfo, decoder *binary.Decoder) (err error) { + if serverInfo.Revision > 0 { + if err = block.info.read(decoder); err != nil { + return err + } + } + + if block.NumColumns, err = decoder.Uvarint(); err != nil { + return err + } + if block.NumRows, err = decoder.Uvarint(); err != nil { + return err + } + block.Values = make([][]interface{}, block.NumColumns) + if block.NumRows > 10 { + for i := 0; i < int(block.NumColumns); i++ { + block.Values[i] = make([]interface{}, 0, block.NumRows) + } + } + for i := 0; i < int(block.NumColumns); i++ { + var ( + value interface{} + columnName string + columnType string + ) + if columnName, err = decoder.String(); err != nil { + return err + } + if columnType, err = decoder.String(); err != nil { + return err + } + c, err := column.Factory(columnName, columnType, serverInfo.Timezone) + if err != nil { + return err + } + block.Columns = append(block.Columns, c) + switch column := c.(type) { + case *column.Array: + if block.Values[i], err = column.ReadArray(decoder, int(block.NumRows)); err != nil { + return err + } + case *column.Nullable: + if block.Values[i], err = column.ReadNull(decoder, int(block.NumRows)); err != nil { + return err + } + case *column.Tuple: + if block.Values[i], err = column.ReadTuple(decoder, int(block.NumRows)); err != nil { + return err + } + default: + for row := 0; row < int(block.NumRows); row++ { + if value, err = column.Read(decoder, false); err != nil { + return err + } + block.Values[i] = append(block.Values[i], value) + } + } + } + return nil +} + +func (block *Block) writeArray(col column.Column, value Value, num, level int) error { + if level > col.Depth() { + arrColumn, ok := col.(*column.Array) + if ok && strings.Contains(col.CHType(), "Nullable") { + return arrColumn.WriteNull(block.buffers[num].Offset, block.buffers[num].Column, value.Interface()) + } + return col.Write(block.buffers[num].Column, value.Interface()) + } + + switch { + case value.Kind() == reflect.Slice: + if len(block.offsets[num]) < level { + block.offsets[num] = append(block.offsets[num], []int{value.Len()}) + } else { + block.offsets[num][level-1] = append( + block.offsets[num][level-1], + block.offsets[num][level-1][len(block.offsets[num][level-1])-1]+value.Len(), + ) + } + for i := 0; i < value.Len(); i++ { + if err := block.writeArray(col, value.Index(i), num, level+1); err != nil { + return err + } + } + default: + if err := col.Write(block.buffers[num].Column, value.Interface()); err != nil { + return err + } + } + return nil +} + +func (block *Block) AppendRow(args []driver.Value) error { + if len(block.Columns) != len(args) { + return fmt.Errorf("block: expected %d arguments (columns: %s), got %d", len(block.Columns), strings.Join(block.ColumnNames(), ", "), len(args)) + } + block.Reserve() + { + block.NumRows++ + } + for num, c := range block.Columns { + switch column := c.(type) { + case *column.Array: + value := reflect.ValueOf(args[num]) + if value.Kind() != reflect.Slice { + return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface()) + } + if err := block.writeArray(c, newValue(value), num, 1); err != nil { + return err + } + case *column.Nullable: + if err := column.WriteNull(block.buffers[num].Offset, block.buffers[num].Column, args[num]); err != nil { + return err + } + default: + if err := column.Write(block.buffers[num].Column, args[num]); err != nil { + return err + } + } + } + return nil +} + +func (block *Block) Reserve() { + if len(block.buffers) == 0 { + block.buffers = make([]*buffer, len(block.Columns)) + block.offsets = make([]offset, len(block.Columns)) + for i := 0; i < len(block.Columns); i++ { + var ( + offsetBuffer = new(bytes.Buffer) + columnBuffer = new(bytes.Buffer) + ) + block.buffers[i] = &buffer{ + Offset: binary.NewEncoder(offsetBuffer), + Column: binary.NewEncoder(columnBuffer), + offsetBuffer: offsetBuffer, + columnBuffer: columnBuffer, + } + } + } +} + +func (block *Block) Reset() { + block.NumRows = 0 + block.NumColumns = 0 + block.Values = block.Values[:0] + block.Columns = block.Columns[:0] + block.info.reset() + for _, buffer := range block.buffers { + buffer.reset() + } + { + block.offsets = nil + block.buffers = nil + } +} + +func (block *Block) Write(serverInfo *ServerInfo, encoder *binary.Encoder) error { + if serverInfo.Revision > 0 { + if err := block.info.write(encoder); err != nil { + return err + } + } + if err := encoder.Uvarint(block.NumColumns); err != nil { + return err + } + encoder.Uvarint(block.NumRows) + defer func() { + block.NumRows = 0 + for i := range block.offsets { + block.offsets[i] = offset{} + } + }() + for i, column := range block.Columns { + encoder.String(column.Name()) + encoder.String(column.CHType()) + if len(block.buffers) == len(block.Columns) { + for _, offsets := range block.offsets[i] { + for _, offset := range offsets { + if err := encoder.UInt64(uint64(offset)); err != nil { + return err + } + } + } + if _, err := block.buffers[i].WriteTo(encoder); err != nil { + return err + } + } + } + return nil +} + +type blockInfo struct { + num1 uint64 + isOverflows bool + num2 uint64 + bucketNum int32 + num3 uint64 +} + +func (info *blockInfo) reset() { + info.num1 = 0 + info.isOverflows = false + info.num2 = 0 + info.bucketNum = 0 + info.num3 = 0 +} + +func (info *blockInfo) read(decoder *binary.Decoder) error { + var err error + if info.num1, err = decoder.Uvarint(); err != nil { + return err + } + if info.isOverflows, err = decoder.Bool(); err != nil { + return err + } + if info.num2, err = decoder.Uvarint(); err != nil { + return err + } + if info.bucketNum, err = decoder.Int32(); err != nil { + return err + } + if info.num3, err = decoder.Uvarint(); err != nil { + return err + } + return nil +} + +func (info *blockInfo) write(encoder *binary.Encoder) error { + if err := encoder.Uvarint(1); err != nil { + return err + } + if err := encoder.Bool(info.isOverflows); err != nil { + return err + } + if err := encoder.Uvarint(2); err != nil { + return err + } + if info.bucketNum == 0 { + info.bucketNum = -1 + } + if err := encoder.Int32(info.bucketNum); err != nil { + return err + } + if err := encoder.Uvarint(0); err != nil { + return err + } + return nil +} + +type buffer struct { + Offset *binary.Encoder + Column *binary.Encoder + offsetBuffer *bytes.Buffer + columnBuffer *bytes.Buffer +} + +func (buf *buffer) WriteTo(w io.Writer) (int64, error) { + var size int64 + { + ln, err := buf.offsetBuffer.WriteTo(w) + if err != nil { + return size, err + } + size += ln + } + { + ln, err := buf.columnBuffer.WriteTo(w) + if err != nil { + return size, err + } + size += ln + } + return size, nil +} + +func (buf *buffer) reset() { + buf.offsetBuffer.Reset() + buf.columnBuffer.Reset() +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block_write_column.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block_write_column.go new file mode 100644 index 0000000000..b769057ee6 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/block_write_column.go @@ -0,0 +1,200 @@ +package data + +import ( + "fmt" + "github.com/ClickHouse/clickhouse-go/lib/column" + "net" + "reflect" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +func (block *Block) WriteDate(c int, v time.Time) error { + _, offset := v.Zone() + nday := (v.Unix() + int64(offset)) / 24 / 3600 + return block.buffers[c].Column.UInt16(uint16(nday)) +} + +func (block *Block) WriteDateNullable(c int, v *time.Time) error { + if v == nil { + return block.buffers[c].Column.UInt16Nullable(nil) + } + unixTime := uint16((*v).Unix() / 24 / 3600) + return block.buffers[c].Column.UInt16Nullable(&unixTime) +} + +func (block *Block) WriteDateTime(c int, v time.Time) error { + return block.buffers[c].Column.UInt32(uint32(v.Unix())) +} + +func (block *Block) WriteBool(c int, v bool) error { + if v { + return block.buffers[c].Column.UInt8(1) + } + return block.buffers[c].Column.UInt8(0) +} + +func (block *Block) WriteDateTimeNullable(c int, v *time.Time) error { + if v == nil { + return block.buffers[c].Column.UInt32Nullable(nil) + } + unixTime := uint32((*v).Unix()) + return block.buffers[c].Column.UInt32Nullable(&unixTime) +} + +func (block *Block) WriteInt8(c int, v int8) error { + return block.buffers[c].Column.Int8(v) +} + +func (block *Block) WriteInt8Nullable(c int, v *int8) error { + return block.buffers[c].Column.Int8Nullable(v) +} + +func (block *Block) WriteInt16(c int, v int16) error { + return block.buffers[c].Column.Int16(v) +} + +func (block *Block) WriteInt16Nullable(c int, v *int16) error { + return block.buffers[c].Column.Int16Nullable(v) +} + +func (block *Block) WriteInt32(c int, v int32) error { + return block.buffers[c].Column.Int32(v) +} + +func (block *Block) WriteInt32Nullable(c int, v *int32) error { + return block.buffers[c].Column.Int32Nullable(v) +} + +func (block *Block) WriteInt64(c int, v int64) error { + return block.buffers[c].Column.Int64(v) +} + +func (block *Block) WriteInt64Nullable(c int, v *int64) error { + return block.buffers[c].Column.Int64Nullable(v) +} + +func (block *Block) WriteUInt8(c int, v uint8) error { + return block.buffers[c].Column.UInt8(v) +} + +func (block *Block) WriteUInt8Nullable(c int, v *uint8) error { + return block.buffers[c].Column.UInt8Nullable(v) +} + +func (block *Block) WriteUInt16(c int, v uint16) error { + return block.buffers[c].Column.UInt16(v) +} + +func (block *Block) WriteUInt16Nullable(c int, v *uint16) error { + return block.buffers[c].Column.UInt16Nullable(v) +} + +func (block *Block) WriteUInt32(c int, v uint32) error { + return block.buffers[c].Column.UInt32(v) +} + +func (block *Block) WriteUInt32Nullable(c int, v *uint32) error { + return block.buffers[c].Column.UInt32Nullable(v) +} + +func (block *Block) WriteUInt64(c int, v uint64) error { + return block.buffers[c].Column.UInt64(v) +} + +func (block *Block) WriteUInt64Nullable(c int, v *uint64) error { + return block.buffers[c].Column.UInt64Nullable(v) +} + +func (block *Block) WriteFloat32(c int, v float32) error { + return block.buffers[c].Column.Float32(v) +} + +func (block *Block) WriteFloat32Nullable(c int, v *float32) error { + return block.buffers[c].Column.Float32Nullable(v) +} + +func (block *Block) WriteFloat64(c int, v float64) error { + return block.buffers[c].Column.Float64(v) +} + +func (block *Block) WriteFloat64Nullable(c int, v *float64) error { + return block.buffers[c].Column.Float64Nullable(v) +} + +func (block *Block) WriteBytes(c int, v []byte) error { + if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil { + return err + } + if _, err := block.buffers[c].Column.Write(v); err != nil { + return err + } + return nil +} + +func (block *Block) WriteBytesNullable(c int, v *[]byte) error { + isNil := v == nil + if err := block.buffers[c].Column.Nullable(isNil); err != nil { + return err + } + if isNil { + return block.WriteBytes(c, []byte{}) + } + return block.WriteBytes(c, *v) +} + +func (block *Block) WriteString(c int, v string) error { + if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil { + return err + } + if _, err := block.buffers[c].Column.Write(binary.Str2Bytes(v)); err != nil { + return err + } + return nil +} + +func (block *Block) WriteStringNullable(c int, v *string) error { + isNil := v == nil + if err := block.buffers[c].Column.Nullable(isNil); err != nil { + return err + } + if isNil { + return block.WriteString(c, "") + } + return block.WriteString(c, *v) +} + +func (block *Block) WriteFixedString(c int, v []byte) error { + return block.Columns[c].Write(block.buffers[c].Column, v) +} + +func (block *Block) WriteIP(c int, v net.IP) error { + return block.Columns[c].Write(block.buffers[c].Column, v) +} + +func (block *Block) WriteArray(c int, v interface{}) error { + return block.WriteArrayWithValue(c, newValue(reflect.ValueOf(v))) +} + +func (block *Block) WriteArrayWithValue(c int, value Value) error { + if value.Kind() != reflect.Slice { + return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface()) + } + return block.writeArray(block.Columns[c], value, c, 1) +} + +func (block *Block) WriteFixedStringNullable(c int, v *[]byte) error { + writer := block.Columns[c].(*column.Nullable) + return writer.WriteNull(block.buffers[c].Offset, block.buffers[c].Column, v) +} + +func (block *Block) WriteArrayNullable(c int, v *interface{}) error { + if err := block.buffers[c].Column.Nullable(v == nil); err != nil { + return err + } + if v == nil { + return block.Columns[c].Write(block.buffers[c].Column, []string{}) + } + return block.WriteArray(c, *v) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/data/client_info.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/client_info.go new file mode 100644 index 0000000000..0bff41981f --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/client_info.go @@ -0,0 +1,29 @@ +package data + +import ( + "fmt" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +const ClientName = "Golang SQLDriver" + +const ( + ClickHouseRevision = 54213 + ClickHouseDBMSVersionMajor = 1 + ClickHouseDBMSVersionMinor = 1 +) + +type ClientInfo struct{} + +func (ClientInfo) Write(encoder *binary.Encoder) error { + encoder.String(ClientName) + encoder.Uvarint(ClickHouseDBMSVersionMajor) + encoder.Uvarint(ClickHouseDBMSVersionMinor) + encoder.Uvarint(ClickHouseRevision) + return nil +} + +func (ClientInfo) String() string { + return fmt.Sprintf("%s %d.%d.%d", ClientName, ClickHouseDBMSVersionMajor, ClickHouseDBMSVersionMinor, ClickHouseRevision) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/data/server_info.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/server_info.go new file mode 100644 index 0000000000..a6c90bed42 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/server_info.go @@ -0,0 +1,47 @@ +package data + +import ( + "fmt" + //"io" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/binary" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +type ServerInfo struct { + Name string + Revision uint64 + MinorVersion uint64 + MajorVersion uint64 + Timezone *time.Location +} + +func (srv *ServerInfo) Read(decoder *binary.Decoder) (err error) { + if srv.Name, err = decoder.String(); err != nil { + return fmt.Errorf("could not read server name: %v", err) + } + if srv.MajorVersion, err = decoder.Uvarint(); err != nil { + return fmt.Errorf("could not read server major version: %v", err) + } + if srv.MinorVersion, err = decoder.Uvarint(); err != nil { + return fmt.Errorf("could not read server minor version: %v", err) + } + if srv.Revision, err = decoder.Uvarint(); err != nil { + return fmt.Errorf("could not read server revision: %v", err) + } + if srv.Revision >= protocol.DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE { + timezone, err := decoder.String() + if err != nil { + return fmt.Errorf("could not read server timezone: %v", err) + } + if srv.Timezone, err = time.LoadLocation(timezone); err != nil { + return fmt.Errorf("could not load time location: %v", err) + } + } + return nil +} + +func (srv ServerInfo) String() string { + return fmt.Sprintf("%s %d.%d.%d (%s)", srv.Name, srv.MajorVersion, srv.MinorVersion, srv.Revision, srv.Timezone) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/data/value.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/value.go new file mode 100644 index 0000000000..3c7e4ede9b --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/data/value.go @@ -0,0 +1,33 @@ +package data + +import "reflect" + +// Value is a writable value. +type Value interface { + // Kind returns value's Kind. + Kind() reflect.Kind + + // Len returns value's length. + // It panics if value's Kind is not Array, Chan, Map, Slice, or String. + Len() int + + // Index returns value's i'th element. + // It panics if value's Kind is not Array, Slice, or String or i is out of range. + Index(i int) Value + + // Interface returns value's current value as an interface{}. + Interface() interface{} +} + +// value is a wrapper that wraps reflect.Value to comply with Value interface. +type value struct { + reflect.Value +} + +func newValue(v reflect.Value) Value { + return value{Value: v} +} + +func (v value) Index(i int) Value { + return newValue(v.Value.Index(i)) +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/LICENSE b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/LICENSE new file mode 100644 index 0000000000..85cbc1df59 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/LICENSE @@ -0,0 +1,23 @@ +Copyright 2011-2012 Branimir Karadzic. All rights reserved. +Copyright 2013 Damian Gryski. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/doc.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/doc.go new file mode 100644 index 0000000000..091981c8fb --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/doc.go @@ -0,0 +1,7 @@ +// Copyright 2011-2012 Branimir Karadzic. All rights reserved. +// Copyright 2013 Damian Gryski. All rights reserved. + +// @LINK: https://github.com/bkaradzic/go-lz4 +// @NOTE: The code is modified to be high performance and less memory usage + +package lz4 diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/fuzz.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/fuzz.go new file mode 100644 index 0000000000..e4989de9ff --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/fuzz.go @@ -0,0 +1,23 @@ +// +build gofuzz + +package lz4 + +import "encoding/binary" + +func Fuzz(data []byte) int { + + if len(data) < 4 { + return 0 + } + + ln := binary.LittleEndian.Uint32(data) + if ln > (1 << 21) { + return 0 + } + + if _, err := Decode(nil, data); err != nil { + return 0 + } + + return 1 +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/reader.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/reader.go new file mode 100644 index 0000000000..f8d38ea27f --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/reader.go @@ -0,0 +1,179 @@ +/* + * Copyright 2011-2012 Branimir Karadzic. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +package lz4 + +import ( + "errors" + "io" +) + +var ( + // ErrCorrupt indicates the input was corrupt + ErrCorrupt = errors.New("corrupt input") +) + +const ( + mlBits = 4 + mlMask = (1 << mlBits) - 1 + runBits = 8 - mlBits + runMask = (1 << runBits) - 1 +) + +type decoder struct { + src []byte + dst []byte + spos uint32 + dpos uint32 + ref uint32 +} + +func (d *decoder) readByte() (uint8, error) { + if int(d.spos) == len(d.src) { + return 0, io.EOF + } + b := d.src[d.spos] + d.spos++ + return b, nil +} + +func (d *decoder) getLen() (uint32, error) { + + length := uint32(0) + ln, err := d.readByte() + if err != nil { + return 0, ErrCorrupt + } + for ln == 255 { + length += 255 + ln, err = d.readByte() + if err != nil { + return 0, ErrCorrupt + } + } + length += uint32(ln) + + return length, nil +} + +func (d *decoder) cp(length, decr uint32) { + + if int(d.ref+length) < int(d.dpos) { + copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length]) + } else { + for ii := uint32(0); ii < length; ii++ { + d.dst[d.dpos+ii] = d.dst[d.ref+ii] + } + } + d.dpos += length + d.ref += length - decr +} + +func (d *decoder) finish(err error) error { + if err == io.EOF { + return nil + } + + return err +} + +// Decode returns the decoded form of src. The returned slice may be a +// subslice of dst if it was large enough to hold the entire decoded block. +func Decode(dst, src []byte) (int, error) { + d := decoder{src: src, dst: dst, spos: 0} + + decr := []uint32{0, 3, 2, 3} + + for { + code, err := d.readByte() + if err != nil { + return len(d.dst), d.finish(err) + } + + length := uint32(code >> mlBits) + if length == runMask { + ln, err := d.getLen() + if err != nil { + return 0, ErrCorrupt + } + length += ln + } + + if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) { + return 0, ErrCorrupt + } + + for ii := uint32(0); ii < length; ii++ { + d.dst[d.dpos+ii] = d.src[d.spos+ii] + } + + d.spos += length + d.dpos += length + + if int(d.spos) == len(d.src) { + return len(d.dst), nil + } + + if int(d.spos+2) >= len(d.src) { + return 0, ErrCorrupt + } + + back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8 + + if back > d.dpos { + return 0, ErrCorrupt + } + + d.spos += 2 + d.ref = d.dpos - back + + length = uint32(code & mlMask) + if length == mlMask { + ln, err := d.getLen() + if err != nil { + return 0, ErrCorrupt + } + length += ln + } + + literal := d.dpos - d.ref + + if literal < 4 { + if int(d.dpos+4) > len(d.dst) { + return 0, ErrCorrupt + } + + d.cp(4, decr[literal]) + } else { + length += 4 + } + + if int(d.dpos+length) > len(d.dst) { + return 0, ErrCorrupt + } + + d.cp(length, 0) + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/writer.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/writer.go new file mode 100644 index 0000000000..969311beaf --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/lz4/writer.go @@ -0,0 +1,203 @@ +/* + * Copyright 2011-2012 Branimir Karadzic. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +package lz4 + +import ( + "errors" + "sync" +) + +const ( + minMatch = 4 + hashLog = 16 + hashTableSize = 1 << hashLog + hashShift = (minMatch * 8) - hashLog + incompressible uint32 = 128 + uninitHash = 0x88888888 + + mfLimit = 8 + minMatch // The last match cannot start within the last 12 bytes. + // MaxInputSize is the largest buffer than can be compressed in a single block + MaxInputSize = 0x7E000000 +) + +var ( + // ErrTooLarge indicates the input buffer was too large + ErrTooLarge = errors.New("input too large") + ErrEncodeTooSmall = errors.New("encode buffer too small") + + hashPool = sync.Pool{ + New: func() interface{} { + return make([]uint32, hashTableSize) + }, + } +) + +type encoder struct { + src []byte + dst []byte + hashTable []uint32 + pos uint32 + anchor uint32 + dpos uint32 +} + +// CompressBound returns the maximum length of a lz4 block +func CompressBound(isize int) int { + if isize > MaxInputSize { + return 0 + } + return isize + ((isize) / 255) + 16 +} + +func (e *encoder) writeLiterals(length, mlLen, pos uint32) { + + ln := length + + var code byte + if ln > runMask-1 { + code = runMask + } else { + code = byte(ln) + } + + if mlLen > mlMask-1 { + e.dst[e.dpos] = (code << mlBits) + byte(mlMask) + } else { + e.dst[e.dpos] = (code << mlBits) + byte(mlLen) + } + e.dpos++ + + if code == runMask { + ln -= runMask + for ; ln > 254; ln -= 255 { + e.dst[e.dpos] = 255 + e.dpos++ + } + + e.dst[e.dpos] = byte(ln) + e.dpos++ + } + + for ii := uint32(0); ii < length; ii++ { + e.dst[e.dpos+ii] = e.src[pos+ii] + } + + e.dpos += length +} + +// Encode returns the encoded form of src. The returned array may be a +// sub-slice of dst if it was large enough to hold the entire output. +func Encode(dst, src []byte) (compressedSize int, error error) { + if len(src) >= MaxInputSize { + return 0, ErrTooLarge + } + + if n := CompressBound(len(src)); len(dst) < n { + return 0, ErrEncodeTooSmall + } + + hashTable := hashPool.Get().([]uint32) + for i := range hashTable { + hashTable[i] = 0 + } + e := encoder{src: src, dst: dst, hashTable: hashTable} + defer func() { + hashPool.Put(hashTable) + }() + // binary.LittleEndian.PutUint32(dst, uint32(len(src))) + // e.dpos = 0 + + var ( + step uint32 = 1 + limit = incompressible + ) + + for { + if int(e.pos)+12 >= len(e.src) { + e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor) + return int(e.dpos), nil + } + + sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0]) + + hash := (sequence * 2654435761) >> hashShift + ref := e.hashTable[hash] + uninitHash + e.hashTable[hash] = e.pos - uninitHash + + if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence { + if e.pos-e.anchor > limit { + limit <<= 1 + step += 1 + (step >> 2) + } + e.pos += step + continue + } + + if step > 1 { + e.hashTable[hash] = ref - uninitHash + e.pos -= step - 1 + step = 1 + continue + } + limit = incompressible + + ln := e.pos - e.anchor + back := e.pos - ref + + anchor := e.anchor + + e.pos += minMatch + ref += minMatch + e.anchor = e.pos + + for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] { + e.pos++ + ref++ + } + + mlLen := e.pos - e.anchor + + e.writeLiterals(ln, mlLen, anchor) + e.dst[e.dpos] = uint8(back) + e.dst[e.dpos+1] = uint8(back >> 8) + e.dpos += 2 + + if mlLen > mlMask-1 { + mlLen -= mlMask + for mlLen > 254 { + mlLen -= 255 + + e.dst[e.dpos] = 255 + e.dpos++ + } + + e.dst[e.dpos] = byte(mlLen) + e.dpos++ + } + + e.anchor = e.pos + } +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/README.md b/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/README.md new file mode 100644 index 0000000000..84fdd48c6e --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/README.md @@ -0,0 +1,4 @@ +# ClickHouse Native protocol + +# Handshake + diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/protocol.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/protocol.go new file mode 100644 index 0000000000..ba5fd7c7ba --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/protocol/protocol.go @@ -0,0 +1,35 @@ +package protocol + +const ( + DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE = 54058 + DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO = 54060 +) + +const ( + ClientHello = 0 + ClientQuery = 1 + ClientData = 2 + ClientCancel = 3 + ClientPing = 4 +) + +const ( + CompressEnable uint64 = 1 + CompressDisable uint64 = 0 +) + +const ( + StateComplete = 2 +) + +const ( + ServerHello = 0 + ServerData = 1 + ServerException = 2 + ServerProgress = 3 + ServerPong = 4 + ServerEndOfStream = 5 + ServerProfileInfo = 6 + ServerTotals = 7 + ServerExtremes = 8 +) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/types/date.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/types/date.go new file mode 100644 index 0000000000..a7d0143b63 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/types/date.go @@ -0,0 +1,48 @@ +// Timezoneless date/datetime types + +package types + +import ( + "database/sql/driver" + "time" +) + +// Truncate timezone +// +// clickhouse.Date(time.Date(2017, 1, 1, 0, 0, 0, 0, time.Local)) -> time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) +type Date time.Time + +func (date Date) Value() (driver.Value, error) { + return date.convert(), nil +} + +func (date Date) convert() time.Time { + return time.Date(time.Time(date).Year(), time.Time(date).Month(), time.Time(date).Day(), 0, 0, 0, 0, time.UTC) +} + +// Truncate timezone +// +// clickhouse.DateTime(time.Date(2017, 1, 1, 0, 0, 0, 0, time.Local)) -> time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) +type DateTime time.Time + +func (datetime DateTime) Value() (driver.Value, error) { + return datetime.convert(), nil +} + +func (datetime DateTime) convert() time.Time { + return time.Date( + time.Time(datetime).Year(), + time.Time(datetime).Month(), + time.Time(datetime).Day(), + time.Time(datetime).Hour(), + time.Time(datetime).Minute(), + time.Time(datetime).Second(), + 1, + time.UTC, + ) +} + +var ( + _ driver.Valuer = Date{} + _ driver.Valuer = DateTime{} +) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/lib/types/uuid.go b/vendor/github.com/ClickHouse/clickhouse-go/lib/types/uuid.go new file mode 100644 index 0000000000..dc270a85f8 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/lib/types/uuid.go @@ -0,0 +1,99 @@ +package types + +import ( + "database/sql/driver" + "encoding/hex" + "errors" + "fmt" +) + +var InvalidUUIDFormatError = errors.New("invalid UUID format") + +// this type will be deprecated because the ClickHouse server (>=1.1.54276) has a built-in type UUID +type UUID string + +func (str UUID) Value() (driver.Value, error) { + return uuid2bytes(string(str)) +} + +func (str UUID) MarshalBinary() ([]byte, error) { + return uuid2bytes(string(str)) +} + +func (str *UUID) Scan(v interface{}) error { + var src []byte + switch v := v.(type) { + case string: + src = []byte(v) + case []byte: + src = v + } + + if len(src) != 16 { + return fmt.Errorf("invalid UUID length: %d", len(src)) + } + + var uuid [36]byte + { + hex.Encode(uuid[:], src[:4]) + uuid[8] = '-' + hex.Encode(uuid[9:13], src[4:6]) + uuid[13] = '-' + hex.Encode(uuid[14:18], src[6:8]) + uuid[18] = '-' + hex.Encode(uuid[19:23], src[8:10]) + uuid[23] = '-' + hex.Encode(uuid[24:], src[10:]) + } + *str = UUID(uuid[:]) + return nil +} + +func uuid2bytes(str string) ([]byte, error) { + var uuid [16]byte + if str[8] != '-' || str[13] != '-' || str[18] != '-' || str[23] != '-' { + return nil, InvalidUUIDFormatError + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, 14, 16, + 19, 21, 24, 26, + 28, 30, 32, 34, + } { + if v, ok := xtob(str[x], str[x+1]); !ok { + return nil, InvalidUUIDFormatError + } else { + uuid[i] = v + } + } + return uuid[:], nil +} + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts hex characters x1 and x2 into a byte. +func xtob(x1, x2 byte) (byte, bool) { + b1 := xvalues[x1] + b2 := xvalues[x2] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} + +var _ driver.Valuer = UUID("") diff --git a/vendor/github.com/ClickHouse/clickhouse-go/query_settings.go b/vendor/github.com/ClickHouse/clickhouse-go/query_settings.go new file mode 100644 index 0000000000..506728cc47 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/query_settings.go @@ -0,0 +1,286 @@ +package clickhouse + +import ( + "fmt" + "net/url" + "strconv" + + "github.com/ClickHouse/clickhouse-go/lib/binary" +) + +type querySettingType int + +// all possible query setting's data type +const ( + uintQS querySettingType = iota + 1 + intQS + boolQS + timeQS +) + +// description of single query setting +type querySettingInfo struct { + name string + qsType querySettingType +} + +// all possible query settings +var querySettingList = []querySettingInfo{ + {"min_compress_block_size", uintQS}, + {"max_compress_block_size", uintQS}, + {"max_block_size", uintQS}, + {"max_insert_block_size", uintQS}, + {"min_insert_block_size_rows", uintQS}, + {"min_insert_block_size_bytes", uintQS}, + {"max_read_buffer_size", uintQS}, + {"max_distributed_connections", uintQS}, + {"max_query_size", uintQS}, + {"interactive_delay", uintQS}, + {"poll_interval", uintQS}, + {"distributed_connections_pool_size", uintQS}, + {"connections_with_failover_max_tries", uintQS}, + {"background_pool_size", uintQS}, + {"background_schedule_pool_size", uintQS}, + {"replication_alter_partitions_sync", uintQS}, + {"replication_alter_columns_timeout", uintQS}, + {"min_count_to_compile", uintQS}, + {"min_count_to_compile_expression", uintQS}, + {"group_by_two_level_threshold", uintQS}, + {"group_by_two_level_threshold_bytes", uintQS}, + {"aggregation_memory_efficient_merge_threads", uintQS}, + {"max_parallel_replicas", uintQS}, + {"parallel_replicas_count", uintQS}, + {"parallel_replica_offset", uintQS}, + {"merge_tree_min_rows_for_concurrent_read", uintQS}, + {"merge_tree_min_bytes_for_concurrent_read", uintQS}, + {"merge_tree_min_rows_for_seek", uintQS}, + {"merge_tree_min_bytes_for_seek", uintQS}, + {"merge_tree_coarse_index_granularity", uintQS}, + {"merge_tree_max_rows_to_use_cache", uintQS}, + {"merge_tree_max_bytes_to_use_cache", uintQS}, + {"mysql_max_rows_to_insert", uintQS}, + {"optimize_min_equality_disjunction_chain_length", uintQS}, + {"min_bytes_to_use_direct_io", uintQS}, + {"mark_cache_min_lifetime", uintQS}, + {"priority", uintQS}, + {"log_queries_cut_to_length", uintQS}, + {"max_concurrent_queries_for_user", uintQS}, + {"insert_quorum", uintQS}, + {"select_sequential_consistency", uintQS}, + {"table_function_remote_max_addresses", uintQS}, + {"read_backoff_max_throughput", uintQS}, + {"read_backoff_min_events", uintQS}, + {"output_format_pretty_max_rows", uintQS}, + {"output_format_pretty_max_column_pad_width", uintQS}, + {"output_format_parquet_row_group_size", uintQS}, + {"http_headers_progress_interval_ms", uintQS}, + {"input_format_allow_errors_num", uintQS}, + {"preferred_block_size_bytes", uintQS}, + {"max_replica_delay_for_distributed_queries", uintQS}, + {"preferred_max_column_in_block_size_bytes", uintQS}, + {"insert_distributed_timeout", uintQS}, + {"odbc_max_field_size", uintQS}, + {"max_rows_to_read", uintQS}, + {"max_bytes_to_read", uintQS}, + {"max_rows_to_group_by", uintQS}, + {"max_bytes_before_external_group_by", uintQS}, + {"max_rows_to_sort", uintQS}, + {"max_bytes_to_sort", uintQS}, + {"max_bytes_before_external_sort", uintQS}, + {"max_bytes_before_remerge_sort", uintQS}, + {"max_result_rows", uintQS}, + {"max_result_bytes", uintQS}, + {"min_execution_speed", uintQS}, + {"max_execution_speed", uintQS}, + {"min_execution_speed_bytes", uintQS}, + {"max_execution_speed_bytes", uintQS}, + {"max_columns_to_read", uintQS}, + {"max_temporary_columns", uintQS}, + {"max_temporary_non_const_columns", uintQS}, + {"max_subquery_depth", uintQS}, + {"max_pipeline_depth", uintQS}, + {"max_ast_depth", uintQS}, + {"max_ast_elements", uintQS}, + {"max_expanded_ast_elements", uintQS}, + {"readonly", uintQS}, + {"max_rows_in_set", uintQS}, + {"max_bytes_in_set", uintQS}, + {"max_rows_in_join", uintQS}, + {"max_bytes_in_join", uintQS}, + {"max_rows_to_transfer", uintQS}, + {"max_bytes_to_transfer", uintQS}, + {"max_rows_in_distinct", uintQS}, + {"max_bytes_in_distinct", uintQS}, + {"max_memory_usage", uintQS}, + {"max_memory_usage_for_user", uintQS}, + {"max_memory_usage_for_all_queries", uintQS}, + {"max_network_bandwidth", uintQS}, + {"max_network_bytes", uintQS}, + {"max_network_bandwidth_for_user", uintQS}, + {"max_network_bandwidth_for_all_users", uintQS}, + {"low_cardinality_max_dictionary_size", uintQS}, + {"max_fetch_partition_retries_count", uintQS}, + {"http_max_multipart_form_data_size", uintQS}, + {"max_partitions_per_insert_block", uintQS}, + {"max_threads", uintQS}, + {"optimize_skip_unused_shards_nesting", uintQS}, + {"force_optimize_skip_unused_shards", uintQS}, + {"force_optimize_skip_unused_shards_nesting", uintQS}, + + {"network_zstd_compression_level", intQS}, + {"http_zlib_compression_level", intQS}, + {"distributed_ddl_task_timeout", intQS}, + + {"extremes", boolQS}, + {"use_uncompressed_cache", boolQS}, + {"replace_running_query", boolQS}, + {"distributed_directory_monitor_batch_inserts", boolQS}, + {"optimize_move_to_prewhere", boolQS}, + {"compile", boolQS}, + {"allow_suspicious_low_cardinality_types", boolQS}, + {"compile_expressions", boolQS}, + {"distributed_aggregation_memory_efficient", boolQS}, + {"skip_unavailable_shards", boolQS}, + {"distributed_group_by_no_merge", boolQS}, + {"optimize_skip_unused_shards", boolQS}, + {"merge_tree_uniform_read_distribution", boolQS}, + {"force_index_by_date", boolQS}, + {"force_primary_key", boolQS}, + {"log_queries", boolQS}, + {"insert_deduplicate", boolQS}, + {"enable_http_compression", boolQS}, + {"http_native_compression_disable_checksumming_on_decompress", boolQS}, + {"output_format_write_statistics", boolQS}, + {"add_http_cors_header", boolQS}, + {"input_format_skip_unknown_fields", boolQS}, + {"input_format_with_names_use_header", boolQS}, + {"input_format_import_nested_json", boolQS}, + {"input_format_defaults_for_omitted_fields", boolQS}, + {"input_format_values_interpret_expressions", boolQS}, + {"output_format_json_quote_64bit_integers", boolQS}, + {"output_format_json_quote_denormals", boolQS}, + {"output_format_json_escape_forward_slashes", boolQS}, + {"output_format_pretty_color", boolQS}, + {"use_client_time_zone", boolQS}, + {"send_progress_in_http_headers", boolQS}, + {"fsync_metadata", boolQS}, + {"join_use_nulls", boolQS}, + {"fallback_to_stale_replicas_for_distributed_queries", boolQS}, + {"insert_distributed_sync", boolQS}, + {"insert_allow_materialized_columns", boolQS}, + {"optimize_throw_if_noop", boolQS}, + {"use_index_for_in_with_subqueries", boolQS}, + {"empty_result_for_aggregation_by_empty_set", boolQS}, + {"allow_distributed_ddl", boolQS}, + {"join_any_take_last_row", boolQS}, + {"format_csv_allow_single_quotes", boolQS}, + {"format_csv_allow_double_quotes", boolQS}, + {"log_profile_events", boolQS}, + {"log_query_settings", boolQS}, + {"log_query_threads", boolQS}, + {"enable_optimize_predicate_expression", boolQS}, + {"low_cardinality_use_single_dictionary_for_part", boolQS}, + {"decimal_check_overflow", boolQS}, + {"prefer_localhost_replica", boolQS}, + //{"asterisk_left_columns_only", boolQS}, + {"calculate_text_stack_trace", boolQS}, + {"allow_ddl", boolQS}, + {"parallel_view_processing", boolQS}, + {"enable_debug_queries", boolQS}, + {"enable_unaligned_array_join", boolQS}, + {"low_cardinality_allow_in_native_format", boolQS}, + {"allow_experimental_multiple_joins_emulation", boolQS}, + {"allow_experimental_cross_to_join_conversion", boolQS}, + {"cancel_http_readonly_queries_on_client_close", boolQS}, + {"external_table_functions_use_nulls", boolQS}, + {"allow_experimental_data_skipping_indices", boolQS}, + {"allow_hyperscan", boolQS}, + {"allow_simdjson", boolQS}, + + {"connect_timeout", timeQS}, + {"connect_timeout_with_failover_ms", timeQS}, + {"receive_timeout", timeQS}, + {"send_timeout", timeQS}, + {"tcp_keep_alive_timeout", timeQS}, + {"queue_max_wait_ms", timeQS}, + {"distributed_directory_monitor_sleep_time_ms", timeQS}, + {"insert_quorum_timeout", timeQS}, + {"read_backoff_min_latency_ms", timeQS}, + {"read_backoff_min_interval_between_events_ms", timeQS}, + {"stream_flush_interval_ms", timeQS}, + {"stream_poll_timeout_ms", timeQS}, + {"http_connection_timeout", timeQS}, + {"http_send_timeout", timeQS}, + {"http_receive_timeout", timeQS}, + {"max_execution_time", timeQS}, + {"timeout_before_checking_execution_speed", timeQS}, +} + +type querySettingValueEncoder func(enc *binary.Encoder) error + +type querySettings struct { + settings map[string]querySettingValueEncoder + settingsStr string // used for debug output +} + +func makeQuerySettings(query url.Values) (*querySettings, error) { + qs := &querySettings{ + settings: make(map[string]querySettingValueEncoder), + settingsStr: "", + } + + for _, info := range querySettingList { + valueStr := query.Get(info.name) + if valueStr == "" { + continue + } + + switch info.qsType { + case uintQS, intQS, timeQS: + value, err := strconv.ParseUint(valueStr, 10, 64) + if err != nil { + return nil, err + } + qs.settings[info.name] = func(enc *binary.Encoder) error { return enc.Uvarint(value) } + + case boolQS: + valueBool, err := strconv.ParseBool(valueStr) + if err != nil { + return nil, err + } + value := uint64(0) + if valueBool { + value = 1 + } + qs.settings[info.name] = func(enc *binary.Encoder) error { return enc.Uvarint(value) } + + default: + err := fmt.Errorf("query setting %s has unsupported data type", info.name) + return nil, err + } + + if qs.settingsStr != "" { + qs.settingsStr += "&" + } + qs.settingsStr += info.name + "=" + valueStr + } + + return qs, nil +} + +func (qs *querySettings) IsEmpty() bool { + return len(qs.settings) == 0 +} + +func (qs *querySettings) Serialize(enc *binary.Encoder) error { + for name, fn := range qs.settings { + if err := enc.String(name); err != nil { + return err + } + if err := fn(enc); err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/result.go b/vendor/github.com/ClickHouse/clickhouse-go/result.go new file mode 100644 index 0000000000..fc08cf4dc4 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/result.go @@ -0,0 +1,8 @@ +package clickhouse + +import "errors" + +type result struct{} + +func (*result) LastInsertId() (int64, error) { return 0, errors.New("LastInsertId is not supported") } +func (*result) RowsAffected() (int64, error) { return 0, errors.New("RowsAffected is not supported") } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/rows.go b/vendor/github.com/ClickHouse/clickhouse-go/rows.go new file mode 100644 index 0000000000..30e844f651 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/rows.go @@ -0,0 +1,182 @@ +package clickhouse + +import ( + "database/sql/driver" + "fmt" + "io" + "reflect" + "sync" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/column" + "github.com/ClickHouse/clickhouse-go/lib/data" + "github.com/ClickHouse/clickhouse-go/lib/protocol" +) + +type rows struct { + ch *clickhouse + err error + mutex sync.RWMutex + finish func() + offset int + block *data.Block + totals *data.Block + extremes *data.Block + stream chan *data.Block + columns []string + blockColumns []column.Column +} + +func (rows *rows) Columns() []string { + return rows.columns +} + +func (rows *rows) ColumnTypeScanType(idx int) reflect.Type { + return rows.blockColumns[idx].ScanType() +} + +func (rows *rows) ColumnTypeDatabaseTypeName(idx int) string { + return rows.blockColumns[idx].CHType() +} + +func (rows *rows) Next(dest []driver.Value) error { + if rows.block == nil || int(rows.block.NumRows) <= rows.offset { + switch block, ok := <-rows.stream; true { + case !ok: + if err := rows.error(); err != nil { + return err + } + return io.EOF + default: + rows.block = block + rows.offset = 0 + } + } + for i := range dest { + dest[i] = rows.block.Values[i][rows.offset] + } + rows.offset++ + return nil +} + +func (rows *rows) HasNextResultSet() bool { + return rows.totals != nil || rows.extremes != nil +} + +func (rows *rows) NextResultSet() error { + switch { + case rows.totals != nil: + rows.block = rows.totals + rows.offset = 0 + rows.totals = nil + case rows.extremes != nil: + rows.block = rows.extremes + rows.offset = 0 + rows.extremes = nil + default: + return io.EOF + } + return nil +} + +func (rows *rows) receiveData() error { + defer close(rows.stream) + var ( + err error + packet uint64 + progress *progress + profileInfo *profileInfo + ) + for { + if packet, err = rows.ch.decoder.Uvarint(); err != nil { + return rows.setError(err) + } + switch packet { + case protocol.ServerException: + rows.ch.logf("[rows] <- exception") + return rows.setError(rows.ch.exception()) + case protocol.ServerProgress: + if progress, err = rows.ch.progress(); err != nil { + return rows.setError(err) + } + rows.ch.logf("[rows] <- progress: rows=%d, bytes=%d, total rows=%d", + progress.rows, + progress.bytes, + progress.totalRows, + ) + case protocol.ServerProfileInfo: + if profileInfo, err = rows.ch.profileInfo(); err != nil { + return rows.setError(err) + } + rows.ch.logf("[rows] <- profiling: rows=%d, bytes=%d, blocks=%d", profileInfo.rows, profileInfo.bytes, profileInfo.blocks) + case protocol.ServerData, protocol.ServerTotals, protocol.ServerExtremes: + var ( + block *data.Block + begin = time.Now() + ) + if block, err = rows.ch.readBlock(); err != nil { + return rows.setError(err) + } + rows.ch.logf("[rows] <- data: packet=%d, columns=%d, rows=%d, elapsed=%s", packet, block.NumColumns, block.NumRows, time.Since(begin)) + if block.NumRows == 0 { + continue + } + switch packet { + case protocol.ServerData: + rows.stream <- block + case protocol.ServerTotals: + rows.totals = block + case protocol.ServerExtremes: + rows.extremes = block + } + case protocol.ServerEndOfStream: + rows.ch.logf("[rows] <- end of stream") + return nil + default: + rows.ch.conn.Close() + rows.ch.logf("[rows] unexpected packet [%d]", packet) + return rows.setError(fmt.Errorf("[rows] unexpected packet [%d] from server", packet)) + } + } +} + +func (rows *rows) Close() error { + rows.ch.logf("[rows] close") + rows.columns = nil + for range rows.stream { + } + rows.finish() + return nil +} + +func (rows *rows) error() error { + rows.mutex.RLock() + defer rows.mutex.RUnlock() + return rows.err +} + +func (rows *rows) setError(err error) error { + rows.mutex.Lock() + rows.err = err + rows.mutex.Unlock() + return err +} + +func (rows *rows) ColumnTypeNullable(idx int) (nullable, ok bool) { + _, ok = rows.blockColumns[idx].(*column.Nullable) + return ok, true +} + +func (rows *rows) ColumnTypePrecisionScale(idx int) (precision, scale int64, ok bool) { + decimalVal, ok := rows.blockColumns[idx].(*column.Decimal) + if !ok { + if nullable, nullOk := rows.blockColumns[idx].(*column.Nullable); nullOk { + decimalVal, ok = nullable.GetColumn().(*column.Decimal) + } + } + if ok { + return int64(decimalVal.GetPrecision()), int64(decimalVal.GetScale()), ok + + } + return 0, 0, false +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/stmt.go b/vendor/github.com/ClickHouse/clickhouse-go/stmt.go new file mode 100644 index 0000000000..062b4b1b9d --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/stmt.go @@ -0,0 +1,214 @@ +package clickhouse + +import ( + "bytes" + "context" + "database/sql/driver" + "unicode" + + "github.com/ClickHouse/clickhouse-go/lib/data" +) + +type stmt struct { + ch *clickhouse + query string + counter int + numInput int + isInsert bool +} + +var emptyResult = &result{} + +type key string + +var queryIDKey key + +//Put query ID into context and use it in ExecContext or QueryContext +func WithQueryID(ctx context.Context, queryID string) context.Context { + return context.WithValue(ctx, queryIDKey, queryID) +} + +func (stmt *stmt) NumInput() int { + switch { + case stmt.ch.block != nil: + return len(stmt.ch.block.Columns) + case stmt.numInput < 0: + return 0 + } + return stmt.numInput +} + +func (stmt *stmt) Exec(args []driver.Value) (driver.Result, error) { + return stmt.execContext(context.Background(), args) +} + +func (stmt *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + dargs := make([]driver.Value, len(args)) + for i, nv := range args { + dargs[i] = nv.Value + } + return stmt.execContext(ctx, dargs) +} + +func (stmt *stmt) execContext(ctx context.Context, args []driver.Value) (driver.Result, error) { + if stmt.isInsert { + stmt.counter++ + if err := stmt.ch.block.AppendRow(args); err != nil { + return nil, err + } + if (stmt.counter % stmt.ch.blockSize) == 0 { + stmt.ch.logf("[exec] flush block") + if err := stmt.ch.writeBlock(stmt.ch.block, ""); err != nil { + return nil, err + } + if err := stmt.ch.encoder.Flush(); err != nil { + return nil, err + } + } + return emptyResult, nil + } + query, externalTables := stmt.bind(convertOldArgs(args)) + if err := stmt.ch.sendQuery(ctx, query, externalTables); err != nil { + return nil, err + } + if err := stmt.ch.process(); err != nil { + return nil, err + } + return emptyResult, nil +} + +func (stmt *stmt) Query(args []driver.Value) (driver.Rows, error) { + return stmt.queryContext(context.Background(), convertOldArgs(args)) +} + +func (stmt *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return stmt.queryContext(ctx, args) +} + +func (stmt *stmt) queryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + finish := stmt.ch.watchCancel(ctx) + query, externalTables := stmt.bind(args) + if err := stmt.ch.sendQuery(ctx, query, externalTables); err != nil { + finish() + return nil, err + } + meta, err := stmt.ch.readMeta() + if err != nil { + finish() + return nil, err + } + rows := rows{ + ch: stmt.ch, + finish: finish, + stream: make(chan *data.Block, 50), + columns: meta.ColumnNames(), + blockColumns: meta.Columns, + } + go rows.receiveData() + return &rows, nil +} + +func (stmt *stmt) Close() error { + stmt.ch.logf("[stmt] close") + return nil +} + +func (stmt *stmt) bind(args []driver.NamedValue) (string, []ExternalTable) { + var ( + buf bytes.Buffer + index int + keyword bool + inBetween bool + like = newMatcher("like") + limit = newMatcher("limit") + offset = newMatcher("offset") + between = newMatcher("between") + and = newMatcher("and") + in = newMatcher("in") + from = newMatcher("from") + join = newMatcher("join") + externalTables = make([]ExternalTable, 0) + ) + switch { + case stmt.NumInput() != 0: + reader := bytes.NewReader([]byte(stmt.query)) + for { + if char, _, err := reader.ReadRune(); err == nil { + switch char { + case '@': + if param := paramParser(reader); len(param) != 0 { + for _, v := range args { + if len(v.Name) != 0 && v.Name == param { + switch v := v.Value.(type) { + case ExternalTable: + buf.WriteString(v.Name) + externalTables = append(externalTables, v) + default: + buf.WriteString(quote(v)) + } + } + } + } + case '?': + if keyword && index < len(args) && len(args[index].Name) == 0 { + switch v := args[index].Value.(type) { + case ExternalTable: + buf.WriteString(v.Name) + externalTables = append(externalTables, v) + default: + buf.WriteString(quote(v)) + } + index++ + } else { + buf.WriteRune(char) + } + default: + switch { + case + char == '=', + char == '<', + char == '>', + char == '(', + char == ',', + char == '+', + char == '-', + char == '*', + char == '/', + char == '[': + keyword = true + default: + if limit.matchRune(char) || offset.matchRune(char) || like.matchRune(char) || + in.matchRune(char) || from.matchRune(char) || join.matchRune(char) { + keyword = true + } else if between.matchRune(char) { + keyword = true + inBetween = true + } else if inBetween && and.matchRune(char) { + keyword = true + inBetween = false + } else { + keyword = keyword && unicode.IsSpace(char) + } + } + buf.WriteRune(char) + } + } else { + break + } + } + default: + buf.WriteString(stmt.query) + } + return buf.String(), externalTables +} + +func convertOldArgs(args []driver.Value) []driver.NamedValue { + dargs := make([]driver.NamedValue, len(args)) + for i, v := range args { + dargs[i] = driver.NamedValue{ + Ordinal: i + 1, + Value: v, + } + } + return dargs +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/tls_config.go b/vendor/github.com/ClickHouse/clickhouse-go/tls_config.go new file mode 100644 index 0000000000..d85cd341fa --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/tls_config.go @@ -0,0 +1,44 @@ +package clickhouse + +import ( + "crypto/tls" + "sync" +) + +// Based on the original implementation in the project go-sql-driver/mysql: +// https://github.com/go-sql-driver/mysql/blob/master/utils.go + +var ( + tlsConfigLock sync.RWMutex + tlsConfigRegistry map[string]*tls.Config +) + +// RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. +func RegisterTLSConfig(key string, config *tls.Config) error { + tlsConfigLock.Lock() + if tlsConfigRegistry == nil { + tlsConfigRegistry = make(map[string]*tls.Config) + } + + tlsConfigRegistry[key] = config + tlsConfigLock.Unlock() + return nil +} + +// DeregisterTLSConfig removes the tls.Config associated with key. +func DeregisterTLSConfig(key string) { + tlsConfigLock.Lock() + if tlsConfigRegistry != nil { + delete(tlsConfigRegistry, key) + } + tlsConfigLock.Unlock() +} + +func getTLSConfigClone(key string) (config *tls.Config) { + tlsConfigLock.RLock() + if v, ok := tlsConfigRegistry[key]; ok { + config = v.Clone() + } + tlsConfigLock.RUnlock() + return +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/word_matcher.go b/vendor/github.com/ClickHouse/clickhouse-go/word_matcher.go new file mode 100644 index 0000000000..7cd9dcd750 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/word_matcher.go @@ -0,0 +1,31 @@ +package clickhouse + +import ( + "strings" + "unicode" +) + +// wordMatcher is a simple automata to match a single word (case insensitive) +type wordMatcher struct { + word []rune + position uint8 +} + +// newMatcher returns matcher for word needle +func newMatcher(needle string) *wordMatcher { + return &wordMatcher{word: []rune(strings.ToUpper(needle)), + position: 0} +} + +func (m *wordMatcher) matchRune(r rune) bool { + if m.word[m.position] == unicode.ToUpper(r) { + if m.position == uint8(len(m.word)-1) { + m.position = 0 + return true + } + m.position++ + } else { + m.position = 0 + } + return false +} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/write_column.go b/vendor/github.com/ClickHouse/clickhouse-go/write_column.go new file mode 100644 index 0000000000..eb4f0fe3b3 --- /dev/null +++ b/vendor/github.com/ClickHouse/clickhouse-go/write_column.go @@ -0,0 +1,66 @@ +package clickhouse + +import ( + "database/sql" + "database/sql/driver" + "time" + + "github.com/ClickHouse/clickhouse-go/lib/data" +) + +// Interface for Clickhouse driver +type Clickhouse interface { + Block() (*data.Block, error) + Prepare(query string) (driver.Stmt, error) + Begin() (driver.Tx, error) + Commit() error + Rollback() error + Close() error + WriteBlock(block *data.Block) error +} + +// Interface for Block allowing writes to individual columns +type ColumnWriter interface { + WriteDate(c int, v time.Time) error + WriteDateNullable(c int, v *time.Time) error + WriteDateTime(c int, v time.Time) error + WriteDateTimeNullable(c int, v *time.Time) error + WriteUInt8(c int, v uint8) error + WriteUInt8Nullable(c int, v *uint8) error + WriteUInt16(c int, v uint16) error + WriteUInt16Nullable(c int, v *uint16) error + WriteUInt32(c int, v uint32) error + WriteUInt32Nullable(c int, v *uint32) error + WriteUInt64(c int, v uint64) error + WriteUInt64Nullable(c int, v *uint64) error + WriteFloat32(c int, v float32) error + WriteFloat32Nullable(c int, v *float32) error + WriteFloat64(c int, v float64) error + WriteFloat64Nullable(c int, v *float64) error + WriteBytes(c int, v []byte) error + WriteArray(c int, v interface{}) error + WriteBytesNullable(c int, v *[]byte) error + WriteArrayNullable(c int, v *interface{}) error + WriteString(c int, v string) error + WriteStringNullable(c int, v *string) error + WriteFixedString(c int, v []byte) error + WriteFixedStringNullable(c int, v *[]byte) error +} + +func OpenDirect(dsn string) (Clickhouse, error) { + return open(dsn) +} + +func (ch *clickhouse) Block() (*data.Block, error) { + if ch.block == nil { + return nil, sql.ErrTxDone + } + return ch.block, nil +} + +func (ch *clickhouse) WriteBlock(block *data.Block) error { + if block == nil { + return sql.ErrTxDone + } + return ch.writeBlock(block, "") +} diff --git a/vendor/github.com/cloudflare/golz4/.gitignore b/vendor/github.com/cloudflare/golz4/.gitignore new file mode 100644 index 0000000000..00268614f0 --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/cloudflare/golz4/LICENSE b/vendor/github.com/cloudflare/golz4/LICENSE new file mode 100644 index 0000000000..1579e81a9a --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 CloudFlare, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the CloudFlare, Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/cloudflare/golz4/Makefile b/vendor/github.com/cloudflare/golz4/Makefile new file mode 100644 index 0000000000..2296d80e06 --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/Makefile @@ -0,0 +1,14 @@ +GCFLAGS := +LDFLAGS := + +.PHONY: install +install: + @go install -v . + +.PHONY: test +test: + @go test -gcflags='$(GCFLAGS)' -ldflags='$(LDFLAGS)' . + +.PHONY: bench +bench: + @go test -gcflags='$(GCFLAGS)' -ldflags='$(LDFLAGS)' -bench . diff --git a/vendor/github.com/cloudflare/golz4/README.md b/vendor/github.com/cloudflare/golz4/README.md new file mode 100644 index 0000000000..e1bdb26e82 --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/README.md @@ -0,0 +1,4 @@ +golz4 +===== + +Golang interface to LZ4 compression diff --git a/vendor/github.com/cloudflare/golz4/doc.go b/vendor/github.com/cloudflare/golz4/doc.go new file mode 100644 index 0000000000..4876be870f --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/doc.go @@ -0,0 +1,4 @@ +// Package lz4 implements compression using lz4.c and lz4hc.c +// +// Copyright (c) 2013 CloudFlare, Inc. +package lz4 diff --git a/vendor/github.com/cloudflare/golz4/lz4.go b/vendor/github.com/cloudflare/golz4/lz4.go new file mode 100644 index 0000000000..f9abcb2dbc --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/lz4.go @@ -0,0 +1,55 @@ +package lz4 + +// #cgo CFLAGS: -O3 +// #include "src/lz4.h" +// #include "src/lz4.c" +import "C" + +import ( + "errors" + "fmt" + "unsafe" +) + +// p gets a char pointer to the first byte of a []byte slice +func p(in []byte) *C.char { + if len(in) == 0 { + return (*C.char)(unsafe.Pointer(nil)) + } + return (*C.char)(unsafe.Pointer(&in[0])) +} + +// clen gets the length of a []byte slice as a char * +func clen(s []byte) C.int { + return C.int(len(s)) +} + +// Uncompress with a known output size. len(out) should be equal to +// the length of the uncompressed out. +func Uncompress(in, out []byte) (error) { + if int(C.LZ4_decompress_safe(p(in), p(out), clen(in), clen(out))) < 0 { + return errors.New("Malformed compression stream") + } + + return nil +} + +// CompressBound calculates the size of the output buffer needed by +// Compress. This is based on the following macro: +// +// #define LZ4_COMPRESSBOUND(isize) +// ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +func CompressBound(in []byte) int { + return len(in) + ((len(in) / 255) + 16) +} + +// Compress compresses in and puts the content in out. len(out) +// should have enough space for the compressed data (use CompressBound +// to calculate). Returns the number of bytes in the out slice. +func Compress(in, out []byte) (outSize int, err error) { + outSize = int(C.LZ4_compress_limitedOutput(p(in), p(out), clen(in), clen(out))) + if outSize == 0 { + err = fmt.Errorf("insufficient space for compression") + } + return +} diff --git a/vendor/github.com/cloudflare/golz4/lz4_hc.go b/vendor/github.com/cloudflare/golz4/lz4_hc.go new file mode 100644 index 0000000000..9779352c46 --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/lz4_hc.go @@ -0,0 +1,38 @@ +package lz4 + +// #cgo CFLAGS: -O3 +// #include "src/lz4hc.h" +// #include "src/lz4hc.c" +import "C" + +import ( + "fmt" +) + +// CompressHC compresses in and puts the content in out. len(out) +// should have enough space for the compressed data (use CompressBound +// to calculate). Returns the number of bytes in the out slice. Determines +// the compression level automatically. +func CompressHC(in, out []byte) (int, error) { + // 0 automatically sets the compression level. + return CompressHCLevel(in, out, 0) +} + +// CompressHCLevel compresses in at the given compression level and puts the +// content in out. len(out) should have enough space for the compressed data +// (use CompressBound to calculate). Returns the number of bytes in the out +// slice. To automatically choose the compression level, use 0. Otherwise, use +// any value in the inclusive range 1 (worst) through 16 (best). Most +// applications will prefer CompressHC. +func CompressHCLevel(in, out []byte, level int) (outSize int, err error) { + // LZ4HC does not handle empty buffers. Pass through to Compress. + if len(in) == 0 || len(out) == 0 { + return Compress(in, out) + } + + outSize = int(C.LZ4_compressHC2_limitedOutput(p(in), p(out), clen(in), clen(out), C.int(level))) + if outSize == 0 { + err = fmt.Errorf("insufficient space for compression") + } + return +} diff --git a/vendor/github.com/cloudflare/golz4/sample.txt b/vendor/github.com/cloudflare/golz4/sample.txt new file mode 100644 index 0000000000..3bb27fb76d --- /dev/null +++ b/vendor/github.com/cloudflare/golz4/sample.txt @@ -0,0 +1,143 @@ +CANTO I + + +IN the midway of this our mortal life, +I found me in a gloomy wood, astray +Gone from the path direct: and e'en to tell +It were no easy task, how savage wild +That forest, how robust and rough its growth, +Which to remember only, my dismay +Renews, in bitterness not far from death. +Yet to discourse of what there good befell, +All else will I relate discover'd there. +How first I enter'd it I scarce can say, +Such sleepy dullness in that instant weigh'd +My senses down, when the true path I left, +But when a mountain's foot I reach'd, where clos'd +The valley, that had pierc'd my heart with dread, +I look'd aloft, and saw his shoulders broad +Already vested with that planet's beam, +Who leads all wanderers safe through every way. + +Then was a little respite to the fear, +That in my heart's recesses deep had lain, +All of that night, so pitifully pass'd: +And as a man, with difficult short breath, +Forespent with toiling, 'scap'd from sea to shore, +Turns to the perilous wide waste, and stands +At gaze; e'en so my spirit, that yet fail'd +Struggling with terror, turn'd to view the straits, +That none hath pass'd and liv'd. My weary frame +After short pause recomforted, again +I journey'd on over that lonely steep, + +The hinder foot still firmer. Scarce the ascent +Began, when, lo! a panther, nimble, light, +And cover'd with a speckled skin, appear'd, +Nor, when it saw me, vanish'd, rather strove +To check my onward going; that ofttimes +With purpose to retrace my steps I turn'd. + +The hour was morning's prime, and on his way +Aloft the sun ascended with those stars, +That with him rose, when Love divine first mov'd +Those its fair works: so that with joyous hope +All things conspir'd to fill me, the gay skin +Of that swift animal, the matin dawn +And the sweet season. Soon that joy was chas'd, +And by new dread succeeded, when in view +A lion came, 'gainst me, as it appear'd, + +With his head held aloft and hunger-mad, +That e'en the air was fear-struck. A she-wolf +Was at his heels, who in her leanness seem'd +Full of all wants, and many a land hath made +Disconsolate ere now. She with such fear +O'erwhelmed me, at the sight of her appall'd, +That of the height all hope I lost. As one, +Who with his gain elated, sees the time +When all unwares is gone, he inwardly +Mourns with heart-griping anguish; such was I, +Haunted by that fell beast, never at peace, +Who coming o'er against me, by degrees +Impell'd me where the sun in silence rests. + +While to the lower space with backward step +I fell, my ken discern'd the form one of one, +Whose voice seem'd faint through long disuse of speech. +When him in that great desert I espied, +"Have mercy on me!" cried I out aloud, +"Spirit! or living man! what e'er thou be!" + +He answer'd: "Now not man, man once I was, +And born of Lombard parents, Mantuana both +By country, when the power of Julius yet +Was scarcely firm. At Rome my life was past +Beneath the mild Augustus, in the time +Of fabled deities and false. A bard +Was I, and made Anchises' upright son +The subject of my song, who came from Troy, +When the flames prey'd on Ilium's haughty towers. +But thou, say wherefore to such perils past +Return'st thou? wherefore not this pleasant mount +Ascendest, cause and source of all delight?" +"And art thou then that Virgil, that well-spring, +From which such copious floods of eloquence +Have issued?" I with front abash'd replied. +"Glory and light of all the tuneful train! +May it avail me that I long with zeal +Have sought thy volume, and with love immense +Have conn'd it o'er. My master thou and guide! +Thou he from whom alone I have deriv'd +That style, which for its beauty into fame +Exalts me. See the beast, from whom I fled. +O save me from her, thou illustrious sage!" + +"For every vein and pulse throughout my frame +She hath made tremble." He, soon as he saw +That I was weeping, answer'd, "Thou must needs +Another way pursue, if thou wouldst 'scape +From out that savage wilderness. This beast, +At whom thou criest, her way will suffer none +To pass, and no less hindrance makes than death: +So bad and so accursed in her kind, +That never sated is her ravenous will, +Still after food more craving than before. +To many an animal in wedlock vile +She fastens, and shall yet to many more, +Until that greyhound come, who shall destroy +Her with sharp pain. He will not life support +By earth nor its base metals, but by love, +Wisdom, and virtue, and his land shall be +The land 'twixt either Feltro. In his might +Shall safety to Italia's plains arise, +For whose fair realm, Camilla, virgin pure, +Nisus, Euryalus, and Turnus fell. +He with incessant chase through every town +Shall worry, until he to hell at length +Restore her, thence by envy first let loose. +I for thy profit pond'ring now devise, +That thou mayst follow me, and I thy guide +Will lead thee hence through an eternal space, +Where thou shalt hear despairing shrieks, and see +Spirits of old tormented, who invoke +A second death; and those next view, who dwell +Content in fire, for that they hope to come, +Whene'er the time may be, among the blest, +Into whose regions if thou then desire +T' ascend, a spirit worthier then I +Must lead thee, in whose charge, when I depart, +Thou shalt be left: for that Almighty King, +Who reigns above, a rebel to his law, +Adjudges me, and therefore hath decreed, +That to his city none through me should come. +He in all parts hath sway; there rules, there holds +His citadel and throne. O happy those, +Whom there he chooses!" I to him in few: +"Bard! by that God, whom thou didst not adore, +I do beseech thee (that this ill and worse +I may escape) to lead me, where thou saidst, +That I Saint Peter's gate may view, and those +Who as thou tell'st, are in such dismal plight." + +Onward he mov'd, I close his steps pursu'd. diff --git a/vendor/modules.txt b/vendor/modules.txt index b53ce264d7..8bfaec87fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -34,6 +34,15 @@ github.com/Azure/go-autorest/logger github.com/Azure/go-autorest/tracing # github.com/BurntSushi/toml v0.3.1 github.com/BurntSushi/toml +# github.com/ClickHouse/clickhouse-go v1.4.7 +github.com/ClickHouse/clickhouse-go +github.com/ClickHouse/clickhouse-go/lib/binary +github.com/ClickHouse/clickhouse-go/lib/cityhash102 +github.com/ClickHouse/clickhouse-go/lib/column +github.com/ClickHouse/clickhouse-go/lib/data +github.com/ClickHouse/clickhouse-go/lib/lz4 +github.com/ClickHouse/clickhouse-go/lib/protocol +github.com/ClickHouse/clickhouse-go/lib/types # github.com/DataDog/dd-trace-go v0.6.1 github.com/DataDog/dd-trace-go/opentracing github.com/DataDog/dd-trace-go/tracer @@ -211,6 +220,8 @@ github.com/c-bata/go-prompt # github.com/cheggaaa/pb/v3 v3.0.8 github.com/cheggaaa/pb/v3 github.com/cheggaaa/pb/v3/termutil +# github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 +github.com/cloudflare/golz4 # github.com/coredns/coredns v1.3.0 github.com/coredns/coredns/core/dnsserver github.com/coredns/coredns/coremain @@ -1178,7 +1189,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f +# yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 yunion.io/x/pkg/errors yunion.io/x/pkg/gotypes yunion.io/x/pkg/prettytable @@ -1212,7 +1223,11 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada +# yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1 yunion.io/x/sqlchemy +yunion.io/x/sqlchemy/backends +yunion.io/x/sqlchemy/backends/clickhouse +yunion.io/x/sqlchemy/backends/mysql +yunion.io/x/sqlchemy/backends/sqlite # yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce yunion.io/x/structarg diff --git a/vendor/yunion.io/x/pkg/util/regutils/regutils.go b/vendor/yunion.io/x/pkg/util/regutils/regutils.go index 8d9ea9d259..06be8fb77d 100644 --- a/vendor/yunion.io/x/pkg/util/regutils/regutils.go +++ b/vendor/yunion.io/x/pkg/util/regutils/regutils.go @@ -38,6 +38,9 @@ var DATE_COMPACT_REG *regexp.Regexp var ISO_TIME_REG *regexp.Regexp var ISO_NO_SECOND_TIME_REG *regexp.Regexp var FULLISO_TIME_REG *regexp.Regexp +var ISO_TIME_REG2 *regexp.Regexp +var ISO_NO_SECOND_TIME_REG2 *regexp.Regexp +var FULLISO_TIME_REG2 *regexp.Regexp var ZSTACK_TIME_REG *regexp.Regexp var COMPACT_TIME_REG *regexp.Regexp var MYSQL_TIME_REG *regexp.Regexp @@ -68,6 +71,9 @@ func init() { ISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$`) ISO_NO_SECOND_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$`) FULLISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3,9}(Z|[+-]\d{2}:\d{2})$`) + ISO_TIME_REG2 = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$`) + ISO_NO_SECOND_TIME_REG2 = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$`) + FULLISO_TIME_REG2 = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3,9}(Z|[+-]\d{2}:\d{2})$`) COMPACT_TIME_REG = regexp.MustCompile(`^\d{14}$`) ZSTACK_TIME_REG = regexp.MustCompile(`^\w+ \d{1,2}, \d{4} \d{1,2}:\d{1,2}:\d{1,2} (AM|PM)$`) //ZStack time format "Apr 1, 2019 3:23:17 PM" MYSQL_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$`) @@ -202,6 +208,18 @@ func MatchFullISOTime(str string) bool { return FULLISO_TIME_REG.MatchString(str) } +func MatchISOTime2(str string) bool { + return ISO_TIME_REG2.MatchString(str) +} + +func MatchISONoSecondTime2(str string) bool { + return ISO_NO_SECOND_TIME_REG2.MatchString(str) +} + +func MatchFullISOTime2(str string) bool { + return FULLISO_TIME_REG2.MatchString(str) +} + func MatchCompactTime(str string) bool { return COMPACT_TIME_REG.MatchString(str) } diff --git a/vendor/yunion.io/x/pkg/util/timeutils/timeutils.go b/vendor/yunion.io/x/pkg/util/timeutils/timeutils.go index 3dbd23eb18..c96393292c 100644 --- a/vendor/yunion.io/x/pkg/util/timeutils/timeutils.go +++ b/vendor/yunion.io/x/pkg/util/timeutils/timeutils.go @@ -58,6 +58,10 @@ const ( ShortMonthFormat = "200601" ZStackTimeFormat = "Jan 2, 2006 15:04:05 PM" + IsoTimeFormat2 = "2006-01-02 15:04:05Z07:00" + IsoNoSecondTimeFormat2 = "2006-01-02 15:04Z07:00" + FullIsoNanoTimeFormat2 = "2006-01-02 15:04:05.000000000Z07:00" + RFC2882Format = time.RFC1123 ) @@ -113,10 +117,18 @@ func ParseIsoTime(str string) (time.Time, error) { return time.Parse(IsoTimeFormat, str) } +func ParseIsoTime2(str string) (time.Time, error) { + return time.Parse(IsoTimeFormat2, str) +} + func ParseIsoNoSecondTime(str string) (time.Time, error) { return time.Parse(IsoNoSecondTimeFormat, str) } +func ParseIsoNoSecondTime2(str string) (time.Time, error) { + return time.Parse(IsoNoSecondTimeFormat2, str) +} + func toFullIsoNanoTimeFormat(str string) string { // 2019-09-17T20:50:17.66667134+08:00 // 2019-11-19T18:54:48.084-08:00 @@ -143,6 +155,10 @@ func ParseFullIsoTime(str string) (time.Time, error) { return time.Parse(FullIsoNanoTimeFormat, toFullIsoNanoTimeFormat(str)) } +func ParseFullIsoTime2(str string) (time.Time, error) { + return time.Parse(FullIsoNanoTimeFormat2, toFullIsoNanoTimeFormat(str)) +} + func ParseMysqlTime(str string) (time.Time, error) { return time.Parse(MysqlTimeFormat, str) } @@ -183,6 +199,12 @@ func ParseTimeStr(str string) (time.Time, error) { return ParseIsoTime(str) } else if regutils.MatchISONoSecondTime(str) { return ParseIsoNoSecondTime(str) + } else if regutils.MatchFullISOTime2(str) { + return ParseFullIsoTime2(str) + } else if regutils.MatchISOTime2(str) { + return ParseIsoTime2(str) + } else if regutils.MatchISONoSecondTime2(str) { + return ParseIsoNoSecondTime2(str) } else if regutils.MatchMySQLTime(str) { return ParseMysqlTime(str) } else if regutils.MatchNormalTime(str) { diff --git a/vendor/yunion.io/x/sqlchemy/README.md b/vendor/yunion.io/x/sqlchemy/README.md index 2561351a95..2f786b23e8 100644 --- a/vendor/yunion.io/x/sqlchemy/README.md +++ b/vendor/yunion.io/x/sqlchemy/README.md @@ -11,12 +11,47 @@ Features * Automatic creation and synchronization of table schema based on golang struct * Query syntax inpired by sqlalchemy -* Support MySQL/MariaDB with InnoDB engine ONLY +* Support: MySQL/MariaDB with InnoDB engine/SQLITE(Exprimental)/ClickHouse(Exprimental) * Support select, insert, update and insert or update Quick Examples ---------------- +## Database initialization + +Before using sqlchemy, database connection should be setup first. + +### Setup database with default backend(MySQL with InnoDB) + +```go +dbconn := sql.Open("mysql", "testgo:openstack@tcp(127.0.0.1:3306)/testgo?charset=utf8&parseTime") + +sqlchemy.SetDefaultDB(dbconn) +``` + +### Setup database with MySQL with InnoDB + +```go +dbconn := sql.Open("mysql", "testgo:openstack@tcp(127.0.0.1:3306)/testgo?charset=utf8&parseTime") + +sqlchemy.SetDBWithNameBackend(dbconn, sqlchemy.DBName("mysqldb"), sqlchemy.MySQLBackend) +``` +### Setup database with SQLite backend + +```go +dbconn := sql.Open("sqlite3", "file:mydb.s3db?cache=shared&mode=rwc") + +sqlchemy.SetDBWithNameBackend(dbconn, sqlchemy.DBName("sqlitedb"), sqlchemy.SQLiteBackend) +``` + +### Setup database with clickhosue + +```go +dbconn := sql.Open("clickhouse", "tcp://host1:9000?username=user&password=qwerty&database=clicks") + +sqlchemy.SetDBWithNameBackend(dbconn, sqlchemy.DBName("clickhousedb"), sqlchemy.ClickhouseBackend) +``` + ## Table Schema Table schema is defined by struct field tags @@ -46,6 +81,10 @@ Create a table from a struct schema tablespec := sqlchemy.NewTableSpecFromStruct(TestTable{}, "testtable") ``` +```go +tablespec := sqlchemy.NewTableSpecFromStructWithDBName(TestTable{}, "testtable", sqlchemy.DBName("mydb")) +``` + Check whether table schema definition is consistent with schema in database. ```go diff --git a/vendor/yunion.io/x/sqlchemy/backends.go b/vendor/yunion.io/x/sqlchemy/backends.go new file mode 100644 index 0000000000..56de9ca278 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends.go @@ -0,0 +1,87 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +import ( + "reflect" +) + +type DBBackendName string + +const ( + // MySQL is the backend name for MySQL/MariaDB + MySQLBackend = DBBackendName("MySQL") + // Clickhouse is the backend name of Clickhouse + ClickhouseBackend = DBBackendName("Clickhouse") + // SQLiteBackend is the backend name of Sqlite3 + SQLiteBackend = DBBackendName("SQLite") + // PostgreSQLBackend = DBBackendName("PostgreSQL") +) + +// IBackend is the interface for all kinds of sql backends, e.g. MySQL, ClickHouse, Sqlite, PostgreSQL, etc. +type IBackend interface { + // Name returns the name of the driver + Name() DBBackendName + // GetTableSQL returns the SQL for query tablenames + GetTableSQL() string + // GetCreateSQL returns the SQL for create a table + GetCreateSQLs(ts ITableSpec) []string + // IsSupportIndexAndContraints returns whether the backend supports index and contraints such as foreigh keys + // MySQL: true + // Sqlite: true + // Clickhouse: false + IsSupportIndexAndContraints() bool + // FetchTableColumnSpecs parse the table definition in database to extract columns' specification of a table + FetchTableColumnSpecs(ts ITableSpec) ([]IColumnSpec, error) + // FetchIndexesAndConstraints parse the table defintion in database to extract index and constraints information of a table + FetchIndexesAndConstraints(ts ITableSpec) ([]STableIndex, []STableConstraint, error) + // GetColumnSpecByFieldType parse the field of model struct to extract column specifiction of a field + GetColumnSpecByFieldType(table *STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) IColumnSpec + // + // CurrentUTCTimeStampString() string + + // Capability + + // CanUpdate returns wether the backend supports update + CanUpdate() bool + // CanInsert returns wether the backend supports Insert + CanInsert() bool + // CanInsertOrUpdate returns weather the backend supports InsertOrUpdate + CanInsertOrUpdate() bool + + // DropIndexSQLTemplate returns the template of drop index SQL + DropIndexSQLTemplate() string + // InsertSQLTemplate returns the template of insert SQL + InsertSQLTemplate() string + // InsertOrUpdateSQLTemplate returns the template of insert or update SQL + InsertOrUpdateSQLTemplate() string + + // CanSupportRowAffected returns wether the backend support RowAffected method after update + // MySQL: true + // Sqlite: false + // Clickhouse: false + CanSupportRowAffected() bool +} + +var _driver_tbl = make(map[DBBackendName]IBackend) + +// RegisterBackend registers a backend +func RegisterBackend(drv IBackend) { + _driver_tbl[drv.Name()] = drv +} + +func getBackend(name DBBackendName) IBackend { + return _driver_tbl[name] +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go new file mode 100644 index 0000000000..0cb4c7e18b --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go @@ -0,0 +1,208 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +import ( + "fmt" + "reflect" + "strings" + + _ "github.com/ClickHouse/clickhouse-go" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/utils" + + "yunion.io/x/sqlchemy" +) + +func init() { + sqlchemy.RegisterBackend(&SClickhouseBackend{}) +} + +type SClickhouseBackend struct { + sqlchemy.SBaseBackend +} + +func (click *SClickhouseBackend) Name() sqlchemy.DBBackendName { + return sqlchemy.ClickhouseBackend +} + +// CanUpdate returns wether the backend supports update +func (click *SClickhouseBackend) CanUpdate() bool { + return false +} + +// CanInsert returns wether the backend supports Insert +func (click *SClickhouseBackend) CanInsert() bool { + return true +} + +// CanInsertOrUpdate returns weather the backend supports InsertOrUpdate +func (click *SClickhouseBackend) CanInsertOrUpdate() bool { + return false +} + +func (click *SClickhouseBackend) IsSupportIndexAndContraints() bool { + return false +} + +func (click *SClickhouseBackend) CanSupportRowAffected() bool { + return false +} + +func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string { + cols := make([]string, 0) + primaries := make([]string, 0) + orderbys := make([]string, 0) + partitions := make([]string, 0) + for _, c := range ts.Columns() { + cols = append(cols, c.DefinitionString()) + if c.IsPrimary() { + primaries = append(primaries, fmt.Sprintf("`%s`", c.Name())) + } + if cc, ok := c.(IClickhouseColumnSpec); ok { + if cc.IsOrderBy() { + orderbys = append(orderbys, fmt.Sprintf("`%s`", c.Name())) + } + partition := cc.PartitionBy() + if len(partition) > 0 && !utils.IsInStringArray(partition, partitions) { + partitions = append(partitions, partition) + } + } + } + createSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE MergeTree", ts.Name(), strings.Join(cols, ",\n")) + if len(orderbys) == 0 { + orderbys = primaries + } + if len(orderbys) > 0 { + createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", ")) + } else { + createSql += fmt.Sprintf("\nORDER BY tuple()") + } + if len(partitions) > 0 { + createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", ")) + } + if len(primaries) > 0 { + createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", ")) + } + createSql += "\nSETTINGS index_granularity=8192" + return []string{ + createSql, + } +} + +func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ([]sqlchemy.IColumnSpec, error) { + sql := fmt.Sprintf("DESCRIBE `%s`", ts.Name()) + query := ts.Database().NewRawQuery(sql, "name", "type", "default_type", "default_expression", "comment", "codec_expression", "ttl_expression") + infos := make([]sSqlColumnInfo, 0) + err := query.All(&infos) + if err != nil { + return nil, errors.Wrap(err, "describe table") + } + specs := make([]sqlchemy.IColumnSpec, 0) + for _, info := range infos { + spec := info.toColumnSpec() + specs = append(specs, spec) + } + + sql = fmt.Sprintf("SHOW CREATE TABLE `%s`", ts.Name()) + query = ts.Database().NewRawQuery(sql, "statement") + row := query.Row() + var defStr string + err = row.Scan(&defStr) + if err != nil { + return nil, errors.Wrap(err, "show create table") + } + primaries, orderbys, partition := parseCreateTable(defStr) + for _, spec := range specs { + if utils.IsInStringArray(spec.Name(), primaries) { + spec.SetPrimary(true) + } + if clickSpec, ok := spec.(IClickhouseColumnSpec); ok { + if utils.IsInStringArray(clickSpec.Name(), orderbys) { + clickSpec.SetOrderBy(true) + } + if strings.Contains(partition, clickSpec.Name()) { + clickSpec.SetPartitionBy(partition) + } + } + } + + return specs, nil +} + +func (click *SClickhouseBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec { + switch fieldType { + case tristate.TriStateType: + col := NewTristateColumn(fieldname, tagmap, isPointer) + return &col + case gotypes.TimeType: + col := NewDateTimeColumn(fieldname, tagmap, isPointer) + return &col + } + switch fieldType.Kind() { + case reflect.String: + col := NewTextColumn(fieldname, "String", tagmap, isPointer) + return &col + case reflect.Int, reflect.Int32: + col := NewIntegerColumn(fieldname, "Int32", tagmap, isPointer) + return &col + case reflect.Int8: + col := NewIntegerColumn(fieldname, "Int8", tagmap, isPointer) + return &col + case reflect.Int16: + col := NewIntegerColumn(fieldname, "Int16", tagmap, isPointer) + return &col + case reflect.Int64: + col := NewIntegerColumn(fieldname, "Int64", tagmap, isPointer) + return &col + case reflect.Uint, reflect.Uint32: + col := NewIntegerColumn(fieldname, "UInt32", tagmap, isPointer) + return &col + case reflect.Uint8: + col := NewIntegerColumn(fieldname, "UInt8", tagmap, isPointer) + return &col + case reflect.Uint16: + col := NewIntegerColumn(fieldname, "UInt16", tagmap, isPointer) + return &col + case reflect.Uint64: + col := NewIntegerColumn(fieldname, "UInt64", tagmap, isPointer) + return &col + case reflect.Bool: + col := NewBooleanColumn(fieldname, tagmap, isPointer) + return &col + case reflect.Float32: + if _, ok := tagmap[sqlchemy.TAG_WIDTH]; ok { + col := NewDecimalColumn(fieldname, tagmap, isPointer) + return &col + } + col := NewFloatColumn(fieldname, "Float32", tagmap, isPointer) + return &col + case reflect.Float64: + if _, ok := tagmap[sqlchemy.TAG_WIDTH]; ok { + col := NewDecimalColumn(fieldname, tagmap, isPointer) + return &col + } + col := NewFloatColumn(fieldname, "Float64", tagmap, isPointer) + return &col + } + if fieldType.Implements(gotypes.ISerializableType) { + col := NewCompoundColumn(fieldname, tagmap, isPointer) + return &col + } + return nil +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go new file mode 100644 index 0000000000..e2279f8f23 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go @@ -0,0 +1,646 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +import ( + "bytes" + "database/sql" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/sqlchemy" +) + +type IClickhouseColumnSpec interface { + sqlchemy.IColumnSpec + + // IsOrderBy defines whether the column appears in order by clause + IsOrderBy() bool + + // PartitionBy defines expression that the column appaers in Partition by clause + PartitionBy() string + + // SetOrderBy set isOrderBy field + SetOrderBy(on bool) + + // SetPartitionBy set partitonby field + SetPartitionBy(expr string) +} + +func columnDefinitionBuffer(c sqlchemy.IColumnSpec) bytes.Buffer { + var buf bytes.Buffer + buf.WriteByte('`') + buf.WriteString(c.Name()) + buf.WriteByte('`') + buf.WriteByte(' ') + + if c.IsNullable() { + buf.WriteString("Nullable(") + } + + buf.WriteString(c.ColType()) + + if c.IsNullable() { + buf.WriteString(")") + } + + def := c.Default() + defOk := c.IsSupportDefault() + if def != "" { + if !defOk { + panic(fmt.Errorf("column %q type %q does not support having default value: %q", + c.Name(), c.ColType(), def, + )) + } + def = sqlchemy.GetStringValue(c.ConvertFromString(def)) + buf.WriteString(" DEFAULT ") + if c.IsText() { + buf.WriteByte('\'') + } + buf.WriteString(def) + if c.IsText() { + buf.WriteByte('\'') + } + } + + return buf +} + +type SClickhouseBaseColumn struct { + sqlchemy.SBaseColumn + + partionBy string + isOrderBy bool +} + +func (c *SClickhouseBaseColumn) IsOrderBy() bool { + return c.isOrderBy +} + +func (c *SClickhouseBaseColumn) SetOrderBy(on bool) { + c.isOrderBy = on +} + +func (c *SClickhouseBaseColumn) PartitionBy() string { + return c.partionBy +} + +func (c *SClickhouseBaseColumn) SetPartitionBy(expr string) { + c.partionBy = expr +} + +func NewClickhouseBaseColumn(name string, sqltype string, tagmap map[string]string, isPointer bool) SClickhouseBaseColumn { + var ok bool + var val string + partition := "" + tagmap, val, ok = utils.TagPop(tagmap, TAG_PARTITION) + if ok { + partition = val + } + orderBy := false + tagmap, val, ok = utils.TagPop(tagmap, TAG_ORDER) + if ok { + orderBy = utils.ToBool(val) + } + return SClickhouseBaseColumn{ + SBaseColumn: sqlchemy.NewBaseColumn(name, sqltype, tagmap, isPointer), + partionBy: partition, + isOrderBy: orderBy, + } +} + +// SBooleanColumn represents a boolean type column, which is a int(1) for mysql, with value of true or false +type SBooleanColumn struct { + SClickhouseBaseColumn +} + +// DefinitionString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return uint8(1) + default: + return uint8(0) + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(bool) + if bVal { + return uint8(1) + } + return uint8(0) +} + +// IsZero implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*bool) + return bVal == nil + } + bVal := val.(bool) + return bVal == false +} + +// NewBooleanColumn return an instance of SBooleanColumn +func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn { + bc := SBooleanColumn{SClickhouseBaseColumn: NewClickhouseBaseColumn(name, "UInt8", tagmap, isPointer)} + if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == uint8(1) { + msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap) + panic(msg) + } + return bc +} + +// STristateColumn represents a tristate type column, with value of true, false or none +type STristateColumn struct { + SClickhouseBaseColumn +} + +// DefinitionString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return uint8(1) + case "none", "null", "unknown": + return sql.NullInt32{} + default: + return uint8(0) + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(tristate.TriState) + if bVal == tristate.True { + return uint8(1) + } else if bVal == tristate.False { + return uint8(0) + } else { + return sql.NullInt32{} + } +} + +// IsZero implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*tristate.TriState) + return bVal == nil + } + bVal := val.(tristate.TriState) + return bVal == tristate.None +} + +// NewTristateColumn return an instance of STristateColumn +func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn { + if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok { + // tristate always nullable + delete(tagmap, sqlchemy.TAG_NULLABLE) + } + bc := STristateColumn{SClickhouseBaseColumn: NewClickhouseBaseColumn(name, "UInt8", tagmap, isPointer)} + return bc +} + +// SIntegerColumn represents an integer type of column, with value of integer +type SIntegerColumn struct { + SClickhouseBaseColumn + + // Is this column is a version column for this records + isAutoVersion bool +} + +// IsNumeric implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsZero(val interface{}) bool { + if val == nil || (c.IsPointer() && reflect.ValueOf(val).IsNil()) { + return true + } + switch intVal := val.(type) { + case int8, int16, int32, int64, int, uint, uint8, uint16, uint32, uint64: + return intVal == 0 + } + return true +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *SIntegerColumn) ConvertFromString(str string) interface{} { + ctype := c.SBaseColumn.ColType() + if ctype[0] == 'U' { + // unsigned + val, _ := strconv.ParseUint(str, 10, 64) + switch ctype { + case "UInt8": + return uint8(val) + case "UInt16": + return uint16(val) + case "UInt32": + return uint32(val) + case "UInt64": + return val + default: + panic(fmt.Sprintf("unsupported type %s", ctype)) + } + return val + } else { + val, _ := strconv.ParseInt(str, 10, 64) + switch ctype { + case "Int8": + return int8(val) + case "Int16": + return int16(val) + case "Int32": + return int32(val) + case "Int64": + return val + default: + panic(fmt.Sprintf("unsupported type %s", ctype)) + } + return val + } +} + +// IsAutoVersion implements IsAutoVersion for IColumnSpec +func (c *SIntegerColumn) IsAutoVersion() bool { + return c.isAutoVersion +} + +// NewIntegerColumn return an instance of SIntegerColumn +func NewIntegerColumn(name string, sqltype string, tagmap map[string]string, isPointer bool) SIntegerColumn { + if _, ok := tagmap[sqlchemy.TAG_AUTOVERSION]; ok { + log.Warningf("auto_version field %s not supported by ClickHouse", name) + } + if _, ok := tagmap[sqlchemy.TAG_AUTOINCREMENT]; ok { + log.Warningf("auto_increment field %s not supported by ClickHouse", name) + } + c := SIntegerColumn{ + SClickhouseBaseColumn: NewClickhouseBaseColumn(name, sqltype, tagmap, isPointer), + } + return c +} + +// SFloatColumn represents a float type column, e.g. float32 or float64 +type SFloatColumn struct { + SClickhouseBaseColumn +} + +// IsNumeric implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + switch val.(type) { + case *float32: + return val.(*float32) == nil + case *float64: + return val.(*float64) == nil + } + } else { + switch val.(type) { + case float32: + return val.(float32) == 0.0 + case float64: + return val.(float64) == 0.0 + } + } + return true +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *SFloatColumn) ConvertFromString(str string) interface{} { + ctype := c.SBaseColumn.ColType() + val, _ := strconv.ParseFloat(str, 64) + switch ctype { + case "Float32": + return float32(val) + case "Float64": + return val + default: + panic(fmt.Sprintf("unsupported type %s", ctype)) + } + return val +} + +// NewFloatColumn returns an instance of SFloatColumn +func NewFloatColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) SFloatColumn { + return SFloatColumn{ + SClickhouseBaseColumn: NewClickhouseBaseColumn(name, sqlType, tagmap, isPointer), + } +} + +// SDecimalColumn represents a DECIMAL type of column, i.e. a float with fixed width of digits +type SDecimalColumn struct { + SClickhouseBaseColumn + width int + Precision int +} + +// ColType implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) ColType() string { + str := c.SClickhouseBaseColumn.ColType() + return fmt.Sprintf("%s(%d, %d)", str, c.width, c.Precision) +} + +// IsNumeric implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + switch val.(type) { + case *float32: + return val.(*float32) == nil + case *float64: + return val.(*float64) == nil + } + } else { + switch val.(type) { + case float32: + return val.(float32) == 0.0 + case float64: + return val.(float64) == 0.0 + } + } + return true +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *SDecimalColumn) ConvertFromString(str string) interface{} { + val, _ := strconv.ParseFloat(str, 64) + return val +} + +// NewDecimalColumn returns an instance of SDecimalColumn +func NewDecimalColumn(name string, tagmap map[string]string, isPointer bool) SDecimalColumn { + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_PRECISION) + if !ok { + panic(fmt.Sprintf("Field %q of float misses precision tag", name)) + } + prec, err := strconv.Atoi(v) + if err != nil { + panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) + } + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_WIDTH) + if !ok { + panic(fmt.Sprintf("Field %q of float misses width tag", name)) + } + width, err := strconv.Atoi(v) + if err != nil { + panic(fmt.Sprintf("Field width of %q shoud be integer (%q)", name, v)) + } + var sqlType string + if width <= 9 { + sqlType = "Decimal32" + } else if width <= 18 { + sqlType = "Decimal64" + } else if width <= 38 { + sqlType = "Decimal128" + } else if width <= 76 { + sqlType = "Decimal256" + } else { + panic(fmt.Sprintf("unsupported decimal width %d", width)) + } + c := SDecimalColumn{ + SClickhouseBaseColumn: NewClickhouseBaseColumn(name, sqlType, tagmap, isPointer), + width: width, + Precision: prec, + } + return c +} + +// STextColumn represents a text type of column +type STextColumn struct { + SClickhouseBaseColumn +} + +// IsText implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsText() bool { + return true +} + +// IsSearchable implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsSearchable() bool { + return true +} + +// IsAscii implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsAscii() bool { + return false +} + +// DefinitionString implementation of STextColumn for IColumnSpec +func (c *STextColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + return gotypes.IsNil(val) + } + return reflect.ValueOf(val).Len() == 0 +} + +func (c *STextColumn) IsString() bool { + return true +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *STextColumn) ConvertFromString(str string) interface{} { + return str +} + +// NewTextColumn return an instance of STextColumn +func NewTextColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) STextColumn { + return STextColumn{ + SClickhouseBaseColumn: NewClickhouseBaseColumn(name, sqlType, tagmap, isPointer), + } +} + +// STimeTypeColumn represents a Detetime type of column, e.g. DateTime +type STimeTypeColumn struct { + SClickhouseBaseColumn +} + +// IsText implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsText() bool { + return true +} + +// DefinitionString implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*time.Time) + return bVal == nil + } + bVal := val.(time.Time) + return bVal.IsZero() +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *STimeTypeColumn) ConvertFromString(str string) interface{} { + tm, _ := timeutils.ParseTimeStr(str) + return tm +} + +// NewTimeTypeColumn return an instance of STimeTypeColumn +func NewTimeTypeColumn(name string, typeStr string, tagmap map[string]string, isPointer bool) STimeTypeColumn { + dc := STimeTypeColumn{ + SClickhouseBaseColumn: NewClickhouseBaseColumn(name, typeStr, tagmap, isPointer), + } + return dc +} + +// SDateTimeColumn represents a DateTime type of column +type SDateTimeColumn struct { + STimeTypeColumn + + // Is this column a 'created_at' field, whichi records the time of create this record + isCreatedAt bool + + // Is this column a 'updated_at' field, whichi records the time when this record was updated + isUpdatedAt bool +} + +func (c *SDateTimeColumn) IsCreatedAt() bool { + return c.isCreatedAt +} + +func (c *SDateTimeColumn) IsUpdatedAt() bool { + return c.isUpdatedAt +} + +func (c *SDateTimeColumn) IsDateTime() bool { + return true +} + +// NewDateTimeColumn returns an instance of DateTime column +func NewDateTimeColumn(name string, tagmap map[string]string, isPointer bool) SDateTimeColumn { + createdAt := false + updatedAt := false + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_CREATE_TIMESTAMP) + if ok { + createdAt = utils.ToBool(v) + } + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_UPDATE_TIMESTAMP) + if ok { + updatedAt = utils.ToBool(v) + } + dtc := SDateTimeColumn{ + NewTimeTypeColumn(name, "DateTime", tagmap, isPointer), + createdAt, updatedAt, + } + return dtc +} + +// CompoundColumn represents a column of compound tye, e.g. a JSON, an Array, or a struct +type CompoundColumn struct { + STextColumn +} + +// DefinitionString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) IsZero(val interface{}) bool { + if val == nil { + return true + } + if c.IsPointer() && reflect.ValueOf(val).IsNil() { + return true + } + return false +} + +// ConvertFromString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromString(str string) interface{} { + json, err := jsonutils.ParseString(str) + if err != nil { + log.Errorf("ParseString fail %s", err) + json = jsonutils.JSONNull + } + return json.String() +} + +// ConvertFromValue implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromValue(val interface{}) interface{} { + return jsonutils.Marshal(val).String() +} + +// NewCompoundColumn returns an instance of CompoundColumn +func NewCompoundColumn(name string, tagmap map[string]string, isPointer bool) CompoundColumn { + dtc := CompoundColumn{NewTextColumn(name, "String", tagmap, isPointer)} + return dtc +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go new file mode 100644 index 0000000000..b70e4d3731 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go @@ -0,0 +1,168 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +import ( + "regexp" + "strings" + + "yunion.io/x/log" + + "yunion.io/x/sqlchemy" +) + +// name type default_type default_expression comment codec_expression ttl_expression + +type sSqlColumnInfo struct { + Name string `json:"name"` + Type string `json:"type"` + DefaultType string `json:"default_type"` + DefaultExpression string `json:"default_expression"` + Comment string `json:"comment"` + CodecExpression string `json:"codec_expression"` + TtlExpression string `json:"ttl_expression"` +} + +func (info *sSqlColumnInfo) isNullable() bool { + if strings.HasPrefix(info.Type, "Nullable(") { + return true + } else { + return false + } +} + +func (info *sSqlColumnInfo) getType() string { + if strings.HasPrefix(info.Type, "Nullable(") { + return info.Type[len("Nullable(") : len(info.Type)-1] + } else { + return info.Type + } +} + +func (info *sSqlColumnInfo) getDefault() string { + if len(info.DefaultExpression) > 0 { + if strings.HasPrefix(info.DefaultExpression, "CAST(") { + defaultVals := strings.Split(info.DefaultExpression[len("CAST("):len(info.DefaultExpression)-1], ",") + defaultVal := defaultVals[0] + typeStr := info.getType() + if typeStr == "String" || strings.HasPrefix(typeStr, "FixString") { + defaultVal = defaultVal[1 : len(defaultVal)-1] + } + return defaultVal + } else { + return info.DefaultExpression + } + } + return "" +} + +func (info *sSqlColumnInfo) getTagmap() map[string]string { + tagmap := make(map[string]string) + if info.isNullable() { + tagmap[sqlchemy.TAG_NULLABLE] = "true" + } else { + tagmap[sqlchemy.TAG_NULLABLE] = "false" + } + defVal := info.getDefault() + if len(defVal) > 0 { + tagmap[sqlchemy.TAG_DEFAULT] = defVal + } + return tagmap +} + +func (info *sSqlColumnInfo) toColumnSpec() sqlchemy.IColumnSpec { + sqlType := info.getType() + switch sqlType { + case "String": + c := NewTextColumn(info.Name, sqlType, info.getTagmap(), false) + return &c + case "Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64": + c := NewIntegerColumn(info.Name, sqlType, info.getTagmap(), false) + return &c + case "Float32", "Float64": + c := NewFloatColumn(info.Name, sqlType, info.getTagmap(), false) + return &c + case "DateTime": + c := NewDateTimeColumn(info.Name, info.getTagmap(), false) + return &c + default: + if strings.HasPrefix(sqlType, "Decimal") { + c := NewDecimalColumn(info.Name, info.getTagmap(), false) + return &c + } else if strings.HasPrefix(sqlType, "FixString") { + c := NewTextColumn(info.Name, "FixString", info.getTagmap(), false) + return &c + } + log.Errorf("unsupported type %s", info.Type) + } + return nil +} + +const ( + primaryKeyPrefix = "PRIMARY KEY " + orderByPrefix = "ORDER BY " + partitionByPrefix = "PARTITION BY " + setttingsPrefix = "SETTINGS" + + paramPattern = `(\w+|\([\w,\s]+\))` + primaryKeyPattern = primaryKeyPrefix + paramPattern + orderByPattern = orderByPrefix + paramPattern +) + +var ( + primaryKeyRegexp = regexp.MustCompile(primaryKeyPattern) + orderByRegexp = regexp.MustCompile(orderByPattern) +) + +func parseKeys(keyStr string) []string { + keyStr = strings.TrimSpace(keyStr) + if keyStr[0] == '(' { + keyStr = keyStr[1 : len(keyStr)-1] + } + ret := make([]string, 0) + for _, key := range strings.Split(keyStr, ",") { + key = strings.TrimSpace(key) + ret = append(ret, key) + } + return ret +} + +func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, partition string) { + matches := primaryKeyRegexp.FindAllStringSubmatch(sqlStr, -1) + if len(matches) > 0 { + primaries = parseKeys(matches[0][1]) + } + matches = orderByRegexp.FindAllStringSubmatch(sqlStr, -1) + if len(matches) > 0 { + orderbys = parseKeys(matches[0][1]) + } + partIdx := strings.Index(sqlStr, partitionByPrefix) + if partIdx > 0 { + partIdx += len(partitionByPrefix) + nextIdx := -1 + for _, pattern := range []string{primaryKeyPrefix, orderByPrefix, setttingsPrefix} { + idx := strings.Index(sqlStr[partIdx:], pattern) + if nextIdx < 0 || nextIdx > idx { + nextIdx = idx + } + } + if nextIdx < 0 { + partition = strings.TrimSpace(sqlStr[partIdx:]) + } else { + partition = strings.TrimSpace(sqlStr[partIdx:][:nextIdx]) + } + } + return +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go new file mode 100644 index 0000000000..7ee7be5aab --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go @@ -0,0 +1,23 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +const ( + // TAG_PARTITION defines expression of PARTITION BY + TAG_PARTITION = "clickhouse_partition_by" + + // TAG_ORDER defines fields of ORDER BY + TAG_ORDER = "clickhouse_order_by" +) diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/doc.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/doc.go new file mode 100644 index 0000000000..3c9a83a35e --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse // import "yunion.io/x/sqlchemy/backends/clickhouse" diff --git a/vendor/yunion.io/x/sqlchemy/backends/doc.go b/vendor/yunion.io/x/sqlchemy/backends/doc.go new file mode 100644 index 0000000000..e1a7ef3251 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backends // import "yunion.io/x/sqlchemy/backends" diff --git a/vendor/yunion.io/x/sqlchemy/backends/loader.go b/vendor/yunion.io/x/sqlchemy/backends/loader.go new file mode 100644 index 0000000000..ccc7b385a2 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/loader.go @@ -0,0 +1,21 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backends + +import ( + _ "yunion.io/x/sqlchemy/backends/clickhouse" + _ "yunion.io/x/sqlchemy/backends/mysql" + _ "yunion.io/x/sqlchemy/backends/sqlite" +) diff --git a/vendor/yunion.io/x/sqlchemy/backends/mysql/column.go b/vendor/yunion.io/x/sqlchemy/backends/mysql/column.go new file mode 100644 index 0000000000..4e59e886c7 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/mysql/column.go @@ -0,0 +1,631 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "bytes" + "database/sql" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/sqlchemy" +) + +func columnDefinitionBuffer(c sqlchemy.IColumnSpec) bytes.Buffer { + var buf bytes.Buffer + buf.WriteByte('`') + buf.WriteString(c.Name()) + buf.WriteByte('`') + buf.WriteByte(' ') + buf.WriteString(c.ColType()) + + extra := c.ExtraDefs() + if len(extra) > 0 { + buf.WriteString(" ") + buf.WriteString(extra) + } + + if !c.IsNullable() { + buf.WriteString(" NOT NULL") + } + + def := c.Default() + defOk := c.IsSupportDefault() + if def != "" { + if !defOk { + panic(fmt.Errorf("column %q type %q does not support having default value: %q", + c.Name(), c.ColType(), def, + )) + } + def = sqlchemy.GetStringValue(c.ConvertFromString(def)) + buf.WriteString(" DEFAULT ") + if c.IsText() { + buf.WriteByte('\'') + } + buf.WriteString(def) + if c.IsText() { + buf.WriteByte('\'') + } + } + + return buf +} + +// SBooleanColumn represents a boolean type column, which is a int(1) for mysql, with value of true or false +type SBooleanColumn struct { + sqlchemy.SBaseWidthColumn +} + +// DefinitionString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return 1 + default: + return 0 + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(bool) + if bVal { + return 1 + } + return 0 +} + +// IsZero implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*bool) + return bVal == nil + } + bVal := val.(bool) + return bVal == false +} + +// NewBooleanColumn return an instance of SBooleanColumn +func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn { + bc := SBooleanColumn{SBaseWidthColumn: sqlchemy.NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)} + if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == 1 { + msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap) + panic(msg) + } + return bc +} + +// STristateColumn represents a tristate type column, with value of true, false or none +type STristateColumn struct { + sqlchemy.SBaseWidthColumn +} + +// DefinitionString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return 1 + case "none", "null", "unknown": + return sql.NullInt32{} + default: + return 0 + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(tristate.TriState) + if bVal == tristate.True { + return 1 + } else if bVal == tristate.False { + return 0 + } else { + return sql.NullInt32{} + } +} + +// IsZero implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*tristate.TriState) + return bVal == nil + } + bVal := val.(tristate.TriState) + return bVal == tristate.None +} + +// NewTristateColumn return an instance of STristateColumn +func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn { + if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok { + // tristate always nullable + delete(tagmap, sqlchemy.TAG_NULLABLE) + } + bc := STristateColumn{SBaseWidthColumn: sqlchemy.NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)} + return bc +} + +// SIntegerColumn represents an integer type of column, with value of integer +type SIntegerColumn struct { + sqlchemy.SBaseWidthColumn + + // Is this column an autoincrement colmn + isAutoIncrement bool + + // Is this column is a version column for this records + isAutoVersion bool + + // Is this column a unsigned integer? + isUnsigned bool + + // If this column is an autoincrement column, AutoIncrementOffset records the initial offset + autoIncrementOffset int64 +} + +// IsNumeric implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsNumeric() bool { + return true +} + +// ExtraDefs implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) ExtraDefs() string { + if c.isAutoIncrement { + return "AUTO_INCREMENT" + } + return "" +} + +// DefinitionString implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsZero(val interface{}) bool { + if val == nil || (c.IsPointer() && reflect.ValueOf(val).IsNil()) { + return true + } + switch intVal := val.(type) { + case int8, int16, int32, int64, int, uint, uint8, uint16, uint32, uint64: + return intVal == 0 + } + return true +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SIntegerColumn) ConvertFromString(str string) interface{} { + if c.isUnsigned { + val, _ := strconv.ParseUint(str, 10, 64) + return val + } else { + val, _ := strconv.ParseInt(str, 10, 64) + return val + } +} + +// ColType implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) ColType() string { + str := (&c.SBaseWidthColumn).ColType() + if c.isUnsigned { + str += " UNSIGNED" + } + return str +} + +func (c *SIntegerColumn) IsAutoVersion() bool { + return c.isAutoVersion +} + +func (c *SIntegerColumn) IsAutoIncrement() bool { + return c.isAutoIncrement +} + +func (c *SIntegerColumn) AutoIncrementOffset() int64 { + return c.autoIncrementOffset +} + +func (c *SIntegerColumn) SetAutoIncrement(on bool) { + c.isAutoIncrement = on +} + +func (c *SIntegerColumn) SetAutoIncrementOffset(offset int64) { + c.autoIncrementOffset = offset +} + +// NewIntegerColumn return an instance of SIntegerColumn +func NewIntegerColumn(name string, sqltype string, unsigned bool, tagmap map[string]string, isPointer bool) SIntegerColumn { + autoinc := false + autoincBase := int64(0) + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_AUTOINCREMENT) + if ok { + base, err := strconv.ParseInt(v, 10, 64) + if err == nil && base > 0 { + autoinc = true + autoincBase = base + } else { + autoinc = utils.ToBool(v) + } + } + autover := false + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_AUTOVERSION) + if ok { + autover = utils.ToBool(v) + } + c := SIntegerColumn{ + SBaseWidthColumn: sqlchemy.NewBaseWidthColumn(name, sqltype, tagmap, isPointer), + isAutoIncrement: autoinc, + autoIncrementOffset: autoincBase, + isAutoVersion: autover, + isUnsigned: unsigned, + } + if autoinc { + c.SetPrimary(true) // autoincrement column must be primary key + c.SetNullable(false) + c.isAutoVersion = false + } else if autover { + c.SetPrimary(false) + c.SetNullable(false) + if len(c.Default()) == 0 { + c.SetDefault("0") + } + } + return c +} + +// SFloatColumn represents a float type column, e.g. float32 or float64 +type SFloatColumn struct { + sqlchemy.SBaseColumn +} + +// IsNumeric implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + switch val.(type) { + case *float32: + return val.(*float32) == nil + case *float64: + return val.(*float64) == nil + } + } else { + switch val.(type) { + case float32: + return val.(float32) == 0.0 + case float64: + return val.(float64) == 0.0 + } + } + return true +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SFloatColumn) ConvertFromString(str string) interface{} { + val, _ := strconv.ParseFloat(str, 64) + return val +} + +// NewFloatColumn returns an instance of SFloatColumn +func NewFloatColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) SFloatColumn { + return SFloatColumn{SBaseColumn: sqlchemy.NewBaseColumn(name, sqlType, tagmap, isPointer)} +} + +// SDecimalColumn represents a DECIMAL type of column, i.e. a float with fixed width of digits +type SDecimalColumn struct { + sqlchemy.SBaseWidthColumn + Precision int +} + +// ColType implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) ColType() string { + str := c.SBaseWidthColumn.ColType() + return fmt.Sprintf("%s, %d)", str[:len(str)-1], c.Precision) +} + +// IsNumeric implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SDecimalColumn for IColumnSpec +func (c *SDecimalColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + switch val.(type) { + case *float32: + return val.(*float32) == nil + case *float64: + return val.(*float64) == nil + } + } else { + switch val.(type) { + case float32: + return val.(float32) == 0.0 + case float64: + return val.(float64) == 0.0 + } + } + return true +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SDecimalColumn) ConvertFromString(str string) interface{} { + val, _ := strconv.ParseFloat(str, 64) + return val +} + +// NewDecimalColumn returns an instance of SDecimalColumn +func NewDecimalColumn(name string, tagmap map[string]string, isPointer bool) SDecimalColumn { + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_PRECISION) + if !ok { + panic(fmt.Sprintf("Field %q of float misses precision tag", name)) + } + prec, err := strconv.Atoi(v) + if err != nil { + panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) + } + return SDecimalColumn{ + SBaseWidthColumn: sqlchemy.NewBaseWidthColumn(name, "DECIMAL", tagmap, isPointer), + Precision: prec, + } +} + +// STextColumn represents a text type of column +type STextColumn struct { + sqlchemy.SBaseWidthColumn + Charset string +} + +// IsSupportDefault implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsSupportDefault() bool { + // https://stackoverflow.com/questions/3466872/why-cant-a-text-column-have-a-default-value-in-mysql + // MySQL does not support default for TEXT/BLOB + if c.SBaseColumn.ColType() == "VARCHAR" { + return true + } + return false +} + +// ColType implementation of STextColumn for IColumnSpec +func (c *STextColumn) ColType() string { + var charset string + var collate string + switch c.Charset { + case "ascii": + charset = "ascii" + collate = "ascii_general_ci" + case "utf8": + charset = "utf8mb4" + collate = "utf8mb4_unicode_ci" + } + return fmt.Sprintf("%s CHARACTER SET '%s' COLLATE '%s'", c.SBaseWidthColumn.ColType(), charset, collate) +} + +// IsText implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsText() bool { + return true +} + +// IsSearchable implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsSearchable() bool { + return true +} + +// IsAscii implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsAscii() bool { + if c.Charset == "ascii" { + return true + } + return false +} + +// DefinitionString implementation of STextColumn for IColumnSpec +func (c *STextColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + return gotypes.IsNil(val) + } + return reflect.ValueOf(val).Len() == 0 +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *STextColumn) ConvertFromString(str string) interface{} { + return str +} + +func (c *STextColumn) IsString() bool { + return true +} + +// NewTextColumn return an instance of STextColumn +func NewTextColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) STextColumn { + tagmap, charset, _ := utils.TagPop(tagmap, sqlchemy.TAG_CHARSET) + if len(charset) == 0 { + charset = "utf8" + } else if charset != "utf8" && charset != "ascii" { + panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name)) + } + return STextColumn{ + SBaseWidthColumn: sqlchemy.NewBaseWidthColumn(name, sqlType, tagmap, isPointer), + Charset: charset, + } +} + +// STimeTypeColumn represents a Detetime type of column, e.g. DateTime +type STimeTypeColumn struct { + sqlchemy.SBaseColumn +} + +// IsText implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsText() bool { + return true +} + +// DefinitionString implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*time.Time) + return bVal == nil + } + bVal := val.(time.Time) + return bVal.IsZero() +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *STimeTypeColumn) ConvertFromString(str string) interface{} { + tm, _ := timeutils.ParseTimeStr(str) + return tm +} + +// NewTimeTypeColumn return an instance of STimeTypeColumn +func NewTimeTypeColumn(name string, typeStr string, tagmap map[string]string, isPointer bool) STimeTypeColumn { + dc := STimeTypeColumn{ + sqlchemy.NewBaseColumn(name, typeStr, tagmap, isPointer), + } + return dc +} + +// SDateTimeColumn represents a DateTime type of column +type SDateTimeColumn struct { + STimeTypeColumn + + // Is this column a 'created_at' field, whichi records the time of create this record + isCreatedAt bool + + // Is this column a 'updated_at' field, whichi records the time when this record was updated + isUpdatedAt bool +} + +func (c *SDateTimeColumn) IsCreatedAt() bool { + return c.isCreatedAt +} + +func (c *SDateTimeColumn) IsUpdatedAt() bool { + return c.isUpdatedAt +} + +func (c *SDateTimeColumn) IsDateTime() bool { + return true +} + +// NewDateTimeColumn returns an instance of DateTime column +func NewDateTimeColumn(name string, tagmap map[string]string, isPointer bool) SDateTimeColumn { + createdAt := false + updatedAt := false + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_CREATE_TIMESTAMP) + if ok { + createdAt = utils.ToBool(v) + } + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_UPDATE_TIMESTAMP) + if ok { + updatedAt = utils.ToBool(v) + } + dtc := SDateTimeColumn{ + NewTimeTypeColumn(name, "DATETIME", tagmap, isPointer), + createdAt, updatedAt, + } + return dtc +} + +// CompoundColumn represents a column of compound tye, e.g. a JSON, an Array, or a struct +type CompoundColumn struct { + STextColumn +} + +// DefinitionString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) IsZero(val interface{}) bool { + if val == nil { + return true + } + if c.IsPointer() && reflect.ValueOf(val).IsNil() { + return true + } + return false +} + +// ConvertFromString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromString(str string) interface{} { + json, err := jsonutils.ParseString(str) + if err != nil { + log.Errorf("ParseString fail %s", err) + json = jsonutils.JSONNull + } + return json.String() +} + +// ConvertFromValue implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromValue(val interface{}) interface{} { + return jsonutils.Marshal(val).String() +} + +// NewCompoundColumn returns an instance of CompoundColumn +func NewCompoundColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) CompoundColumn { + dtc := CompoundColumn{NewTextColumn(name, sqlType, tagmap, isPointer)} + return dtc +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/mysql/columninfo.go b/vendor/yunion.io/x/sqlchemy/backends/mysql/columninfo.go new file mode 100644 index 0000000000..49e199e38e --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/mysql/columninfo.go @@ -0,0 +1,148 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "fmt" + "math/bits" + "regexp" + "strconv" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + + "yunion.io/x/sqlchemy" +) + +type sSqlColumnInfo struct { + Field string + Type string + Collation string + Null string + Key string + Default string + Extra string + Privileges string + Comment string +} + +func decodeSqlTypeString(typeStr string) []string { + typeReg := regexp.MustCompile(`(\w+)\((\d+)(,\s*(\d+))?\)`) + matches := typeReg.FindStringSubmatch(typeStr) + if len(matches) >= 3 { + return matches[1:] + } + parts := strings.Split(typeStr, " ") + return []string{parts[0]} +} + +func (info *sSqlColumnInfo) toColumnSpec() sqlchemy.IColumnSpec { + tagmap := make(map[string]string) + + matches := decodeSqlTypeString(info.Type) + typeStr := strings.ToUpper(matches[0]) + width := 0 + if len(matches) > 1 { + width, _ = strconv.Atoi(matches[1]) + } + if width > 0 { + tagmap[sqlchemy.TAG_WIDTH] = fmt.Sprintf("%d", width) + } + if info.Null == "YES" { + tagmap[sqlchemy.TAG_NULLABLE] = "true" + } else { + tagmap[sqlchemy.TAG_NULLABLE] = "false" + } + if info.Key == "PRI" { + tagmap[sqlchemy.TAG_PRIMARY] = "true" + } else { + tagmap[sqlchemy.TAG_PRIMARY] = "false" + } + charset := "" + if info.Collation == "ascii_general_ci" { + charset = "ascii" + } else if info.Collation == "utf8_general_ci" || info.Collation == "utf8mb4_unicode_ci" { + charset = "utf8" + } else { + charset = "ascii" + } + if len(charset) > 0 { + tagmap[sqlchemy.TAG_CHARSET] = charset + } + if info.Default != "NULL" { + tagmap[sqlchemy.TAG_DEFAULT] = info.Default + } + if strings.HasSuffix(typeStr, "CHAR") { + c := NewTextColumn(info.Field, typeStr, tagmap, false) + return &c + } else if strings.HasSuffix(typeStr, "TEXT") { + tagmap[sqlchemy.TAG_TEXT_LENGTH] = typeStr[:len(typeStr)-4] + c := NewTextColumn(info.Field, typeStr, tagmap, false) + return &c + } else if strings.HasSuffix(typeStr, "INT") { + if info.Extra == "auto_increment" { + tagmap[sqlchemy.TAG_AUTOINCREMENT] = "true" + } + unsigned := false + if strings.HasSuffix(info.Type, " unsigned") { + unsigned = true + } + if _, ok := tagmap[sqlchemy.TAG_WIDTH]; !ok { + if unsigned { + tagmap[sqlchemy.TAG_WIDTH] = uintWidthString(typeStr) + } else { + tagmap[sqlchemy.TAG_WIDTH] = intWidthString(typeStr) + } + } + c := NewIntegerColumn(info.Field, typeStr, unsigned, tagmap, false) + return &c + } else if typeStr == "FLOAT" || typeStr == "DOUBLE" { + c := NewFloatColumn(info.Field, typeStr, tagmap, false) + return &c + } else if typeStr == "DECIMAL" { + if len(matches) > 3 { + precision, _ := strconv.Atoi(matches[3]) + if precision > 0 { + tagmap[sqlchemy.TAG_PRECISION] = fmt.Sprintf("%d", precision) + } + } + c := NewDecimalColumn(info.Field, tagmap, false) + return &c + } else if typeStr == "DATETIME" { + c := NewDateTimeColumn(info.Field, tagmap, false) + return &c + } else if typeStr == "DATE" || typeStr == "TIMESTAMP" { + c := NewTimeTypeColumn(info.Field, typeStr, tagmap, false) + return &c + } else if strings.HasPrefix(typeStr, "ENUM(") { + // enum type, force convert to text + // discourage use of enum, use text instead + enums := utils.FindWords([]byte(typeStr[5:len(typeStr)-1]), 0) + + width := 0 + for i := range enums { + if width < len(enums[i]) { + width = len(enums[i]) + } + } + tagmap[sqlchemy.TAG_WIDTH] = fmt.Sprintf("%d", 1< 0 { + autoInc = fmt.Sprintf(" AUTO_INCREMENT=%d", intC.autoIncrementOffset) + } + } + if c.IsIndex() { + indexes = append(indexes, fmt.Sprintf("KEY `ix_%s_%s` (`%s`)", ts.Name(), c.Name(), c.Name())) + } + } + if len(primaries) > 0 { + cols = append(cols, fmt.Sprintf("PRIMARY KEY (%s)", strings.Join(primaries, ", "))) + } + if len(indexes) > 0 { + cols = append(cols, indexes...) + } + return []string{ + fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci%s", ts.Name(), strings.Join(cols, ",\n"), autoInc), + } +} + +func (msyql *SMySQLBackend) IsSupportIndexAndContraints() bool { + return true +} + +func (mysql *SMySQLBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ([]sqlchemy.IColumnSpec, error) { + sql := fmt.Sprintf("SHOW FULL COLUMNS IN `%s`", ts.Name()) + query := ts.Database().NewRawQuery(sql, "field", "type", "collation", "null", "key", "default", "extra", "privileges", "comment") + infos := make([]sSqlColumnInfo, 0) + err := query.All(&infos) + if err != nil { + return nil, err + } + specs := make([]sqlchemy.IColumnSpec, 0) + for _, info := range infos { + specs = append(specs, info.toColumnSpec()) + } + return specs, nil +} + +func (mysql *SMySQLBackend) FetchIndexesAndConstraints(ts sqlchemy.ITableSpec) ([]sqlchemy.STableIndex, []sqlchemy.STableConstraint, error) { + sql := fmt.Sprintf("SHOW CREATE TABLE `%s`", ts.Name()) + query := ts.Database().NewRawQuery(sql, "table", "create table") + row := query.Row() + var name, defStr string + err := row.Scan(&name, &defStr) + if err != nil { + if isMysqlError(err, mysqlErrorTableNotExist) { + err = sqlchemy.ErrTableNotExists + } + return nil, nil, err + } + indexes := parseIndexes(defStr) + constraints := parseConstraints(defStr) + return indexes, constraints, nil +} + +func getTextSqlType(tagmap map[string]string) string { + var width int + var sqltype string + widthStr, _ := tagmap[sqlchemy.TAG_WIDTH] + if len(widthStr) > 0 && regutils.MatchInteger(widthStr) { + width, _ = strconv.Atoi(widthStr) + } + txtLen, _ := tagmap[sqlchemy.TAG_TEXT_LENGTH] + if width == 0 { + switch strings.ToLower(txtLen) { + case "medium": + sqltype = "MEDIUMTEXT" + case "long": + sqltype = "LONGTEXT" + default: + sqltype = "TEXT" + } + } else { + sqltype = "VARCHAR" + } + return sqltype +} + +func (mysql *SMySQLBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec { + switch fieldType { + case tristate.TriStateType: + tagmap[sqlchemy.TAG_WIDTH] = "1" + col := NewTristateColumn(fieldname, tagmap, isPointer) + return &col + case gotypes.TimeType: + col := NewDateTimeColumn(fieldname, tagmap, isPointer) + return &col + } + switch fieldType.Kind() { + case reflect.String: + col := NewTextColumn(fieldname, getTextSqlType(tagmap), tagmap, isPointer) + return &col + case reflect.Int, reflect.Int32: + tagmap[sqlchemy.TAG_WIDTH] = intWidthString("INT") + col := NewIntegerColumn(fieldname, "INT", false, tagmap, isPointer) + return &col + case reflect.Int8: + tagmap[sqlchemy.TAG_WIDTH] = intWidthString("TINYINT") + col := NewIntegerColumn(fieldname, "TINYINT", false, tagmap, isPointer) + return &col + case reflect.Int16: + tagmap[sqlchemy.TAG_WIDTH] = intWidthString("SMALLINT") + col := NewIntegerColumn(fieldname, "SMALLINT", false, tagmap, isPointer) + return &col + case reflect.Int64: + tagmap[sqlchemy.TAG_WIDTH] = intWidthString("BIGINT") + col := NewIntegerColumn(fieldname, "BIGINT", false, tagmap, isPointer) + return &col + case reflect.Uint, reflect.Uint32: + tagmap[sqlchemy.TAG_WIDTH] = uintWidthString("INT") + col := NewIntegerColumn(fieldname, "INT", true, tagmap, isPointer) + return &col + case reflect.Uint8: + tagmap[sqlchemy.TAG_WIDTH] = uintWidthString("TINYINT") + col := NewIntegerColumn(fieldname, "TINYINT", true, tagmap, isPointer) + return &col + case reflect.Uint16: + tagmap[sqlchemy.TAG_WIDTH] = uintWidthString("SMALLINT") + col := NewIntegerColumn(fieldname, "SMALLINT", true, tagmap, isPointer) + return &col + case reflect.Uint64: + tagmap[sqlchemy.TAG_WIDTH] = uintWidthString("BIGINT") + col := NewIntegerColumn(fieldname, "BIGINT", true, tagmap, isPointer) + return &col + case reflect.Bool: + tagmap[sqlchemy.TAG_WIDTH] = "1" + col := NewBooleanColumn(fieldname, tagmap, isPointer) + return &col + case reflect.Float32, reflect.Float64: + if _, ok := tagmap[sqlchemy.TAG_WIDTH]; ok { + col := NewDecimalColumn(fieldname, tagmap, isPointer) + return &col + } + colType := "FLOAT" + if fieldType == gotypes.Float64Type { + colType = "DOUBLE" + } + col := NewFloatColumn(fieldname, colType, tagmap, isPointer) + return &col + } + if fieldType.Implements(gotypes.ISerializableType) { + col := NewCompoundColumn(fieldname, getTextSqlType(tagmap), tagmap, isPointer) + return &col + } + return nil +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/mysql/parse.go b/vendor/yunion.io/x/sqlchemy/backends/mysql/parse.go new file mode 100644 index 0000000000..7be358cbdb --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/mysql/parse.go @@ -0,0 +1,62 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "regexp" + + "yunion.io/x/sqlchemy" +) + +const ( + indexPattern = `(?PUNIQUE\s+)?KEY ` + "`" + `(?P\w+)` + "`" + ` \((?P` + "`" + `\w+` + "`" + `(\(\d+\))?(,\s*` + "`" + `\w+` + "`" + `(\(\d+\))?)*)\)` + constraintPattern = `CONSTRAINT ` + "`" + `(?P\w+)` + "`" + ` FOREIGN KEY \((?P` + "`" + `\w+` + "`" + `(,\s*` + "`" + `\w+` + "`" + `)*)\) REFERENCES ` + "`" + `(?P\w+)` + "`" + ` \((?P` + "`" + `\w+` + "`" + `(,\s*` + "`" + `\w+` + "`" + `)*)\)` +) + +var ( + indexRegexp = regexp.MustCompile(indexPattern) + constraintRegexp = regexp.MustCompile(constraintPattern) +) + +func fetchColumns(match string) []string { + return sqlchemy.FetchColumns(match) +} + +func parseConstraints(defStr string) []sqlchemy.STableConstraint { + matches := constraintRegexp.FindAllStringSubmatch(defStr, -1) + tcs := make([]sqlchemy.STableConstraint, len(matches)) + for i := range matches { + tcs[i] = sqlchemy.NewTableConstraint( + matches[i][1], + fetchColumns(matches[i][2]), + matches[i][4], + fetchColumns(matches[i][5]), + ) + } + return tcs +} + +func parseIndexes(defStr string) []sqlchemy.STableIndex { + matches := indexRegexp.FindAllStringSubmatch(defStr, -1) + tcs := make([]sqlchemy.STableIndex, len(matches)) + for i := range matches { + tcs[i] = sqlchemy.NewTableIndex( + matches[i][2], + fetchColumns(matches[i][3]), + len(matches[i][1]) > 0, + ) + } + return tcs +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/column.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/column.go new file mode 100644 index 0000000000..d62a2aaa71 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/column.go @@ -0,0 +1,506 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite + +import ( + "bytes" + "database/sql" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "yunion.io/x/log" + + "yunion.io/x/jsonutils" + + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/sqlchemy" +) + +func columnDefinitionBuffer(c sqlchemy.IColumnSpec) bytes.Buffer { + var buf bytes.Buffer + buf.WriteByte('`') + buf.WriteString(c.Name()) + buf.WriteByte('`') + buf.WriteByte(' ') + + if c.IsAutoIncrement() { + // https://www.sqlite.org/autoinc.html + buf.WriteString("INTEGER PRIMARY KEY") + } else { + buf.WriteString(c.ColType()) + } + + if !c.IsNullable() { + buf.WriteString(" NOT NULL") + } + + def := c.Default() + defOk := c.IsSupportDefault() + if def != "" { + if !defOk { + panic(fmt.Errorf("column %q type %q does not support having default value: %q", + c.Name(), c.ColType(), def, + )) + } + def = sqlchemy.GetStringValue(c.ConvertFromString(def)) + buf.WriteString(" DEFAULT ") + if c.IsText() { + buf.WriteByte('\'') + } + buf.WriteString(def) + if c.IsText() { + buf.WriteByte('\'') + } + } + + if c.IsText() { + buf.WriteString(" COLLATE NOCASE") + } + + return buf +} + +// SBooleanColumn represents a boolean type column, which is a int(1) for mysql, with value of true or false +type SBooleanColumn struct { + sqlchemy.SBaseColumn +} + +// DefinitionString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return 1 + default: + return 0 + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *SBooleanColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(bool) + if bVal { + return 1 + } + return 0 +} + +// IsZero implementation of SBooleanColumn for IColumnSpec +func (c *SBooleanColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*bool) + return bVal == nil + } + bVal := val.(bool) + return bVal == false +} + +// NewBooleanColumn return an instance of SBooleanColumn +func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn { + bc := SBooleanColumn{SBaseColumn: sqlchemy.NewBaseColumn(name, "INTEGER", tagmap, isPointer)} + if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == "1" { + msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap) + panic(msg) + } + return bc +} + +// STristateColumn represents a tristate type column, with value of true, false or none +type STristateColumn struct { + sqlchemy.SBaseColumn +} + +// DefinitionString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// ConvertFromString implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromString(str string) interface{} { + switch strings.ToLower(str) { + case "true", "yes", "on", "ok", "1": + return 1 + case "none", "null", "unknown": + return sql.NullInt32{} + default: + return 0 + } +} + +// ConvertFromValue implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) ConvertFromValue(val interface{}) interface{} { + bVal := val.(tristate.TriState) + if bVal == tristate.True { + return 1 + } else if bVal == tristate.None { + return sql.NullInt32{} + } else { + return 0 + } +} + +// IsZero implementation of STristateColumn for IColumnSpec +func (c *STristateColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*tristate.TriState) + return bVal == nil + } + bVal := val.(tristate.TriState) + return bVal == tristate.None +} + +// NewTristateColumn return an instance of STristateColumn +func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn { + if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok { + // tristate always nullable + delete(tagmap, sqlchemy.TAG_NULLABLE) + } + bc := STristateColumn{SBaseColumn: sqlchemy.NewBaseColumn(name, "INTEGER", tagmap, isPointer)} + return bc +} + +// SIntegerColumn represents an integer type of column, with value of integer +type SIntegerColumn struct { + sqlchemy.SBaseColumn + + // Is this column an autoincrement colmn + isAutoIncrement bool + + // Is this column is a version column for this records + isAutoVersion bool + + // If this column is an autoincrement column, AutoIncrementOffset records the initial offset + autoIncrementOffset int64 +} + +// IsNumeric implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SIntegerColumn for IColumnSpec +func (c *SIntegerColumn) IsZero(val interface{}) bool { + if val == nil || (c.IsPointer() && reflect.ValueOf(val).IsNil()) { + return true + } + switch intVal := val.(type) { + case int8, int16, int32, int64, int, uint, uint8, uint16, uint32, uint64: + return intVal == 0 + } + return true +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SIntegerColumn) ConvertFromString(str string) interface{} { + val, _ := strconv.ParseInt(str, 10, 64) + return val +} + +func (c *SIntegerColumn) IsAutoVersion() bool { + return c.isAutoVersion +} + +func (c *SIntegerColumn) IsAutoIncrement() bool { + return c.isAutoIncrement +} + +func (c *SIntegerColumn) SetAutoIncrement(on bool) { + c.isAutoIncrement = on +} + +func (c *SIntegerColumn) SetAutoIncrementOffset(offset int64) { + c.autoIncrementOffset = offset +} + +// NewIntegerColumn return an instance of SIntegerColumn +func NewIntegerColumn(name string, tagmap map[string]string, isPointer bool) SIntegerColumn { + autoinc := false + autoincBase := int64(0) + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_AUTOINCREMENT) + if ok { + base, err := strconv.ParseInt(v, 10, 64) + if err == nil && base > 0 { + autoinc = true + autoincBase = base + } else { + autoinc = utils.ToBool(v) + } + } + autover := false + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_AUTOVERSION) + if ok { + autover = utils.ToBool(v) + } + c := SIntegerColumn{ + SBaseColumn: sqlchemy.NewBaseColumn(name, "INTEGER", tagmap, isPointer), + isAutoIncrement: autoinc, + autoIncrementOffset: autoincBase, + isAutoVersion: autover, + } + if autoinc { + c.SetPrimary(true) // autoincrement column must be primary key + c.SetNullable(false) + c.isAutoVersion = false + } else if autover { + c.SetPrimary(false) + c.SetNullable(false) + if len(c.Default()) == 0 { + c.SetDefault("0") + } + } + return c +} + +// SFloatColumn represents a float type column, e.g. float32 or float64 +type SFloatColumn struct { + sqlchemy.SBaseColumn +} + +// IsNumeric implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsNumeric() bool { + return true +} + +// DefinitionString implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of SFloatColumn for IColumnSpec +func (c *SFloatColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + switch val.(type) { + case *float32: + return val.(*float32) == nil + case *float64: + return val.(*float64) == nil + } + } else { + switch val.(type) { + case float32: + return val.(float32) == 0.0 + case float64: + return val.(float64) == 0.0 + } + } + return true +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *SFloatColumn) ConvertFromString(str string) interface{} { + val, _ := strconv.ParseFloat(str, 64) + return val +} + +// NewFloatColumn returns an instance of SFloatColumn +func NewFloatColumn(name string, tagmap map[string]string, isPointer bool) SFloatColumn { + return SFloatColumn{SBaseColumn: sqlchemy.NewBaseColumn(name, "REAL", tagmap, isPointer)} +} + +// STextColumn represents a text type of column +type STextColumn struct { + sqlchemy.SBaseColumn +} + +// IsText implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsText() bool { + return true +} + +// IsSearchable implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsSearchable() bool { + return true +} + +// IsAscii implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsAscii() bool { + return false +} + +// DefinitionString implementation of STextColumn for IColumnSpec +func (c *STextColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STextColumn for IColumnSpec +func (c *STextColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + return gotypes.IsNil(val) + } + return reflect.ValueOf(val).Len() == 0 +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *STextColumn) ConvertFromString(str string) interface{} { + return str +} + +func (c *STextColumn) IsString() bool { + return true +} + +// NewTextColumn return an instance of STextColumn +func NewTextColumn(name string, tagmap map[string]string, isPointer bool) STextColumn { + return STextColumn{ + SBaseColumn: sqlchemy.NewBaseColumn(name, "TEXT", tagmap, isPointer), + } +} + +// STimeTypeColumn represents a Detetime type of column, e.g. DateTime +type STimeTypeColumn struct { + sqlchemy.SBaseColumn +} + +// IsText implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsText() bool { + return true +} + +// DefinitionString implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of STimeTypeColumn for IColumnSpec +func (c *STimeTypeColumn) IsZero(val interface{}) bool { + if c.IsPointer() { + bVal := val.(*time.Time) + return bVal == nil + } + bVal := val.(time.Time) + return bVal.IsZero() +} + +// ConvertFromString implementation of SBooleanColumn for IColumnSpec +func (c *STimeTypeColumn) ConvertFromString(str string) interface{} { + tm, _ := timeutils.ParseTimeStr(str) + return tm +} + +// NewTimeTypeColumn return an instance of STimeTypeColumn +func NewTimeTypeColumn(name string, tagmap map[string]string, isPointer bool) STimeTypeColumn { + dc := STimeTypeColumn{ + SBaseColumn: sqlchemy.NewBaseColumn(name, "TEXT", tagmap, isPointer), + } + return dc +} + +// SDateTimeColumn represents a DateTime type of column +type SDateTimeColumn struct { + STimeTypeColumn + + // Is this column a 'created_at' field, whichi records the time of create this record + isCreatedAt bool + + // Is this column a 'updated_at' field, whichi records the time when this record was updated + isUpdatedAt bool +} + +func (c *SDateTimeColumn) IsCreatedAt() bool { + return c.isCreatedAt +} + +func (c *SDateTimeColumn) IsUpdatedAt() bool { + return c.isUpdatedAt +} + +func (c *SDateTimeColumn) IsDateTime() bool { + return true +} + +// NewDateTimeColumn returns an instance of DateTime column +func NewDateTimeColumn(name string, tagmap map[string]string, isPointer bool) SDateTimeColumn { + createdAt := false + updatedAt := false + tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_CREATE_TIMESTAMP) + if ok { + createdAt = utils.ToBool(v) + } + tagmap, v, ok = utils.TagPop(tagmap, sqlchemy.TAG_UPDATE_TIMESTAMP) + if ok { + updatedAt = utils.ToBool(v) + } + dtc := SDateTimeColumn{ + STimeTypeColumn: NewTimeTypeColumn(name, tagmap, isPointer), + isCreatedAt: createdAt, + isUpdatedAt: updatedAt, + } + return dtc +} + +// CompoundColumn represents a column of compound tye, e.g. a JSON, an Array, or a struct +type CompoundColumn struct { + STextColumn +} + +// DefinitionString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) DefinitionString() string { + buf := columnDefinitionBuffer(c) + return buf.String() +} + +// IsZero implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) IsZero(val interface{}) bool { + if val == nil { + return true + } + if c.IsPointer() && reflect.ValueOf(val).IsNil() { + return true + } + return false +} + +// ConvertFromString implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromString(str string) interface{} { + json, err := jsonutils.ParseString(str) + if err != nil { + log.Errorf("ParseString fail %s", err) + json = jsonutils.JSONNull + } + return json.String() +} + +// ConvertFromValue implementation of CompoundColumn for IColumnSpec +func (c *CompoundColumn) ConvertFromValue(val interface{}) interface{} { + return jsonutils.Marshal(val).String() +} + +// NewCompoundColumn returns an instance of CompoundColumn +func NewCompoundColumn(name string, tagmap map[string]string, isPointer bool) CompoundColumn { + dtc := CompoundColumn{NewTextColumn(name, tagmap, isPointer)} + return dtc +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/columninfo.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/columninfo.go new file mode 100644 index 0000000000..5bce7cb28e --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/columninfo.go @@ -0,0 +1,71 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite + +import ( + "yunion.io/x/log" + + "yunion.io/x/sqlchemy" +) + +type sSqlColumnInfo struct { + Cid int `json:"cid"` + Name string `json:"name"` + Type string `json:"type"` + Notnull bool `json:"notnull"` + DfltValue string `json:"dflt_value"` + Pk bool `json:"pk"` +} + +func (info *sSqlColumnInfo) getTagmap() map[string]string { + tagmap := make(map[string]string) + if info.Notnull { + tagmap[sqlchemy.TAG_NULLABLE] = "false" + } else { + tagmap[sqlchemy.TAG_NULLABLE] = "true" + } + if len(info.DfltValue) > 0 { + tagmap[sqlchemy.TAG_DEFAULT] = info.DfltValue + if info.Type == "TEXT" { + tagmap[sqlchemy.TAG_DEFAULT] = info.DfltValue[1 : len(info.DfltValue)-1] + } + } + if info.Pk { + tagmap[sqlchemy.TAG_PRIMARY] = "true" + } + return tagmap +} + +func (info *sSqlColumnInfo) toColumnSpec() sqlchemy.IColumnSpec { + switch info.Type { + case "TEXT", "BLOB": + c := NewTextColumn(info.Name, info.getTagmap(), false) + return &c + case "INTEGER": + c := NewIntegerColumn(info.Name, info.getTagmap(), false) + return &c + case "INTEGER AUTO_INCREMENT": + tagmap := info.getTagmap() + tagmap[sqlchemy.TAG_AUTOINCREMENT] = "true" + c := NewIntegerColumn(info.Name, tagmap, false) + return &c + case "REAL": + c := NewFloatColumn(info.Name, info.getTagmap(), false) + return &c + default: + log.Errorf("unsupported type %s", info.Type) + } + return nil +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/doc.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/doc.go new file mode 100644 index 0000000000..0f1b29d23f --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite // import "yunion.io/x/sqlchemy/backends/sqlite" diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/parse.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/parse.go new file mode 100644 index 0000000000..748cd3664e --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/parse.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite + +import ( + "regexp" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/sqlchemy" +) + +const ( + indexPattern = `\((?P` + "`" + `\w+` + "`" + `(,\s*` + "`" + `\w+` + "`" + `)*)\)` +) + +var ( + indexRegexp = regexp.MustCompile(indexPattern) +) + +type sSqliteTableInfo struct { + Type string + Name string + Sql string +} + +func (ti *sSqliteTableInfo) parseTableIndex() (sqlchemy.STableIndex, error) { + matches := indexRegexp.FindAllStringSubmatch(ti.Sql, -1) + if len(matches) > 0 { + return sqlchemy.NewTableIndex(ti.Name, sqlchemy.FetchColumns(matches[0][1]), false), nil + } + return sqlchemy.STableIndex{}, errors.ErrNotFound +} diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/sqlite.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sqlite.go new file mode 100644 index 0000000000..483f3ffb7a --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sqlite.go @@ -0,0 +1,175 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite + +import ( + "fmt" + "reflect" + "strings" + + _ "github.com/mattn/go-sqlite3" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/tristate" + + "yunion.io/x/sqlchemy" +) + +func init() { + sqlchemy.RegisterBackend(&SSqliteBackend{}) +} + +type SSqliteBackend struct { + sqlchemy.SBaseBackend +} + +func (sqlite *SSqliteBackend) Name() sqlchemy.DBBackendName { + return sqlchemy.SQLiteBackend +} + +// CanUpdate returns wether the backend supports update +func (sqlite *SSqliteBackend) CanUpdate() bool { + return true +} + +// CanInsert returns wether the backend supports Insert +func (sqlite *SSqliteBackend) CanInsert() bool { + return true +} + +// CanInsertOrUpdate returns weather the backend supports InsertOrUpdate +func (sqlite *SSqliteBackend) CanInsertOrUpdate() bool { + return true +} + +func (sqlite *SSqliteBackend) DropIndexSQLTemplate() string { + return "DROP INDEX IF EXISTS `{{ .Table }}`.`{{ .Index }}`" +} + +func (sqlite *SSqliteBackend) InsertOrUpdateSQLTemplate() string { + return "INSERT INTO `{{ .Table }}` ({{ .Columns }}) VALUES ({{ .Values }}) ON CONFLICT({{ .PrimaryKeys }}) DO UPDATE SET {{ .SetValues }}" +} + +func (sqlite *SSqliteBackend) GetTableSQL() string { + return "SELECT name FROM sqlite_master WHERE type='table'" +} + +func (sqlite *SSqliteBackend) IsSupportIndexAndContraints() bool { + return true +} + +func (sqlite *SSqliteBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string { + cols := make([]string, 0) + primaries := make([]string, 0) + indexes := make([]string, 0) + for _, c := range ts.Columns() { + cols = append(cols, c.DefinitionString()) + if c.IsPrimary() && !c.IsAutoIncrement() { + primaries = append(primaries, fmt.Sprintf("`%s`", c.Name())) + } + if c.IsIndex() { + indexes = append(indexes, fmt.Sprintf("CREATE INDEX `ix_%s_%s` ON `%s`(`%s`)", ts.Name(), c.Name(), ts.Name(), c.Name())) + } + } + if len(primaries) > 0 { + cols = append(cols, fmt.Sprintf("PRIMARY KEY (%s)", strings.Join(primaries, ", "))) + } + ret := []string{ + "PRAGMA encoding=\"UTF-8\"", + fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n)", ts.Name(), strings.Join(cols, ",\n")), + } + if len(indexes) > 0 { + ret = append(ret, indexes...) + } + return ret +} + +func (sqlite *SSqliteBackend) FetchIndexesAndConstraints(ts sqlchemy.ITableSpec) ([]sqlchemy.STableIndex, []sqlchemy.STableConstraint, error) { + sql := fmt.Sprintf("SELECT `name`, `sql` FROM `sqlite_master` WHERE `tbl_name`='%s' AND `type`='index' AND `sql`!=''", ts.Name()) + query := ts.Database().NewRawQuery(sql, "name", "sql") + results := make([]sSqliteTableInfo, 0) + err := query.All(&results) + if err != nil { + return nil, nil, errors.Wrapf(err, "Raw Query Scan %s", sql) + } + indexes := make([]sqlchemy.STableIndex, 0) + for i := range results { + ti, err := results[i].parseTableIndex() + if err != nil { + return nil, nil, errors.Wrapf(err, "parseTableIndex fail %s", results[i].Sql) + } + indexes = append(indexes, ti) + } + return indexes, nil, nil +} + +func (sqlite *SSqliteBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ([]sqlchemy.IColumnSpec, error) { + sql := fmt.Sprintf("PRAGMA table_info(`%s`);", ts.Name()) + query := ts.Database().NewRawQuery(sql, "cid", "name", "type", "notnull", "dflt_value", "pk") + infos := make([]sSqlColumnInfo, 0) + err := query.All(&infos) + if err != nil { + return nil, err + } + specs := make([]sqlchemy.IColumnSpec, 0) + // find out integer primary key + var primaryCol sqlchemy.IColumnSpec + var primaryCount int + for _, info := range infos { + spec := info.toColumnSpec() + if spec.IsPrimary() { + primaryCol = spec + primaryCount++ + } + specs = append(specs, spec) + } + if primaryCount == 1 { + if intc, ok := primaryCol.(*SIntegerColumn); ok { + intc.isAutoIncrement = true + } + } + return specs, nil +} + +func (sqlite *SSqliteBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec { + switch fieldType { + case tristate.TriStateType: + col := NewTristateColumn(fieldname, tagmap, isPointer) + return &col + case gotypes.TimeType: + col := NewDateTimeColumn(fieldname, tagmap, isPointer) + return &col + } + switch fieldType.Kind() { + case reflect.String: + col := NewTextColumn(fieldname, tagmap, isPointer) + return &col + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + col := NewIntegerColumn(fieldname, tagmap, isPointer) + return &col + case reflect.Bool: + col := NewBooleanColumn(fieldname, tagmap, isPointer) + return &col + case reflect.Float32, reflect.Float64: + col := NewFloatColumn(fieldname, tagmap, isPointer) + return &col + } + if fieldType.Implements(gotypes.ISerializableType) { + col := NewCompoundColumn(fieldname, tagmap, isPointer) + return &col + } + return nil +} diff --git a/vendor/yunion.io/x/sqlchemy/backends_base.go b/vendor/yunion.io/x/sqlchemy/backends_base.go new file mode 100644 index 0000000000..86ecf564bb --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends_base.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +type SBaseBackend struct { +} + +func (bb *SBaseBackend) GetTableSQL() string { + return "SHOW TABLES" +} + +func (bb *SBaseBackend) FetchIndexesAndConstraints(ts ITableSpec) ([]STableIndex, []STableConstraint, error) { + return nil, nil, nil +} + +func (bb *SBaseBackend) DropIndexSQLTemplate() string { + return "DROP INDEX `{{ .Index }}` ON `{{ .Table }}`" +} + +func (bb *SBaseBackend) CanSupportRowAffected() bool { + return true +} + +func (bb *SBaseBackend) InsertSQLTemplate() string { + return "INSERT INTO `{{ .Table }}` ({{ .Columns }}) VALUES ({{ .Values }})" +} + +func (bb *SBaseBackend) InsertOrUpdateSQLTemplate() string { + return "" +} diff --git a/vendor/yunion.io/x/sqlchemy/backends_mock.go b/vendor/yunion.io/x/sqlchemy/backends_mock.go new file mode 100644 index 0000000000..fb83d6c3d5 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/backends_mock.go @@ -0,0 +1,150 @@ +package sqlchemy + +import ( + "database/sql" + "fmt" + "reflect" + + "yunion.io/x/pkg/utils" +) + +const ( + mockBackendName = DBBackendName("mock") +) + +type sMockColumn struct { + SBaseColumn + isCreatedAt bool + isUpdatedAt bool + isAutoVersion bool +} + +func (c *sMockColumn) DefinitionString() string { + return fmt.Sprintf("%s %s", c.name, c.sqlType) +} + +func (c *sMockColumn) ConvertFromString(str string) interface{} { + return str +} + +func (c *sMockColumn) ConvertFromValue(v interface{}) interface{} { + return v +} + +func (c *sMockColumn) IsAutoVersion() bool { + return c.isAutoVersion +} + +func (c *sMockColumn) IsUpdatedAt() bool { + return c.isUpdatedAt +} + +func (c *sMockColumn) IsCreatedAt() bool { + return c.isCreatedAt +} + +func (c *sMockColumn) IsNumeric() bool { + switch c.sqlType { + case "int", "uint", "int8", "int16", "int32", "int64", "uin8", "uint16", "uint32", "uint64", "float32", "float64": + return true + } + return false +} + +func (c *sMockColumn) IsZero(val interface{}) bool { + return reflect.ValueOf(val).IsZero() +} + +func newMockColumn(name string, sqlType string, tagMap map[string]string, isPointer bool) sMockColumn { + var ( + isCreatedAt = false + isUpdatedAt = false + isAutoVersion = false + val string + ok bool + ) + tagMap, val, ok = utils.TagPop(tagMap, TAG_CREATE_TIMESTAMP) + if ok { + isCreatedAt = utils.ToBool(val) + } + tagMap, val, ok = utils.TagPop(tagMap, TAG_UPDATE_TIMESTAMP) + if ok { + isUpdatedAt = utils.ToBool(val) + } + tagMap, val, ok = utils.TagPop(tagMap, TAG_AUTOVERSION) + if ok { + isAutoVersion = utils.ToBool(val) + } + return sMockColumn{ + SBaseColumn: NewBaseColumn(name, sqlType, tagMap, isPointer), + isCreatedAt: isCreatedAt, + isUpdatedAt: isUpdatedAt, + isAutoVersion: isAutoVersion, + } +} + +type sMockBackend struct { + SBaseBackend +} + +func SetupMockDatabaseBackend() { + RegisterBackend(&sMockBackend{}) + + SetDBWithNameBackend(&sql.DB{}, DefaultDB, mockBackendName) +} + +func (mock *sMockBackend) Name() DBBackendName { + return mockBackendName +} + +// CanUpdate returns wether the backend supports update +func (mock *sMockBackend) CanUpdate() bool { + return true +} + +// CanInsert returns wether the backend supports Insert +func (mock *sMockBackend) CanInsert() bool { + return true +} + +// CanInsertOrUpdate returns weather the backend supports InsertOrUpdate +func (mock *sMockBackend) CanInsertOrUpdate() bool { + return true +} + +func (mock *sMockBackend) DropIndexSQLTemplate() string { + return "" +} + +func (mock *sMockBackend) InsertOrUpdateSQLTemplate() string { + return "" +} + +func (mock *sMockBackend) GetTableSQL() string { + return "" +} + +func (mock *sMockBackend) IsSupportIndexAndContraints() bool { + return true +} + +func (mock *sMockBackend) GetCreateSQLs(ts ITableSpec) []string { + return nil +} + +func (mock *sMockBackend) FetchIndexesAndConstraints(ts ITableSpec) ([]STableIndex, []STableConstraint, error) { + return nil, nil, nil +} + +func (mock *sMockBackend) FetchTableColumnSpecs(ts ITableSpec) ([]IColumnSpec, error) { + return nil, nil +} + +func (mock *sMockBackend) GetColumnSpecByFieldType(table *STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) IColumnSpec { + col := newMockColumn(fieldname, fieldType.String(), tagmap, isPointer) + return &col +} + +func (mock *sMockBackend) CurrentUTCTimeStampString() string { + return "" +} diff --git a/vendor/yunion.io/x/sqlchemy/case.go b/vendor/yunion.io/x/sqlchemy/case.go index a40cd05d9e..063d0adc8c 100644 --- a/vendor/yunion.io/x/sqlchemy/case.go +++ b/vendor/yunion.io/x/sqlchemy/case.go @@ -68,7 +68,7 @@ func (cf *SCaseFunction) expression() string { buf.WriteString(cf.branches[i].thenField.Reference()) } buf.WriteString(" ELSE ") - buf.WriteString(cf.elseField.Reference()) + buf.WriteString(cf.elseField.Expression()) buf.WriteString(" END") return buf.String() } diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index feb0dd6fa2..d8256d5668 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -15,16 +15,9 @@ package sqlchemy import ( - "bytes" "fmt" - "reflect" "strconv" - "strings" - "time" - "yunion.io/x/pkg/gotypes" - "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" ) @@ -51,6 +44,8 @@ type IColumnSpec interface { // IsPrimary returns whether this column is part of the primary keys IsPrimary() bool + SetPrimary(on bool) + // IsUnique returns whether the value of this column unique for each row IsUnique() bool @@ -76,7 +71,7 @@ type IColumnSpec interface { IsNumeric() bool // ConvertFromString returns the SQL representation of a value in string format for this column - ConvertFromString(str string) string + ConvertFromString(str string) interface{} // ConvertToString(str string) string @@ -101,6 +96,28 @@ type IColumnSpec interface { // SetDefault sets the default value in the format of string for this column SetDefault(defStr string) + + // IsAutoVersion + IsAutoVersion() bool + + // IsUpdatedAt + IsUpdatedAt() bool + + // IsCreatedAt + IsCreatedAt() bool + + // IsAutoIncrement + IsAutoIncrement() bool + + AutoIncrementOffset() int64 + + SetAutoIncrement(val bool) + + SetAutoIncrementOffset(offset int64) + + IsString() bool + + IsDateTime() bool } // SBaseColumn is the base structure represents a column @@ -166,6 +183,10 @@ func (c *SBaseColumn) IsPrimary() bool { return c.isPrimary } +func (c *SBaseColumn) SetPrimary(on bool) { + c.isPrimary = on +} + // IsUnique implementation of SBaseColumn for IColumnSpec func (c *SBaseColumn) IsUnique() bool { return c.isUnique @@ -207,67 +228,52 @@ func (c *SBaseColumn) AllowZero() bool { } // ConvertFromString implementation of SBaseColumn for IColumnSpec -func (c *SBaseColumn) ConvertFromString(str string) string { - return str -} - -/*func (c *SBaseColumn) ConvertToString(str string) string { - return str -}*/ +//func (c *SBaseColumn) ConvertFromString(str string) interface{} { +// return str +//} // ConvertFromValue implementation of SBaseColumn for IColumnSpec func (c *SBaseColumn) ConvertFromValue(val interface{}) interface{} { return val } -/*func (c *SBaseColumn) ConvertToValue(val interface{}) interface{} { - return val -}*/ - // Tags implementation of SBaseColumn for IColumnSpec func (c *SBaseColumn) Tags() map[string]string { return c.tags } -// generate SQL representation of a column -func definitionBuffer(c IColumnSpec) bytes.Buffer { - var buf bytes.Buffer - buf.WriteByte('`') - buf.WriteString(c.Name()) - buf.WriteByte('`') - buf.WriteByte(' ') - buf.WriteString(c.ColType()) +func (c *SBaseColumn) IsAutoVersion() bool { + return false +} - extra := c.ExtraDefs() - if len(extra) > 0 { - buf.WriteString(" ") - buf.WriteString(extra) - } +func (c *SBaseColumn) IsUpdatedAt() bool { + return false +} - if !c.IsNullable() { - buf.WriteString(" NOT NULL") - } +func (c *SBaseColumn) IsCreatedAt() bool { + return false +} - def := c.Default() - defOk := c.IsSupportDefault() - if def != "" { - if !defOk { - panic(fmt.Errorf("column %q type %q does not support having default value: %q", - c.Name(), c.ColType(), def, - )) - } - def = c.ConvertFromString(def) - buf.WriteString(" DEFAULT ") - if c.IsText() { - buf.WriteByte('\'') - } - buf.WriteString(def) - if c.IsText() { - buf.WriteByte('\'') - } - } +func (c *SBaseColumn) IsAutoIncrement() bool { + return false +} - return buf +func (c *SBaseColumn) AutoIncrementOffset() int64 { + return 0 +} + +func (c *SBaseColumn) SetAutoIncrement(val bool) { +} + +func (c *SBaseColumn) SetAutoIncrementOffset(offset int64) { +} + +func (c *SBaseColumn) IsString() bool { + return false +} + +func (c *SBaseColumn) IsDateTime() bool { + return false } // NewBaseColumn returns an instance of SBaseColumn @@ -354,523 +360,3 @@ func NewBaseWidthColumn(name string, sqltype string, tagmap map[string]string, i } return wc } - -// SBooleanColumn represents a boolean type column, which is a int(1) for mysql, with value of true or false -type SBooleanColumn struct { - SBaseWidthColumn -} - -// DefinitionString implementation of SBooleanColumn for IColumnSpec -func (c *SBooleanColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// ConvertFromString implementation of SBooleanColumn for IColumnSpec -func (c *SBooleanColumn) ConvertFromString(str string) string { - switch strings.ToLower(str) { - case "true", "yes", "on", "ok", "1": - return "1" - default: - return "0" - } -} - -/*func (c *SBooleanColumn) ConvertFromValue(val interface{}) interface{} { - switch bVal := val.(type) { - case bool: - if bVal { - return 1 - } else { - return 0 - } - case *bool: - if gotypes.IsNil(bVal) { - return 0 - } else if *bVal { - return 1 - } else { - return 0 - } - default: - return 0 - } -}*/ - -// IsZero implementation of SBooleanColumn for IColumnSpec -func (c *SBooleanColumn) IsZero(val interface{}) bool { - if c.isPointer { - bVal := val.(*bool) - return bVal == nil - } - bVal := val.(bool) - return bVal == false -} - -// NewBooleanColumn return an instance of SBooleanColumn -func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn { - bc := SBooleanColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)} - if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == "1" { - msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap) - panic(msg) - } - return bc -} - -// STristateColumn represents a tristate type column, with value of true, false or none -type STristateColumn struct { - SBaseWidthColumn -} - -// DefinitionString implementation of STristateColumn for IColumnSpec -func (c *STristateColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// ConvertFromString implementation of STristateColumn for IColumnSpec -func (c *STristateColumn) ConvertFromString(str string) string { - switch strings.ToLower(str) { - case "true", "yes", "on", "ok", "1": - return "1" - case "none", "null", "unknown": - return "" - default: - return "0" - } -} - -// ConvertFromValue implementation of STristateColumn for IColumnSpec -func (c *STristateColumn) ConvertFromValue(val interface{}) interface{} { - bVal := val.(tristate.TriState) - if bVal == tristate.True { - return 1 - } - return 0 -} - -// IsZero implementation of STristateColumn for IColumnSpec -func (c *STristateColumn) IsZero(val interface{}) bool { - if c.isPointer { - bVal := val.(*tristate.TriState) - return bVal == nil - } - bVal := val.(tristate.TriState) - return bVal == tristate.None -} - -// NewTristateColumn return an instance of STristateColumn -func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn { - bc := STristateColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)} - return bc -} - -// SIntegerColumn represents an integer type of column, with value of integer -type SIntegerColumn struct { - SBaseWidthColumn - - // Is this column an autoincrement colmn - IsAutoIncrement bool - - // Is this column is a version column for this records - IsAutoVersion bool - - // Is this column a unsigned integer? - IsUnsigned bool - - // If this column is an autoincrement column, AutoIncrementOffset records the initial offset - AutoIncrementOffset int64 -} - -// IsNumeric implementation of SIntegerColumn for IColumnSpec -func (c *SIntegerColumn) IsNumeric() bool { - return true -} - -// ExtraDefs implementation of SIntegerColumn for IColumnSpec -func (c *SIntegerColumn) ExtraDefs() string { - if c.IsAutoIncrement { - return "AUTO_INCREMENT" - } - return "" -} - -// DefinitionString implementation of SIntegerColumn for IColumnSpec -func (c *SIntegerColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of SIntegerColumn for IColumnSpec -func (c *SIntegerColumn) IsZero(val interface{}) bool { - if val == nil || (c.isPointer && reflect.ValueOf(val).IsNil()) { - return true - } - switch intVal := val.(type) { - case int8, int16, int32, int64, int, uint, uint8, uint16, uint32, uint64: - return intVal == 0 - } - return true -} - -// ColType implementation of SIntegerColumn for IColumnSpec -func (c *SIntegerColumn) ColType() string { - str := (&c.SBaseWidthColumn).ColType() - if c.IsUnsigned { - str += " UNSIGNED" - } - return str -} - -// NewIntegerColumn return an instance of SIntegerColumn -func NewIntegerColumn(name string, sqltype string, unsigned bool, tagmap map[string]string, isPointer bool) SIntegerColumn { - autoinc := false - autoincBase := int64(0) - tagmap, v, ok := utils.TagPop(tagmap, TAG_AUTOINCREMENT) - if ok { - base, err := strconv.ParseInt(v, 10, 64) - if err == nil && base > 0 { - autoinc = true - autoincBase = base - } else { - autoinc = utils.ToBool(v) - } - } - autover := false - tagmap, v, ok = utils.TagPop(tagmap, TAG_AUTOVERSION) - if ok { - autover = utils.ToBool(v) - } - c := SIntegerColumn{ - SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap, isPointer), - IsAutoIncrement: autoinc, - AutoIncrementOffset: autoincBase, - IsAutoVersion: autover, - IsUnsigned: unsigned, - } - if autoinc { - c.isPrimary = true // autoincrement column must be primary key - c.isNullable = false - c.IsAutoVersion = false - } else if autover { - c.isPrimary = false - c.isNullable = false - if len(c.defaultString) == 0 { - c.defaultString = "0" - } - } - return c -} - -// SFloatColumn represents a float type column, e.g. float32 or float64 -type SFloatColumn struct { - SBaseColumn -} - -// IsNumeric implementation of SFloatColumn for IColumnSpec -func (c *SFloatColumn) IsNumeric() bool { - return true -} - -// DefinitionString implementation of SFloatColumn for IColumnSpec -func (c *SFloatColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of SFloatColumn for IColumnSpec -func (c *SFloatColumn) IsZero(val interface{}) bool { - if c.isPointer { - switch val.(type) { - case *float32: - return val.(*float32) == nil - case *float64: - return val.(*float64) == nil - } - } else { - switch val.(type) { - case float32: - return val.(float32) == 0.0 - case float64: - return val.(float64) == 0.0 - } - } - return true -} - -// NewFloatColumn returns an instance of SFloatColumn -func NewFloatColumn(name string, sqlType string, tagmap map[string]string, isPointer bool) SFloatColumn { - return SFloatColumn{SBaseColumn: NewBaseColumn(name, sqlType, tagmap, isPointer)} -} - -// SDecimalColumn represents a DECIMAL type of column, i.e. a float with fixed width of digits -type SDecimalColumn struct { - SBaseWidthColumn - Precision int -} - -// ColType implementation of SDecimalColumn for IColumnSpec -func (c *SDecimalColumn) ColType() string { - return fmt.Sprintf("%s(%d, %d)", c.sqlType, c.width, c.Precision) -} - -// IsNumeric implementation of SDecimalColumn for IColumnSpec -func (c *SDecimalColumn) IsNumeric() bool { - return true -} - -// DefinitionString implementation of SDecimalColumn for IColumnSpec -func (c *SDecimalColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of SDecimalColumn for IColumnSpec -func (c *SDecimalColumn) IsZero(val interface{}) bool { - if c.isPointer { - switch val.(type) { - case *float32: - return val.(*float32) == nil - case *float64: - return val.(*float64) == nil - } - } else { - switch val.(type) { - case float32: - return val.(float32) == 0.0 - case float64: - return val.(float64) == 0.0 - } - } - return true -} - -// NewDecimalColumn returns an instance of SDecimalColumn -func NewDecimalColumn(name string, tagmap map[string]string, isPointer bool) SDecimalColumn { - tagmap, v, ok := utils.TagPop(tagmap, TAG_PRECISION) - if !ok { - panic(fmt.Sprintf("Field %q of float misses precision tag", name)) - } - prec, err := strconv.Atoi(v) - if err != nil { - panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) - } - return SDecimalColumn{ - SBaseWidthColumn: NewBaseWidthColumn(name, "DECIMAL", tagmap, isPointer), - Precision: prec, - } -} - -// STextColumn represents a text type of column -type STextColumn struct { - SBaseWidthColumn - Charset string -} - -// IsSupportDefault implementation of STextColumn for IColumnSpec -func (c *STextColumn) IsSupportDefault() bool { - // https://stackoverflow.com/questions/3466872/why-cant-a-text-column-have-a-default-value-in-mysql - // MySQL does not support default for TEXT/BLOB - if c.sqlType == "VARCHAR" { - return true - } - return false -} - -// ColType implementation of STextColumn for IColumnSpec -func (c *STextColumn) ColType() string { - var charset string - var collate string - switch c.Charset { - case "ascii": - charset = "ascii" - collate = "ascii_general_ci" - default: - charset = "utf8mb4" - collate = "utf8mb4_unicode_ci" - } - return fmt.Sprintf("%s CHARACTER SET '%s' COLLATE '%s'", c.SBaseWidthColumn.ColType(), charset, collate) -} - -// IsText implementation of STextColumn for IColumnSpec -func (c *STextColumn) IsText() bool { - return true -} - -// IsSearchable implementation of STextColumn for IColumnSpec -func (c *STextColumn) IsSearchable() bool { - return true -} - -// IsAscii implementation of STextColumn for IColumnSpec -func (c *STextColumn) IsAscii() bool { - if c.Charset == "ascii" { - return true - } - return false -} - -// DefinitionString implementation of STextColumn for IColumnSpec -func (c *STextColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of STextColumn for IColumnSpec -func (c *STextColumn) IsZero(val interface{}) bool { - if c.isPointer { - return gotypes.IsNil(val) - } - return reflect.ValueOf(val).Len() == 0 -} - -// NewTextColumn return an instance of STextColumn -func NewTextColumn(name string, tagmap map[string]string, isPointer bool) STextColumn { - var width int - var sqltype string - widthStr, _ := tagmap[TAG_WIDTH] - if len(widthStr) > 0 && regutils.MatchInteger(widthStr) { - width, _ = strconv.Atoi(widthStr) - } - tagmap, txtLen, _ := utils.TagPop(tagmap, TAG_TEXT_LENGTH) - if width == 0 { - switch strings.ToLower(txtLen) { - case "medium": - sqltype = "MEDIUMTEXT" - case "long": - sqltype = "LONGTEXT" - default: - sqltype = "TEXT" - } - } else { - sqltype = "VARCHAR" - } - tagmap, charset, _ := utils.TagPop(tagmap, TAG_CHARSET) - if len(charset) == 0 { - charset = "utf8" - } else if charset != "utf8" && charset != "ascii" { - panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name)) - } - return STextColumn{ - SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap, isPointer), - Charset: charset, - } -} - -/*type SStringColumn struct { - STextColumn -} - -func (c *SStringColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -func NewStringColumn(name string, sqltype string, tagmap map[string]string) SStringColumn { - sc := SStringColumn{STextColumn: NewTextColumn(name, sqltype, tagmap)} - // if sc.width > 768 { - // log.Fatalf("Field %s width %d too with(>768)", name, sc.width) - // } - return sc -}*/ - -// STimeTypeColumn represents a Detetime type of column, e.g. DateTime -type STimeTypeColumn struct { - SBaseColumn -} - -// IsText implementation of STimeTypeColumn for IColumnSpec -func (c *STimeTypeColumn) IsText() bool { - return true -} - -// DefinitionString implementation of STimeTypeColumn for IColumnSpec -func (c *STimeTypeColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of STimeTypeColumn for IColumnSpec -func (c *STimeTypeColumn) IsZero(val interface{}) bool { - if c.isPointer { - bVal := val.(*time.Time) - return bVal == nil - } - bVal := val.(time.Time) - return bVal.IsZero() -} - -// NewTimeTypeColumn return an instance of STimeTypeColumn -func NewTimeTypeColumn(name string, typeStr string, tagmap map[string]string, isPointer bool) STimeTypeColumn { - dc := STimeTypeColumn{ - NewBaseColumn(name, typeStr, tagmap, isPointer), - } - return dc -} - -// SDateTimeColumn represents a DateTime type of column -type SDateTimeColumn struct { - STimeTypeColumn - - // Is this column a 'created_at' field, whichi records the time of create this record - IsCreatedAt bool - - // Is this column a 'updated_at' field, whichi records the time when this record was updated - IsUpdatedAt bool -} - -// NewDateTimeColumn returns an instance of DateTime column -func NewDateTimeColumn(name string, tagmap map[string]string, isPointer bool) SDateTimeColumn { - createdAt := false - updatedAt := false - tagmap, v, ok := utils.TagPop(tagmap, TAG_CREATE_TIMESTAMP) - if ok { - createdAt = utils.ToBool(v) - } - tagmap, v, ok = utils.TagPop(tagmap, TAG_UPDATE_TIMESTAMP) - if ok { - updatedAt = utils.ToBool(v) - } - dtc := SDateTimeColumn{ - NewTimeTypeColumn(name, "DATETIME", tagmap, isPointer), - createdAt, updatedAt, - } - return dtc -} - -// CompoundColumn represents a column of compound tye, e.g. a JSON, an Array, or a struct -type CompoundColumn struct { - STextColumn -} - -// DefinitionString implementation of CompoundColumn for IColumnSpec -func (c *CompoundColumn) DefinitionString() string { - buf := definitionBuffer(c) - return buf.String() -} - -// IsZero implementation of CompoundColumn for IColumnSpec -func (c *CompoundColumn) IsZero(val interface{}) bool { - if val == nil { - return true - } - if c.isPointer && reflect.ValueOf(val).IsNil() { - return true - } - return false -} - -// ConvertFromValue implementation of CompoundColumn for IColumnSpec -func (c *CompoundColumn) ConvertFromValue(val interface{}) interface{} { - bVal, ok := val.(gotypes.ISerializable) - if ok && bVal != nil { - return bVal.String() - } - return "" -} - -// NewCompoundColumn returns an instance of CompoundColumn -func NewCompoundColumn(name string, tagmap map[string]string, isPointer bool) CompoundColumn { - dtc := CompoundColumn{NewTextColumn(name, tagmap, isPointer)} - return dtc -} diff --git a/vendor/yunion.io/x/sqlchemy/const.go b/vendor/yunion.io/x/sqlchemy/const.go index d4478788ab..2fcf03cb38 100644 --- a/vendor/yunion.io/x/sqlchemy/const.go +++ b/vendor/yunion.io/x/sqlchemy/const.go @@ -14,11 +14,6 @@ package sqlchemy -import ( - "strconv" - "strings" -) - const ( // SQL_OP_AND represents AND operator SQL_OP_AND = "AND" @@ -54,10 +49,13 @@ const ( // TAG_NAME is a field tag that indicates the column name of this field TAG_NAME = "name" // TAG_WIDTH is a field tag that indicates the width of the column, like VARCHAR(15) + // Supported by: mysql TAG_WIDTH = "width" // TAG_TEXT_LENGTH is a field tag that indicates the length of a text column + // Supported by: mysql TAG_TEXT_LENGTH = "length" // TAG_CHARSET is a field tag that indicates the charset of a text column + // Supported by: mysql TAG_CHARSET = "charset" // TAG_PRECISION is a field tag that indicates the precision of a float column TAG_PRECISION = "precision" @@ -82,28 +80,3 @@ const ( // TAG_ALLOW_ZERO is a field tag that indicates whether the column allow zero value TAG_ALLOW_ZERO = "allow_zero" ) - -var ( - // INT_WIDTH_DEFAULT records the default width of integer type - INT_WIDTH_DEFAULT = map[string]int{ - "TINYINT": 4, - "SMALLINT": 6, - "INT": 11, - "BIGINT": 20, - } - // UNSIGNED_INT_WIDTH_DEFAULT records the default width of unsigned integer type - UNSIGNED_INT_WIDTH_DEFAULT = map[string]int{ - "TINYINT": 3, - "SMALLINT": 5, - "INT": 10, - "BIGINT": 20, - } -) - -func intWidthString(typeStr string) string { - return strconv.FormatInt(int64(INT_WIDTH_DEFAULT[strings.ToUpper(typeStr)]), 10) -} - -func uintWidthString(typeStr string) string { - return strconv.FormatInt(int64(UNSIGNED_INT_WIDTH_DEFAULT[strings.ToUpper(typeStr)]), 10) -} diff --git a/vendor/yunion.io/x/sqlchemy/constraint.go b/vendor/yunion.io/x/sqlchemy/constraint.go index d0379a0deb..7eef68b5e4 100644 --- a/vendor/yunion.io/x/sqlchemy/constraint.go +++ b/vendor/yunion.io/x/sqlchemy/constraint.go @@ -15,67 +15,37 @@ package sqlchemy import ( - "regexp" "strings" ) -type sTableConstraint struct { +type STableConstraint struct { name string columns []string foreignTable string foreignKeys []string } -const ( - indexPattern = `(?PUNIQUE\s+)?KEY ` + "`" + `(?P\w+)` + "`" + ` \((?P` + "`" + `\w+` + "`" + `(\(\d+\))?(,\s*` + "`" + `\w+` + "`" + `(\(\d+\))?)*)\)` - constraintPattern = `CONSTRAINT ` + "`" + `(?P\w+)` + "`" + ` FOREIGN KEY \((?P` + "`" + `\w+` + "`" + `(,\s*` + "`" + `\w+` + "`" + `)*)\) REFERENCES ` + "`" + `(?P
\w+)` + "`" + ` \((?P` + "`" + `\w+` + "`" + `(,\s*` + "`" + `\w+` + "`" + `)*)\)` -) +func NewTableConstraint(name string, cols []string, foreignTable string, fcols []string) STableConstraint { + return STableConstraint{ + name: name, + columns: cols, + foreignTable: foreignTable, + foreignKeys: fcols, + } +} -var ( - indexRegexp = regexp.MustCompile(indexPattern) - constraintRegexp = regexp.MustCompile(constraintPattern) -) - -func fetchColumns(match string) []string { +func FetchColumns(match string) []string { ret := make([]string, 0) if len(match) > 0 { for _, part := range strings.Split(match, ",") { if part[len(part)-1] == ')' { part = part[:strings.LastIndexByte(part, '(')] } - part = strings.Trim(part, "`") + part = strings.Trim(part, " `") if len(part) > 0 { ret = append(ret, part) } } } - // log.Debugf("%s", ret) return ret } - -func parseConstraints(defStr string) []sTableConstraint { - matches := constraintRegexp.FindAllStringSubmatch(defStr, -1) - tcs := make([]sTableConstraint, len(matches)) - for i := range matches { - tcs[i] = sTableConstraint{ - name: matches[i][1], - foreignTable: matches[i][4], - columns: fetchColumns(matches[i][2]), - foreignKeys: fetchColumns(matches[i][5]), - } - } - return tcs -} - -func parseIndexes(defStr string) []sTableIndex { - matches := indexRegexp.FindAllStringSubmatch(defStr, -1) - tcs := make([]sTableIndex, len(matches)) - for i := range matches { - tcs[i] = sTableIndex{ - name: matches[i][2], - isUnique: len(matches[i][1]) > 0, - columns: fetchColumns(matches[i][3]), - } - } - return tcs -} diff --git a/vendor/yunion.io/x/sqlchemy/debug.go b/vendor/yunion.io/x/sqlchemy/debug.go index 0ca6168d8e..c9d159c839 100644 --- a/vendor/yunion.io/x/sqlchemy/debug.go +++ b/vendor/yunion.io/x/sqlchemy/debug.go @@ -28,17 +28,22 @@ var ( ) func sqlDebug(sqlstr string, variables []interface{}) { + sqlstr = _sqlDebug(sqlstr, variables) + log.Debugln("SQuery ", sqlstr) +} + +func _sqlDebug(sqlstr string, variables []interface{}) string { for _, v := range variables { switch v.(type) { case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: sqlstr = strings.Replace(sqlstr, "?", fmt.Sprintf(`%v`, v), 1) case string, time.Time: - sqlstr = strings.Replace(sqlstr, "?", fmt.Sprintf(`"%s"`, v), 1) + sqlstr = strings.Replace(sqlstr, "?", fmt.Sprintf(`'%s'`, v), 1) default: - sqlstr = strings.Replace(sqlstr, "?", fmt.Sprintf(`"%v"`, v), 1) + sqlstr = strings.Replace(sqlstr, "?", fmt.Sprintf(`'%v'`, v), 1) } } - log.Debugln("SQuery ", sqlstr) + return sqlstr } // DebugQuery show the full query string for debug diff --git a/vendor/yunion.io/x/sqlchemy/errors.go b/vendor/yunion.io/x/sqlchemy/errors.go index 9aebf37494..b2250a55d9 100644 --- a/vendor/yunion.io/x/sqlchemy/errors.go +++ b/vendor/yunion.io/x/sqlchemy/errors.go @@ -15,8 +15,6 @@ package sqlchemy import ( - "github.com/go-sql-driver/mysql" - "yunion.io/x/pkg/errors" ) @@ -53,15 +51,7 @@ const ( // ErrUnionFieldsNotMatch is an Error constant: fields of union queries not match ErrUnionFieldsNotMatch = errors.Error("cannot union, name of fields not match") -) -const ( - mysqlErrorTableNotExist = 0x47a + // ErrUnionDatabasesNotMatch is an Error constant: backend database of union queries not match + ErrUnionDatabasesNotMatch = errors.Error("cannot union across different databases") ) - -func isMysqlError(err error, code uint16) bool { - if myErr, ok := err.(*mysql.MySQLError); ok { - return myErr.Number == code - } - return false -} diff --git a/vendor/yunion.io/x/sqlchemy/fetch.go b/vendor/yunion.io/x/sqlchemy/fetch.go index 4088b5bb89..27ad01c5dd 100644 --- a/vendor/yunion.io/x/sqlchemy/fetch.go +++ b/vendor/yunion.io/x/sqlchemy/fetch.go @@ -18,7 +18,6 @@ import ( "reflect" "yunion.io/x/log" - "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" @@ -30,7 +29,7 @@ func (ts *STableSpec) Fetch(dt interface{}) error { q := ts.Query() dataValue := reflect.ValueOf(dt).Elem() fields := reflectutils.FetchStructFieldValueSet(dataValue) - for _, c := range ts.columns { + for _, c := range ts.Columns() { priVal, _ := fields.GetInterface(c.Name()) if c.IsPrimary() && !gotypes.IsNil(priVal) { // skip update primary key q = q.Equals(c.Name(), priVal) @@ -74,7 +73,7 @@ func (ts *STableSpec) FetchAll(dest interface{}) error { } for i := 0; i < arrayValue.Len(); i++ { - keyValueStr := getStringValue(keyValues[i]) + keyValueStr := GetStringValue(keyValues[i]) if tmpMap, ok := tmpDestMapMap[keyValueStr]; ok { err = mapString2Struct(tmpMap, arrayValue.Index(i)) if err != nil { diff --git a/vendor/yunion.io/x/sqlchemy/field_update.go b/vendor/yunion.io/x/sqlchemy/field_update.go index 2633405b2b..3ec63f2773 100644 --- a/vendor/yunion.io/x/sqlchemy/field_update.go +++ b/vendor/yunion.io/x/sqlchemy/field_update.go @@ -19,38 +19,14 @@ import ( "fmt" "reflect" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" ) -/* -func (ts *STableSpec) GetUpdateColumnValue(dataType reflect.Type, dataValue reflect.Value, cv map[string]interface{}, fields map[string]interface{}) error { - for i := 0; i < dataType.NumField(); i++ { - fieldType := dataType.Field(i) - if gotypes.IsFieldExportable(fieldType.Name) { - fieldValue := dataValue.Field(i) - newValue, ok := fields[fieldType.Name] - if ok && fieldType.Anonymous { - return errors.New("Unsupported update anonymous field") - } - if ok { - columnName := reflectutils.GetStructFieldName(&fieldType) - cv[columnName] = newValue - continue - } - if fieldType.Anonymous { - err := ts.GetUpdateColumnValue(fieldType.Type, fieldValue, cv, fields) - if err != nil { - return err - } - } - } - } - return nil -} -*/ - // UpdateFields update a record with the values provided by fields stringmap // params dt: model struct, fileds: {struct-field-name-string: update-value} func (ts *STableSpec) UpdateFields(dt interface{}, fields map[string]interface{}) error { @@ -61,17 +37,14 @@ func (ts *STableSpec) UpdateFields(dt interface{}, fields map[string]interface{} // find primary key and index key // find fields correlatively columns // joint sql and executed -func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{}, debug bool) error { +func (ts *STableSpec) updateFieldSql(dt interface{}, fields map[string]interface{}, debug bool) (*sUpdateSQLResult, error) { dataValue := reflect.Indirect(reflect.ValueOf(dt)) - // cv: {"column name": "update value"} cv := make(map[string]interface{}) - // dataType := dataValue.Type() - // ts.GetUpdateColumnValue(dataType, dataValue, cv, fields) - // if len(cv) == 0 { - // log.Infof("Nothing update") - // return nil - // } + // use field to store field order + cnames := make([]string, 0) + + now := timeutils.UtcNow() fullFields := reflectutils.FetchStructFieldValueSet(dataValue) versionFields := make([]string, 0) @@ -83,33 +56,40 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} if !ok { continue } - if col.IsPrimary() && !col.IsZero(colValue) { - primaryCols[name] = colValue + if col.IsPrimary() { + if !gotypes.IsNil(colValue) && !col.IsZero(colValue) { + primaryCols[name] = colValue + } else if col.IsText() { + primaryCols[name] = "" + } else { + return nil, ErrEmptyPrimaryKey + } continue } - intCol, ok := col.(*SIntegerColumn) - if ok && intCol.IsAutoVersion { + if col.IsAutoVersion() { versionFields = append(versionFields, name) continue } - dateCol, ok := col.(*SDateTimeColumn) - if ok && dateCol.IsUpdatedAt { + if col.IsUpdatedAt() { updatedFields = append(updatedFields, name) continue } if _, exist := fields[name]; exist { cv[name] = col.ConvertFromValue(fields[name]) + cnames = append(cnames, name) } } + if len(primaryCols) == 0 { + return nil, ErrEmptyPrimaryKey + } + vars := make([]interface{}, 0) var buf bytes.Buffer buf.WriteString(fmt.Sprintf("UPDATE `%s` SET ", ts.name)) - first := true - for k, v := range cv { - if first { - first = false - } else { + for i, k := range cnames { + v := cv[k] + if i > 0 { buf.WriteString(", ") } buf.WriteString(fmt.Sprintf("`%s` = ?", k)) @@ -119,14 +99,11 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = ?", updatedField)) + vars = append(vars, now) } buf.WriteString(" WHERE ") - first = true - if len(primaryCols) == 0 { - return ErrEmptyPrimaryKey - } - + first := true for k, v := range primaryCols { if first { first = false @@ -140,16 +117,24 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} if DEBUG_SQLCHEMY || debug { log.Infof("Update: %s", buf.String()) } - results, err := _db.Exec(buf.String(), vars...) + + return &sUpdateSQLResult{ + sql: buf.String(), + vars: vars, + primaries: primaryCols, + }, nil +} + +func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{}, debug bool) error { + results, err := ts.updateFieldSql(dt, fields, debug) if err != nil { - return err + return errors.Wrap(err, "updateFieldSql") } - aCnt, err := results.RowsAffected() + + err = ts.execUpdateSql(dt, results) if err != nil { - return err - } - if aCnt > 1 { - return errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) + return errors.Wrap(err, "execUpdateSql") } + return nil } diff --git a/vendor/yunion.io/x/sqlchemy/functions.go b/vendor/yunion.io/x/sqlchemy/functions.go index d2b8e4ebef..0cbe64b307 100644 --- a/vendor/yunion.io/x/sqlchemy/functions.go +++ b/vendor/yunion.io/x/sqlchemy/functions.go @@ -82,7 +82,7 @@ type sExprFunction struct { func (ff *sExprFunction) expression() string { fieldRefs := make([]interface{}, 0) for _, f := range ff.fields { - fieldRefs = append(fieldRefs, f.Reference()) + fieldRefs = append(fieldRefs, f.Expression()) } return fmt.Sprintf(ff.function, fieldRefs...) } @@ -237,7 +237,12 @@ func CONCAT(name string, fields ...IQueryField) IQueryField { } // SubStr represents a SQL function SUBSTR +// Deprecated func SubStr(name string, field IQueryField, pos, length int) IQueryField { + return SUBSTR(name, field, pos, length) +} + +func SUBSTR(name string, field IQueryField, pos, length int) IQueryField { var rightStr string if length <= 0 { rightStr = fmt.Sprintf("%d)", pos) @@ -268,3 +273,7 @@ func INET_ATON(field IQueryField) IQueryField { func TimestampAdd(name string, field IQueryField, offsetSeconds int) IQueryField { return NewFunctionField(name, `TIMESTAMPADD(SECOND, `+fmt.Sprintf("%d", offsetSeconds)+`, %s)`, field) } + +func CAST(field IQueryField, typeStr string, fieldname string) IQueryField { + return NewFunctionField(fieldname, `CAST(%s AS `+typeStr+`)`, field) +} diff --git a/vendor/yunion.io/x/sqlchemy/inc.go b/vendor/yunion.io/x/sqlchemy/inc.go index 399db674d8..30acb8da9b 100644 --- a/vendor/yunion.io/x/sqlchemy/inc.go +++ b/vendor/yunion.io/x/sqlchemy/inc.go @@ -16,7 +16,6 @@ package sqlchemy import ( "bytes" - "database/sql" "fmt" "reflect" @@ -24,6 +23,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/pkg/util/timeutils" ) // Increment perform an incremental update on a record, the primary key of the record is specified in diff, @@ -31,26 +31,28 @@ import ( // if target is given as a pointer to a variable, the result will be stored in the target // if target is not given, the updated result will be stored in diff func (t *STableSpec) Increment(diff interface{}, target interface{}) error { + if !t.Database().backend.CanUpdate() { + return errors.ErrNotSupported + } return t.incrementInternal(diff, "+", target) } // Decrement is similar to Increment methods, the difference is that this method will atomically decrease the numeric fields // with the value of diff func (t *STableSpec) Decrement(diff interface{}, target interface{}) error { + if !t.Database().backend.CanUpdate() { + return errors.ErrNotSupported + } return t.incrementInternal(diff, "-", target) } -func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target interface{}) error { - if target == nil { - if reflect.ValueOf(diff).Kind() != reflect.Ptr { - return errors.Wrap(ErrNeedsPointer, "Incremental input must be a Pointer") - } - } else { - if reflect.ValueOf(target).Kind() != reflect.Ptr { - return errors.Wrap(ErrNeedsPointer, "Incremental update target must be a Pointer") - } - } +type incrementSqlResult struct { + sql string + vars []interface{} + primaries map[string]interface{} +} +func (t *STableSpec) incrementInternalSql(diff interface{}, opcode string, target interface{}) (*sUpdateSQLResult, error) { dataValue := reflect.Indirect(reflect.ValueOf(diff)) fields := reflectutils.FetchStructFieldValueSet(dataValue) var targetFields reflectutils.SStructFieldValueSet @@ -59,13 +61,15 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i targetFields = reflectutils.FetchStructFieldValueSet(targetValue) } + now := timeutils.UtcNow() + primaries := make(map[string]interface{}) vars := make([]interface{}, 0) versionFields := make([]string, 0) updatedFields := make([]string, 0) incFields := make([]string, 0) - for _, c := range t.columns { + for _, c := range t.Columns() { k := c.Name() v, _ := fields.GetInterface(k) if c.IsPrimary() { @@ -77,17 +81,15 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i } else if c.IsText() { primaries[k] = "" } else { - return ErrEmptyPrimaryKey + return nil, ErrEmptyPrimaryKey } continue } - dtc, ok := c.(*SDateTimeColumn) - if ok && dtc.IsUpdatedAt { + if c.IsUpdatedAt() { updatedFields = append(updatedFields, k) continue } - nc, ok := c.(*SIntegerColumn) - if ok && nc.IsAutoVersion { + if c.IsAutoVersion() { versionFields = append(versionFields, k) continue } @@ -99,10 +101,10 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i } if len(vars) == 0 { - return ErrNoDataToUpdate + return nil, ErrNoDataToUpdate } if len(primaries) == 0 { - return ErrEmptyPrimaryKey + return nil, ErrEmptyPrimaryKey } var buf bytes.Buffer @@ -120,7 +122,8 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = ?", updatedField)) + vars = append(vars, now) } buf.WriteString(" WHERE ") @@ -139,31 +142,34 @@ func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target i log.Infof("Update: %s %s", buf.String(), vars) } - results, err := _db.Exec(buf.String(), vars...) - if err != nil { - return errors.Wrapf(err, "_db.Exec %s %#v", buf.String(), vars) - } - aCnt, err := results.RowsAffected() - if err != nil { - return errors.Wrap(err, "results.RowsAffected") - } - if aCnt != 1 { - if aCnt == 0 { - return sql.ErrNoRows + return &sUpdateSQLResult{ + sql: buf.String(), + vars: vars, + primaries: primaries, + }, nil +} + +func (t *STableSpec) incrementInternal(diff interface{}, opcode string, target interface{}) error { + if target == nil { + if reflect.ValueOf(diff).Kind() != reflect.Ptr { + return errors.Wrap(ErrNeedsPointer, "Incremental input must be a Pointer") } - return errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) - } - q := t.Query() - for k, v := range primaries { - q = q.Equals(k, v) - } - if target != nil { - err = q.First(target) } else { - err = q.First(diff) + if reflect.ValueOf(target).Kind() != reflect.Ptr { + return errors.Wrap(ErrNeedsPointer, "Incremental update target must be a Pointer") + } + } + + intResult, err := t.incrementInternalSql(diff, opcode, target) + + if target != nil { + err = t.execUpdateSql(target, intResult) + } else { + err = t.execUpdateSql(diff, intResult) } if err != nil { return errors.Wrap(err, "query after update failed") } + return nil } diff --git a/vendor/yunion.io/x/sqlchemy/index.go b/vendor/yunion.io/x/sqlchemy/index.go index ee30008fd9..cbf5245c21 100644 --- a/vendor/yunion.io/x/sqlchemy/index.go +++ b/vendor/yunion.io/x/sqlchemy/index.go @@ -16,16 +16,26 @@ package sqlchemy import ( "fmt" + "sort" "strings" ) -type sTableIndex struct { +type STableIndex struct { name string columns []string isUnique bool } -/*type TColumnNames []string +func NewTableIndex(name string, cols []string, unique bool) STableIndex { + sort.Sort(TColumnNames(cols)) + return STableIndex{ + name: name, + columns: cols, + isUnique: unique, + } +} + +type TColumnNames []string func (cols TColumnNames) Len() int { return len(cols) @@ -41,12 +51,13 @@ func (cols TColumnNames) Less(i, j int) bool { } else { return false } -}*/ +} -func (index *sTableIndex) IsIdentical(cols ...string) bool { +func (index *STableIndex) IsIdentical(cols ...string) bool { if len(index.columns) != len(cols) { return false } + sort.Sort(TColumnNames(cols)) for i := 0; i < len(index.columns); i++ { if index.columns[i] != cols[i] { return false @@ -55,7 +66,7 @@ func (index *sTableIndex) IsIdentical(cols ...string) bool { return true } -func (index *sTableIndex) QuotedColumns() []string { +func (index *STableIndex) QuotedColumns() []string { ret := make([]string, len(index.columns)) for i := 0; i < len(ret); i++ { ret[i] = fmt.Sprintf("`%s`", index.columns[i]) @@ -66,13 +77,13 @@ func (index *sTableIndex) QuotedColumns() []string { // AddIndex adds a SQL index over multiple columns for a Table // param unique: indicates a unique index cols: name of columns func (ts *STableSpec) AddIndex(unique bool, cols ...string) bool { - for i := 0; i < len(ts.indexes); i++ { - if ts.indexes[i].IsIdentical(cols...) { + for i := 0; i < len(ts._indexes); i++ { + if ts._indexes[i].IsIdentical(cols...) { return false } } name := fmt.Sprintf("ix_%s_%s", ts.name, strings.Join(cols, "_")) - idx := sTableIndex{name: name, columns: cols, isUnique: unique} - ts.indexes = append(ts.indexes, idx) + idx := STableIndex{name: name, columns: cols, isUnique: unique} + ts._indexes = append(ts._indexes, idx) return true } diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index 95eb1207ca..e91377a5e6 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -23,10 +23,14 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/pkg/util/timeutils" ) // Insert perform a insert operation, the value of the record is store in dt func (t *STableSpec) Insert(dt interface{}) error { + if !t.Database().backend.CanInsert() { + return errors.ErrNotSupported + } return t.insert(dt, false, false) } @@ -34,10 +38,24 @@ func (t *STableSpec) Insert(dt interface{}) error { // MySQL: INSERT INTO ... ON DUPLICATE KEY UPDATE ... // works only for the cases that all values of primary keys are determeted before insert func (t *STableSpec) InsertOrUpdate(dt interface{}) error { + if !t.Database().backend.CanInsertOrUpdate() { + return errors.ErrNotSupported + } return t.insert(dt, true, false) } -func (t *STableSpec) insertSqlPrep(dataFields reflectutils.SStructFieldValueSet, update bool) (string, []interface{}, error) { +type InsertSqlResult struct { + Sql string + Values []interface{} + Primaries map[string]interface{} +} + +func (t *STableSpec) InsertSqlPrep(data interface{}, update bool) (*InsertSqlResult, error) { + beforeInsert(reflect.ValueOf(data)) + + dataValue := reflect.ValueOf(data).Elem() + dataFields := reflectutils.FetchStructFieldValueSet(dataValue) + var autoIncField string createdAtFields := make([]string, 0) @@ -48,77 +66,105 @@ func (t *STableSpec) insertSqlPrep(dataFields reflectutils.SStructFieldValueSet, updates := make([]string, 0) updateValues := make([]interface{}, 0) - for _, c := range t.columns { + primaryKeys := make([]string, 0) + primaries := make(map[string]interface{}) + + now := timeutils.UtcNow() + + for _, c := range t.Columns() { isAutoInc := false - nc, ok := c.(*SIntegerColumn) - if ok && nc.IsAutoIncrement { + if c.IsAutoIncrement() { isAutoInc = true } k := c.Name() - dtc, isDate := c.(*SDateTimeColumn) - inc, isInt := c.(*SIntegerColumn) ov, find := dataFields.GetInterface(k) if !find { continue } - if isDate && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { + if c.IsPrimary() { + primaryKeys = append(primaryKeys, fmt.Sprintf("`%s`", k)) + } + + // created_at or updated_at but must not be a primary key + if c.IsCreatedAt() || c.IsUpdatedAt() { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) if c.IsZero(ov) { - format = append(format, "UTC_TIMESTAMP()") + // format = append(format, t.Database().backend.CurrentUTCTimeStampString()) + values = append(values, now) + format = append(format, "?") } else { values = append(values, ov) format = append(format, "?") } - if update && dtc.IsUpdatedAt { + if update && c.IsUpdatedAt() && !c.IsPrimary() { if c.IsZero(ov) { - updates = append(updates, fmt.Sprintf("`%s` = UTC_TIMESTAMP()", k)) + updates = append(updates, fmt.Sprintf("`%s` = ?", k)) + updateValues = append(updateValues, now) } else { updates = append(updates, fmt.Sprintf("`%s` = ?", k)) updateValues = append(updateValues, ov) } } + if c.IsPrimary() { + if c.IsZero(ov) { + primaries[k] = now + } else { + primaries[k] = ov + } + } continue } - if update && isInt && inc.IsAutoVersion { + // auto_version and must not be a primary key + if update && c.IsAutoVersion() { updates = append(updates, fmt.Sprintf("`%s` = `%s` + 1", k, k)) continue } - _, isTextCol := c.(*STextColumn) - if c.IsSupportDefault() && (len(c.Default()) > 0 || isTextCol) && !gotypes.IsNil(ov) && c.IsZero(ov) && !c.AllowZero() { // empty text value + // empty but with default + if c.IsSupportDefault() && (len(c.Default()) > 0 || c.IsString()) && !gotypes.IsNil(ov) && c.IsZero(ov) && !c.AllowZero() { // empty text value val := c.ConvertFromString(c.Default()) values = append(values, val) names = append(names, fmt.Sprintf("`%s`", k)) format = append(format, "?") - if update { + if update && !c.IsPrimary() { updates = append(updates, fmt.Sprintf("`%s` = ?", k)) updateValues = append(updateValues, val) } + + if c.IsPrimary() { + primaries[k] = val + } continue } + // not empty if !gotypes.IsNil(ov) && (!c.IsZero(ov) || (!c.IsPointer() && !c.IsText())) && !isAutoInc { v := c.ConvertFromValue(ov) values = append(values, v) names = append(names, fmt.Sprintf("`%s`", k)) format = append(format, "?") - if update { + if update && !c.IsPrimary() { updates = append(updates, fmt.Sprintf("`%s` = ?", k)) updateValues = append(updateValues, v) } + + if c.IsPrimary() { + primaries[k] = v + } continue } + // empty primary but is autoinc or text if c.IsPrimary() { if isAutoInc { if len(autoIncField) > 0 { @@ -129,24 +175,48 @@ func (t *STableSpec) insertSqlPrep(dataFields reflectutils.SStructFieldValueSet, values = append(values, "") names = append(names, fmt.Sprintf("`%s`", k)) format = append(format, "?") + primaries[k] = "" } else { - return "", nil, errors.Wrapf(ErrEmptyPrimaryKey, "cannot insert for null primary key %q", k) + return nil, errors.Wrapf(ErrEmptyPrimaryKey, "cannot insert for null primary key %q", k) } continue } } - insertSql := fmt.Sprintf("INSERT INTO `%s` (%s) VALUES(%s)", - t.name, - strings.Join(names, ", "), - strings.Join(format, ", ")) - - if update { - insertSql += " ON DUPLICATE KEY UPDATE " + strings.Join(updates, ", ") + var insertSql string + if !update { + insertSql = templateEval(t.Database().backend.InsertSQLTemplate(), struct { + Table string + Columns string + Values string + }{ + Table: t.name, + Columns: strings.Join(names, ", "), + Values: strings.Join(format, ", "), + }) + } else { + insertSql = templateEval(t.Database().backend.InsertOrUpdateSQLTemplate(), struct { + Table string + Columns string + Values string + PrimaryKeys string + SetValues string + }{ + Table: t.name, + Columns: strings.Join(names, ", "), + Values: strings.Join(format, ", "), + PrimaryKeys: strings.Join(primaryKeys, ", "), + SetValues: strings.Join(updates, ", "), + }) values = append(values, updateValues...) } - return insertSql, values, nil + + return &InsertSqlResult{ + Sql: insertSql, + Values: values, + Primaries: primaries, + }, nil } func beforeInsert(val reflect.Value) { @@ -170,37 +240,35 @@ func beforeInsert(val reflect.Value) { } func (t *STableSpec) insert(data interface{}, update bool, debug bool) error { - beforeInsert(reflect.ValueOf(data)) - - dataValue := reflect.ValueOf(data).Elem() - dataFields := reflectutils.FetchStructFieldValueSet(dataValue) - insertSql, values, err := t.insertSqlPrep(dataFields, update) + insertResult, err := t.InsertSqlPrep(data, update) if err != nil { - return err + return errors.Wrap(err, "insertSqlPrep") } if DEBUG_SQLCHEMY || debug { - log.Debugf("%s values: %v", insertSql, values) + log.Debugf("%s values: %#v", insertResult.Sql, insertResult.Values) } - results, err := _db.Exec(insertSql, values...) + results, err := t.Database().TxExec(insertResult.Sql, insertResult.Values...) if err != nil { - return err - } - affectCnt, err := results.RowsAffected() - if err != nil { - return err + return errors.Wrap(err, "TxExec") } - targetCnt := int64(1) - if update { - // for insertOrUpdate cases, if no duplication, targetCnt=1, else targetCnt=2 - targetCnt = 2 - } - if (!update && affectCnt < 1) || affectCnt > targetCnt { - return errors.Wrapf(ErrUnexpectRowCount, "Insert affected cnt %d != (1, %d)", affectCnt, targetCnt) - } + if t.Database().backend.CanSupportRowAffected() { + affectCnt, err := results.RowsAffected() + if err != nil { + return err + } + targetCnt := int64(1) + if update { + // for insertOrUpdate cases, if no duplication, targetCnt=1, else targetCnt=2 + targetCnt = 2 + } + if (!update && affectCnt < 1) || affectCnt > targetCnt { + return errors.Wrapf(ErrUnexpectRowCount, "Insert affected cnt %d != (1, %d)", affectCnt, targetCnt) + } + } /* if len(autoIncField) > 0 { lastId, err := results.LastInsertId() @@ -216,17 +284,16 @@ func (t *STableSpec) insert(data interface{}, update bool, debug bool) error { // query the value, so default value can be feedback into the object // fields = reflectutils.FetchStructFieldNameValueInterfaces(dataValue) q := t.Query() - for _, c := range t.columns { + for _, c := range t.Columns() { if c.IsPrimary() { - nc, ok := c.(*SIntegerColumn) - if ok && nc.IsAutoIncrement { + if c.IsAutoIncrement() { lastId, err := results.LastInsertId() if err != nil { return errors.Wrap(err, "fetching lastInsertId failed") } q = q.Equals(c.Name(), lastId) } else { - priVal, _ := dataFields.GetInterface(c.Name()) + priVal, _ := insertResult.Primaries[c.Name()] if !gotypes.IsNil(priVal) { q = q.Equals(c.Name(), priVal) } diff --git a/vendor/yunion.io/x/sqlchemy/parser.go b/vendor/yunion.io/x/sqlchemy/parser.go index f02284ebe2..d021274ca1 100644 --- a/vendor/yunion.io/x/sqlchemy/parser.go +++ b/vendor/yunion.io/x/sqlchemy/parser.go @@ -18,21 +18,26 @@ import ( "fmt" "reflect" - "yunion.io/x/pkg/gotypes" - "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" ) -func structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec { +func (table *STableSpec) structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec { fieldname := field.Info.MarshalName() tagmap := field.Info.Tags if _, ok := tagmap[TAG_IGNORE]; ok { return nil } + db := table.Database() + if db == nil { + panic("structField2ColumnSpec: empty database") + } + if db.backend == nil { + panic("structField2ColumnSpec: empty backend") + } fieldType := field.Value.Type() - var retCol = getFiledTypeCol(fieldType, fieldname, tagmap, false) + var retCol = db.backend.GetColumnSpecByFieldType(table, fieldType, fieldname, tagmap, false) if retCol == nil && fieldType.Kind() == reflect.Ptr { - retCol = getFiledTypeCol(fieldType.Elem(), fieldname, tagmap, true) + retCol = db.backend.GetColumnSpecByFieldType(table, fieldType.Elem(), fieldname, tagmap, true) } if retCol == nil { panic(fmt.Sprintf("unsupported colume %s data type %s", fieldname, fieldType.Name())) @@ -40,82 +45,13 @@ func structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec { return retCol } -func getFiledTypeCol(fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) IColumnSpec { - switch fieldType { - case tristate.TriStateType: - tagmap[TAG_WIDTH] = "1" - col := NewTristateColumn(fieldname, tagmap, isPointer) - return &col - case gotypes.TimeType: - col := NewDateTimeColumn(fieldname, tagmap, isPointer) - return &col - } - switch fieldType.Kind() { - case reflect.String: - col := NewTextColumn(fieldname, tagmap, isPointer) - return &col - case reflect.Int, reflect.Int32: - tagmap[TAG_WIDTH] = intWidthString("INT") - col := NewIntegerColumn(fieldname, "INT", false, tagmap, isPointer) - return &col - case reflect.Int8: - tagmap[TAG_WIDTH] = intWidthString("TINYINT") - col := NewIntegerColumn(fieldname, "TINYINT", false, tagmap, isPointer) - return &col - case reflect.Int16: - tagmap[TAG_WIDTH] = intWidthString("SMALLINT") - col := NewIntegerColumn(fieldname, "SMALLINT", false, tagmap, isPointer) - return &col - case reflect.Int64: - tagmap[TAG_WIDTH] = intWidthString("BIGINT") - col := NewIntegerColumn(fieldname, "BIGINT", false, tagmap, isPointer) - return &col - case reflect.Uint, reflect.Uint32: - tagmap[TAG_WIDTH] = uintWidthString("INT") - col := NewIntegerColumn(fieldname, "INT", true, tagmap, isPointer) - return &col - case reflect.Uint8: - tagmap[TAG_WIDTH] = uintWidthString("TINYINT") - col := NewIntegerColumn(fieldname, "TINYINT", true, tagmap, isPointer) - return &col - case reflect.Uint16: - tagmap[TAG_WIDTH] = uintWidthString("SMALLINT") - col := NewIntegerColumn(fieldname, "SMALLINT", true, tagmap, isPointer) - return &col - case reflect.Uint64: - tagmap[TAG_WIDTH] = uintWidthString("BIGINT") - col := NewIntegerColumn(fieldname, "BIGINT", true, tagmap, isPointer) - return &col - case reflect.Bool: - tagmap[TAG_WIDTH] = "1" - col := NewBooleanColumn(fieldname, tagmap, isPointer) - return &col - case reflect.Float32, reflect.Float64: - if _, ok := tagmap[TAG_WIDTH]; ok { - col := NewDecimalColumn(fieldname, tagmap, isPointer) - return &col - } - colType := "FLOAT" - if fieldType == gotypes.Float64Type { - colType = "DOUBLE" - } - col := NewFloatColumn(fieldname, colType, tagmap, isPointer) - return &col - } - if fieldType.Implements(gotypes.ISerializableType) { - col := NewCompoundColumn(fieldname, tagmap, isPointer) - return &col - } - return nil -} - -func struct2TableSpec(sv reflect.Value, table *STableSpec) { +func (table *STableSpec) struct2TableSpec(sv reflect.Value) { fields := reflectutils.FetchStructFieldValueSet(sv) autoIncCnt := 0 for i := 0; i < len(fields); i++ { - column := structField2ColumnSpec(&fields[i]) + column := table.structField2ColumnSpec(&fields[i]) if column != nil { - if intC, ok := column.(*SIntegerColumn); ok && intC.IsAutoIncrement { + if column.IsAutoIncrement() { autoIncCnt++ if autoIncCnt > 1 { panic(fmt.Sprintf("Table %s contains multiple autoincremental columns!!", table.name)) @@ -124,7 +60,7 @@ func struct2TableSpec(sv reflect.Value, table *STableSpec) { if column.IsIndex() { table.AddIndex(column.IsUnique(), column.Name()) } - table.columns = append(table.columns, column) + table._columns = append(table._columns, column) } } } diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 460fa1f03f..2d5bb6c6a7 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -30,39 +30,45 @@ import ( // IQuery is an interface that reprsents a SQL query, e.g. // SELECT ... FROM ... WHERE ... type IQuery interface { - // queryString + // String returns the queryString String(fields ...IQueryField) string - // fields in the select clause + // QueryFields returns fields in the select clause QueryFields() []IQueryField - // variables in statement + // Variables returns variables in statement Variables() []interface{} - // convert this SQL to a subquery + // SubQuery convert this SQL to a subquery SubQuery() *SSubQuery - // reference to a field by name + // Field reference to a field by name Field(name string) IQueryField + + // Database returns the database for this query + Database() *SDatabase } // IQuerySource is an interface that represents a data source of a SQL query. the source can be a table or a subquery // e.g. SELECT ... FROM (SELECT * FROM tbl) AS A type IQuerySource interface { - // string in select ... from (expresson here) + // Expression string in select ... from (expresson here) Expression() string - // alias in select ... from (express) as alias + // Alias is the alias in select ... from (express) as alias Alias() string // variables in statement Variables() []interface{} - // reference to a field by name, optionally giving an alias name + // Field reference to a field by name, optionally giving an alias name Field(id string, alias ...string) IQueryField - // return all the fields that this source provides + // Fields return all the fields that this source provides Fields() []IQueryField + + // Database returns the database of this IQuerySource + Database() *SDatabase } // IQueryField is an interface that represents a select field in a SQL query @@ -139,6 +145,13 @@ type SQuery struct { fieldCache map[string]IQueryField snapshot string + + db *SDatabase +} + +// IsGroupBy returns wether the query contains group by clauses +func (tq *SQuery) IsGroupBy() bool { + return len(tq.groupBy) > 0 } // SSubQuery represents a subquery. A subquery is a query used as a query source @@ -255,12 +268,20 @@ func (sq *SSubQuery) Fields() []IQueryField { return ret } +// Database implementation of SSubQuery for IQuerySource +func (sq *SSubQuery) Database() *SDatabase { + return sq.query.Database() +} + // DoQuery returns a SQuery instance that query specified fields from a query source func DoQuery(from IQuerySource, f ...IQueryField) *SQuery { + if from.Database() == nil { + panic("DoQuery IQuerySource with empty database") + } // if len(f) == 0 { // f = from.Fields() // } - tq := SQuery{fields: f, from: from} + tq := SQuery{fields: f, from: from, db: from.Database()} return &tq } @@ -473,6 +494,9 @@ func (tq *SQuery) RightJoin(from IQuerySource, on ICondition) *SQuery { }*/ func (tq *SQuery) _join(from IQuerySource, on ICondition, joinType QueryJoinType) *SQuery { + if from.Database() != tq.db { + panic(fmt.Sprintf("Cannot join across databases %s!=%s", tq.db.name, from.Database().name)) + } if tq.joins == nil { tq.joins = make([]sQueryJoin, 0) } @@ -523,6 +547,10 @@ func (tq *SQuery) SubQuery() *SSubQuery { return &sq } +func (tq *SQuery) Database() *SDatabase { + return tq.db +} + // Row of SQuery returns an instance of sql.Row for native data fetching func (tq *SQuery) Row() *sql.Row { sqlstr := tq.String() @@ -530,7 +558,13 @@ func (tq *SQuery) Row() *sql.Row { if DEBUG_SQLCHEMY { sqlDebug(sqlstr, vars) } - return _db.QueryRow(sqlstr, vars...) + if tq.db == nil { + panic("tq.db") + } + if tq.db.db == nil { + panic("tq.db.db") + } + return tq.db.db.QueryRow(sqlstr, vars...) } // Rows of SQuery returns an instance of sql.Rows for native data fetching @@ -540,7 +574,7 @@ func (tq *SQuery) Rows() (*sql.Rows, error) { if DEBUG_SQLCHEMY { sqlDebug(sqlstr, vars) } - return _db.Query(sqlstr, vars...) + return tq.db.db.Query(sqlstr, vars...) } // Count of SQuery returns the count of a query @@ -551,7 +585,7 @@ func (tq *SQuery) Count() int { return cnt } -func (tq *SQuery) countQuery() *SQuery { +func (tq *SQuery) CountQuery() *SQuery { tq2 := *tq tq2.limit = 0 tq2.offset = 0 @@ -560,13 +594,14 @@ func (tq *SQuery) countQuery() *SQuery { COUNT("count"), }, from: tq2.SubQuery(), + db: tq.Database(), } return cq } // CountWithError of SQuery returns the row count of a query func (tq *SQuery) CountWithError() (int, error) { - cq := tq.countQuery() + cq := tq.CountQuery() count := 0 err := cq.Row().Scan(&count) if err == nil { @@ -655,7 +690,7 @@ func rowScan2StringMap(fields []string, row IRowScanner) (map[string]string, err } else { value := rawValue.Interface() // log.Infof("%s %s", value, reflect.TypeOf(value)) - results[f] = getStringValue(value) + results[f] = GetStringValue(value) } } return results, nil diff --git a/vendor/yunion.io/x/sqlchemy/rawquery.go b/vendor/yunion.io/x/sqlchemy/rawquery.go index a466ec17c8..cc575a8e67 100644 --- a/vendor/yunion.io/x/sqlchemy/rawquery.go +++ b/vendor/yunion.io/x/sqlchemy/rawquery.go @@ -49,11 +49,20 @@ func (rqf *SRawQueryField) Variables() []interface{} { // NewRawQuery returns an instance of SQuery with raw SQL query. e.g. show tables func NewRawQuery(sqlStr string, fields ...string) *SQuery { + return GetDefaultDB().NewRawQuery(sqlStr, fields...) +} + +// NewRawQuery returns an instance of SQuery with raw SQL query for a database, e.g. show tables +func (db *SDatabase) NewRawQuery(sqlStr string, fields ...string) *SQuery { qfs := make([]IQueryField, len(fields)) for i, f := range fields { rqf := SRawQueryField{name: f} qfs[i] = &rqf } - q := SQuery{rawSql: sqlStr, fields: qfs} + q := SQuery{ + db: db, + rawSql: sqlStr, + fields: qfs, + } return &q } diff --git a/vendor/yunion.io/x/sqlchemy/reflect.go b/vendor/yunion.io/x/sqlchemy/reflect.go index 0e2ddaa829..368ec478d8 100644 --- a/vendor/yunion.io/x/sqlchemy/reflect.go +++ b/vendor/yunion.io/x/sqlchemy/reflect.go @@ -38,10 +38,10 @@ func getQuoteStringValue(dat interface{}) string { case reflect.Float32, reflect.Float64: return fmt.Sprintf("%f", value.Float()) } - return strconv.Quote(getStringValue(dat)) + return strconv.Quote(GetStringValue(dat)) } -func getStringValue(dat interface{}) string { +func GetStringValue(dat interface{}) string { value := reflect.ValueOf(dat) switch value.Type() { case tristate.TriStateType: diff --git a/vendor/yunion.io/x/sqlchemy/sql.go b/vendor/yunion.io/x/sqlchemy/sql.go index bf6ea29697..6a91dfc9f8 100644 --- a/vendor/yunion.io/x/sqlchemy/sql.go +++ b/vendor/yunion.io/x/sqlchemy/sql.go @@ -16,37 +16,111 @@ package sqlchemy import ( "database/sql" + "fmt" "yunion.io/x/log" + "yunion.io/x/pkg/errors" ) -// the global DB connection -var _db *sql.DB +// DBName is a type of string for name of database +type DBName string +// SDatabase represents a SQL database +type SDatabase struct { + db *sql.DB + name DBName + backend IBackend +} + +// DefaultDB is the name for the default database instance +const DefaultDB = DBName("__default__") + +// the global DB connection table +var _db_tbl = make(map[DBName]*SDatabase) + +// Deprecated // SetDB sets global DB instance func SetDB(db *sql.DB) { - _db = db + SetDefaultDB(db) +} + +// SetDefaultDB save default global DB instance +func SetDefaultDB(db *sql.DB) { + SetDBWithNameBackend(db, DefaultDB, MySQLBackend) +} + +// SetDBWithName sets a DB instance with given name +// param: name DBName +func SetDBWithNameBackend(db *sql.DB, name DBName, backend DBBackendName) { + _db_tbl[name] = &SDatabase{ + name: name, + db: db, + backend: getBackend(backend), + } } // GetDB get DB instance +// Deprecated func GetDB() *sql.DB { - return _db + return GetDefaultDB().db +} + +// GetDefaultDB get the DB instance set by default +func GetDefaultDB() *SDatabase { + return GetDBWithName(DefaultDB) +} + +// GetDBWithName returns the db instance with given name +func GetDBWithName(name DBName) *SDatabase { + if name == DefaultDB && len(_db_tbl) == 1 { + for _, db := range _db_tbl { + return db + } + } + if db, ok := _db_tbl[name]; ok { + return db + } + panic(fmt.Sprintf("no such database %s", name)) +} + +type sDBReferer struct { + dbName DBName + _db_cache *SDatabase +} + +func (r *sDBReferer) Database() *SDatabase { + if r._db_cache == nil { + r._db_cache = GetDBWithName(r.dbName) + } + return r._db_cache } // CloseDB close DB connection func CloseDB() { - _db.Close() - _db = nil + names := make([]DBName, 0) + for n, db := range _db_tbl { + names = append(names, n) + db.db.Close() + } + for _, n := range names { + delete(_db_tbl, n) + } } type tableName struct { Name string } -// GetTables get all tables' name in database +// GetTables get all tables' name in default database +// Deprecated func GetTables() []string { + return GetDefaultDB().GetTables() +} + +// GetTables get all tables' name in database +func (db *SDatabase) GetTables() []string { tables := make([]tableName, 0) - q := NewRawQuery("SHOW TABLES", "name") + q := db.NewRawQuery(db.backend.GetTableSQL(), "name") err := q.All(&tables) if err != nil { log.Errorf("show tables fail %s", err) @@ -59,7 +133,56 @@ func GetTables() []string { return ret } -// Exec execute a raw SQL query +// Exec execute a raw SQL query for the default db instance +// Deprecated func Exec(sql string, args ...interface{}) (sql.Result, error) { - return _db.Exec(sql, args...) + return GetDefaultDB().Exec(sql, args...) +} + +// Exec execute a raw SQL query for a db instance +func (db *SDatabase) Exec(sql string, args ...interface{}) (sql.Result, error) { + return db.db.Exec(sql, args...) +} + +type SSqlResult struct { + Result sql.Result + Error error +} + +func (db *SDatabase) TxBatchExec(sqlstr string, varsList [][]interface{}) ([]SSqlResult, error) { + tx, err := db.db.Begin() + if err != nil { + return nil, errors.Wrap(err, "Begin transaction") + } + defer tx.Rollback() + stmt, err := tx.Prepare(sqlstr) + if err != nil { + return nil, errors.Wrapf(err, "Prepare sql %s", sqlstr) + } + defer stmt.Close() + + results := make([]SSqlResult, len(varsList)) + for i := range varsList { + vars := varsList[i] + result, err := stmt.Exec(vars...) + results[i] = SSqlResult{ + Result: result, + Error: err, + } + } + + err = tx.Commit() + if err != nil { + return nil, errors.Wrap(err, "Commit transaction") + } + + return results, nil +} + +func (db *SDatabase) TxExec(sqlstr string, vars ...interface{}) (sql.Result, error) { + results, err := db.TxBatchExec(sqlstr, [][]interface{}{vars}) + if err != nil { + return nil, errors.Wrap(err, "TxBatchExec") + } + return results[0].Result, results[0].Error } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 3fdc0e5e6a..aed468363c 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -16,10 +16,7 @@ package sqlchemy import ( "fmt" - "math/bits" - "regexp" "sort" - "strconv" "strings" "yunion.io/x/log" @@ -27,156 +24,8 @@ import ( "yunion.io/x/pkg/utils" ) -type sSqlColumnInfo struct { - Field string - Type string - Collation string - Null string - Key string - Default string - Extra string - Privileges string - Comment string -} - -func decodeSqlTypeString(typeStr string) []string { - typeReg := regexp.MustCompile(`(\w+)\((\d+)(,\s*(\d+))?\)`) - matches := typeReg.FindStringSubmatch(typeStr) - if len(matches) >= 3 { - return matches[1:] - } - parts := strings.Split(typeStr, " ") - return []string{parts[0]} -} - -func (info *sSqlColumnInfo) toColumnSpec() IColumnSpec { - tagmap := make(map[string]string) - - matches := decodeSqlTypeString(info.Type) - typeStr := strings.ToUpper(matches[0]) - width := 0 - if len(matches) > 1 { - width, _ = strconv.Atoi(matches[1]) - } - if width > 0 { - tagmap[TAG_WIDTH] = fmt.Sprintf("%d", width) - } - if info.Null == "YES" { - tagmap[TAG_NULLABLE] = "true" - } else { - tagmap[TAG_NULLABLE] = "false" - } - if info.Key == "PRI" { - tagmap[TAG_PRIMARY] = "true" - } else { - tagmap[TAG_PRIMARY] = "false" - } - charset := "" - if info.Collation == "ascii_general_ci" { - charset = "ascii" - } else if info.Collation == "utf8_general_ci" || info.Collation == "utf8mb4_unicode_ci" { - charset = "utf8" - } else { - charset = "ascii" - } - if len(charset) > 0 { - tagmap[TAG_CHARSET] = charset - } - if info.Default != "NULL" { - tagmap[TAG_DEFAULT] = info.Default - } - if strings.HasSuffix(typeStr, "CHAR") { - c := NewTextColumn(info.Field, tagmap, false) - return &c - } else if strings.HasSuffix(typeStr, "TEXT") { - tagmap[TAG_TEXT_LENGTH] = typeStr[:len(typeStr)-4] - c := NewTextColumn(info.Field, tagmap, false) - return &c - } else if strings.HasSuffix(typeStr, "INT") { - if info.Extra == "auto_increment" { - tagmap[TAG_AUTOINCREMENT] = "true" - } - unsigned := false - if strings.HasSuffix(info.Type, " unsigned") { - unsigned = true - } - if _, ok := tagmap[TAG_WIDTH]; !ok { - if unsigned { - tagmap[TAG_WIDTH] = uintWidthString(typeStr) - } else { - tagmap[TAG_WIDTH] = intWidthString(typeStr) - } - } - c := NewIntegerColumn(info.Field, typeStr, unsigned, tagmap, false) - return &c - } else if typeStr == "FLOAT" || typeStr == "DOUBLE" { - c := NewFloatColumn(info.Field, typeStr, tagmap, false) - return &c - } else if typeStr == "DECIMAL" { - if len(matches) > 3 { - precision, _ := strconv.Atoi(matches[3]) - if precision > 0 { - tagmap[TAG_PRECISION] = fmt.Sprintf("%d", precision) - } - } - c := NewDecimalColumn(info.Field, tagmap, false) - return &c - } else if typeStr == "DATETIME" { - c := NewDateTimeColumn(info.Field, tagmap, false) - return &c - } else if typeStr == "DATE" || typeStr == "TIMESTAMP" { - c := NewTimeTypeColumn(info.Field, typeStr, tagmap, false) - return &c - } else if strings.HasPrefix(typeStr, "ENUM(") { - // enum type, force convert to text - // discourage use of enum, use text instead - enums := utils.FindWords([]byte(typeStr[5:len(typeStr)-1]), 0) - - width := 0 - for i := range enums { - if width < len(enums[i]) { - width = len(enums[i]) - } - } - tagmap[TAG_WIDTH] = fmt.Sprintf("%d", 1< 1 { - autoInc = fmt.Sprintf(" AUTO_INCREMENT=%d", intC.AutoIncrementOffset) - } - } - if c.IsIndex() { - indexes = append(indexes, fmt.Sprintf("KEY `ix_%s_%s` (`%s`)", ts.name, c.Name(), c.Name())) - } - } - if len(primaries) > 0 { - cols = append(cols, fmt.Sprintf("PRIMARY KEY (%s)", strings.Join(primaries, ", "))) - } - if len(indexes) > 0 { - cols = append(cols, indexes...) - } - return fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci%s", ts.name, strings.Join(cols, ",\n"), autoInc) +func (ts *STableSpec) CreateSQLs() []string { + return ts.Database().backend.GetCreateSQLs(ts) } // NewTableInstance return an new table instance from an ITableSpec @@ -238,6 +236,11 @@ func (tbl *STable) Fields() []IQueryField { return ret } +// Database implementaion of STable for IQuerySource +func (tbl *STable) Database() *SDatabase { + return tbl.spec.Database() +} + // Expression implementation of STableField for IQueryField func (c *STableField) Expression() string { if len(c.alias) > 0 { diff --git a/vendor/yunion.io/x/sqlchemy/tablealias.go b/vendor/yunion.io/x/sqlchemy/tablealias.go index 96b14f118b..f44e185119 100644 --- a/vendor/yunion.io/x/sqlchemy/tablealias.go +++ b/vendor/yunion.io/x/sqlchemy/tablealias.go @@ -30,3 +30,9 @@ func getTableAliasName() string { tableID++ return fmt.Sprintf("t%d", tableID) } + +func ResetTableID() { + tableIDLock.Lock() + defer tableIDLock.Unlock() + tableID = 0 +} diff --git a/vendor/yunion.io/x/sqlchemy/template.go b/vendor/yunion.io/x/sqlchemy/template.go new file mode 100644 index 0000000000..845e77b311 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/template.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +import ( + "bytes" + "text/template" +) + +var ( + templateTbl = make(map[string]*template.Template) +) + +func templateEval(temp string, variables interface{}) string { + if eval, ok := templateTbl[temp]; !ok { + eval = template.Must(template.New(temp).Parse(temp)) + templateTbl[temp] = eval + } + buf := new(bytes.Buffer) + templateTbl[temp].Execute(buf, variables) + return buf.String() +} diff --git a/vendor/yunion.io/x/sqlchemy/union.go b/vendor/yunion.io/x/sqlchemy/union.go index fb23d0c2d5..31fb8beeda 100644 --- a/vendor/yunion.io/x/sqlchemy/union.go +++ b/vendor/yunion.io/x/sqlchemy/union.go @@ -69,9 +69,11 @@ type SUnion struct { alias string queries []IQuery fields []IQueryField - orderBy []sQueryOrder - limit int - offset int + // orderBy []sQueryOrder + // limit int + // offset int + + isAll bool } // Alias implementation of SUnion for IQuerySource @@ -79,19 +81,27 @@ func (uq *SUnion) Alias() string { return uq.alias } +func (uq *SUnion) operator() string { + if uq.isAll { + return " UNION ALL " + } else { + return " UNION " + } +} + // Expression implementation of SUnion for IQuerySource func (uq *SUnion) Expression() string { var buf strings.Builder - buf.WriteByte('(') + buf.WriteString("(") for i := range uq.queries { if i != 0 { - buf.WriteString(" UNION ") + buf.WriteString(uq.operator()) } - buf.WriteByte('(') + // buf.WriteByte('(') buf.WriteString(uq.queries[i].String()) - buf.WriteByte(')') + // buf.WriteByte(')') } - if uq.orderBy != nil && len(uq.orderBy) > 0 { + /*if uq.orderBy != nil && len(uq.orderBy) > 0 { buf.WriteString(" ORDER BY ") for i, f := range uq.orderBy { if i > 0 { @@ -105,7 +115,7 @@ func (uq *SUnion) Expression() string { } if uq.offset > 0 { buf.WriteString(fmt.Sprintf(" OFFSET %d", uq.offset)) - } + }*/ buf.WriteByte(')') return buf.String() } @@ -130,16 +140,16 @@ func (tq *SUnion) Desc(fields ...interface{}) *SUnion { */ // Limit adds limit to a union query -func (uq *SUnion) Limit(limit int) *SUnion { - uq.limit = limit - return uq -} +// func (uq *SUnion) Limit(limit int) *SUnion { +// uq.limit = limit +// return uq +// } // Offset adds offset to a union query -func (uq *SUnion) Offset(offset int) *SUnion { - uq.offset = offset - return uq -} +// func (uq *SUnion) Offset(offset int) *SUnion { +// uq.offset = offset +// return uq +// } // Fields implementation of SUnion for IQuerySource func (uq *SUnion) Fields() []IQueryField { @@ -168,6 +178,11 @@ func (uq *SUnion) Variables() []interface{} { return ret } +// Database implementation of SUnion for IQUerySource +func (uq *SUnion) Database() *SDatabase { + return uq.queries[0].Database() +} + // Union method returns union query of several queries. // Require the fields of all queries should exactly match // deprecated @@ -182,6 +197,14 @@ func Union(query ...IQuery) *SUnion { // UnionWithError constructs union query of several Queries // Require the fields of all queries should exactly match func UnionWithError(query ...IQuery) (*SUnion, error) { + return unionWithError(false, query...) +} + +func UnionAllWithError(query ...IQuery) (*SUnion, error) { + return unionWithError(true, query...) +} + +func unionWithError(isAll bool, query ...IQuery) (*SUnion, error) { if len(query) == 0 { return nil, errors.Wrap(sql.ErrNoRows, "empty union query") } @@ -191,7 +214,13 @@ func UnionWithError(query ...IQuery) (*SUnion, error) { fieldNames = append(fieldNames, f.Name()) } + var db *SDatabase for i := 1; i < len(query); i++ { + if db == nil { + db = query[i].Database() + } else if db != query[i].Database() { + panic(ErrUnionDatabasesNotMatch) + } qfields := query[i].QueryFields() if len(fieldNames) != len(qfields) { return nil, errors.Wrap(ErrUnionFieldsNotMatch, "number not match") @@ -209,6 +238,7 @@ func UnionWithError(query ...IQuery) (*SUnion, error) { alias: getTableAliasName(), queries: query, fields: fields, + isAll: isAll, } for i := range fieldNames { diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 9f8b38feb3..356dad3199 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -20,6 +20,8 @@ import ( "reflect" "strings" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -42,7 +44,7 @@ func (ts *STableSpec) prepareUpdate(dt interface{}) (*SUpdateSession, error) { fields := reflectutils.FetchStructFieldValueSet(dataValue) // fetchStructFieldNameValue(dataType, dataValue) zeroPrimary := make([]string, 0) - for _, c := range ts.columns { + for _, c := range ts.Columns() { k := c.Name() ov, ok := fields.GetInterface(k) if !ok { @@ -96,12 +98,21 @@ func (uds UpdateDiffs) String() string { return obj.String() } -func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { +type sUpdateSQLResult struct { + sql string + vars []interface{} + setters UpdateDiffs + primaries map[string]interface{} +} + +func (us *SUpdateSession) saveUpdateSql(dt interface{}) (*sUpdateSQLResult, error) { beforeUpdateFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") if beforeUpdateFunc.IsValid() && !beforeUpdateFunc.IsNil() { beforeUpdateFunc.Call([]reflect.Value{}) } + now := timeutils.UtcNow() + // dataType := reflect.TypeOf(dt).Elem() dataValue := reflect.ValueOf(dt).Elem() ofields := reflectutils.FetchStructFieldValueSet(us.oValue) @@ -111,7 +122,7 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { updatedFields := make([]string, 0) primaries := make(map[string]interface{}) setters := UpdateDiffs{} - for _, c := range us.tableSpec.columns { + for _, c := range us.tableSpec.Columns() { k := c.Name() of, _ := ofields.GetInterface(k) nf, _ := fields.GetInterface(k) @@ -125,13 +136,11 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { } continue } - nc, ok := c.(*SIntegerColumn) - if ok && nc.IsAutoVersion { + if c.IsAutoVersion() { versionFields = append(versionFields, k) continue } - dtc, ok := c.(*SDateTimeColumn) - if ok && dtc.IsUpdatedAt { + if c.IsUpdatedAt() { updatedFields = append(updatedFields, k) continue } @@ -148,6 +157,10 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { return nil, ErrNoDataToUpdate } + if len(primaries) == 0 { + return nil, ErrEmptyPrimaryKey + } + vars := make([]interface{}, 0) var buf bytes.Buffer buf.WriteString(fmt.Sprintf("UPDATE `%s` SET ", us.tableSpec.name)) @@ -169,13 +182,11 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = ?", updatedField)) + vars = append(vars, now) } buf.WriteString(" WHERE ") first = true - if len(primaries) == 0 { - return nil, ErrEmptyPrimaryKey - } for k, v := range primaries { if first { first = false @@ -189,47 +200,77 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { if DEBUG_SQLCHEMY { log.Infof("Update: %s %s", buf.String(), vars) } - results, err := _db.Exec(buf.String(), vars...) + + return &sUpdateSQLResult{ + sql: buf.String(), + vars: vars, + setters: setters, + primaries: primaries, + }, nil +} + +func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { + sqlResult, err := us.saveUpdateSql(dt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "saveUpateSql") } - aCnt, err := results.RowsAffected() + + err = us.tableSpec.execUpdateSql(dt, sqlResult) if err != nil { - return nil, err + return nil, errors.Wrap(err, "execUpdateSql") } - if aCnt > 1 { - return nil, errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) + + return sqlResult.setters, nil +} + +func (ts *STableSpec) execUpdateSql(dt interface{}, result *sUpdateSQLResult) error { + results, err := ts.Database().TxExec(result.sql, result.vars...) + if err != nil { + return errors.Wrap(err, "TxExec") } - q := us.tableSpec.Query() - for k, v := range primaries { + + if ts.Database().backend.CanSupportRowAffected() { + aCnt, err := results.RowsAffected() + if err != nil { + return errors.Wrap(err, "results.RowsAffected") + } + if aCnt > 1 { + return errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) + } + } + q := ts.Query() + for k, v := range result.primaries { q = q.Equals(k, v) } err = q.First(dt) if err != nil { - return nil, errors.Wrap(err, "query after update failed") + return errors.Wrap(err, "query after update failed") } - return setters, nil + return nil } // Update method of STableSpec updates a record of a table, // dt is the point to the struct storing the record // doUpdate provides method to update the field of the record func (ts *STableSpec) Update(dt interface{}, doUpdate func() error) (UpdateDiffs, error) { + if !ts.Database().backend.CanUpdate() { + return nil, errors.ErrNotSupported + } session, err := ts.prepareUpdate(dt) if err != nil { - return nil, err + return nil, errors.Wrap(err, "prepareUpdate") } err = doUpdate() if err != nil { - return nil, err + return nil, errors.Wrap(err, "") } uds, err := session.saveUpdate(dt) - if err == ErrNoDataToUpdate { + if err != nil && errors.Cause(err) == ErrNoDataToUpdate { return nil, nil } else if err == nil { if DEBUG_SQLCHEMY { log.Debugf("Update diff: %s", uds) } } - return uds, err + return uds, errors.Wrap(err, "saveUpdate") }