opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12065 from swordqiu/feature/qj-support-clickhouse

Browse Source 
feature: adapt to new version of sqlchemy that supports sqlite&clickhouse
This commit is contained in:
Zexi Li
2021-11-17 15:57:19 +08:00
committed by GitHub
parent 4669ac1fff 97a3ab12a2
commit ef0a970a7c
176 changed files with 12654 additions and 1326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/climc/shell/misc/version.go
+32 -1
View File
@@ -23,6 +23,7 @@ import (
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
modules "yunion.io/x/onecloud/pkg/mcclient/modules"
"yunion.io/x/onecloud/pkg/mcclient/modules/identity"
)
@@ -32,7 +33,7 @@ func init() {
SERVICE string `help:"Service type"`
}
R(&VersionOptions{}, "version-show", "query backend service for its version", func(s *mcclient.ClientSession, args *VersionOptions) error {
body, err := modules.GetVersion(s, args.SERVICE)
body, err := modulebase.GetVersion(s, args.SERVICE)
if err != nil {
return err
}
@@ -40,6 +41,36 @@ func init() {
return nil
})
type StatsOptions struct {
SERVICE string `help:"Service type"`
}
R(&StatsOptions{}, "api-stats-show", "query backend service for its stats", func(s *mcclient.ClientSession, args *StatsOptions) error {
body, err := modulebase.GetStats(s, "stats", args.SERVICE)
if err != nil {
return err
}
printObject(body)
return nil
})
R(&StatsOptions{}, "db-stats-show", "query backend service for its db stats", func(s *mcclient.ClientSession, args *StatsOptions) error {
body, err := modulebase.GetStats(s, "db_stats", args.SERVICE)
if err != nil {
return err
}
stats, _ := body.Get("db_stats")
printObject(stats)
return nil
})
R(&StatsOptions{}, "worker-stats-show", "query backend service for its worker stats", func(s *mcclient.ClientSession, args *StatsOptions) error {
body, err := modulebase.GetStats(s, "worker_stats", args.SERVICE)
if err != nil {
return err
}
data, _ := body.GetArray("workers")
printList(&modulebase.ListResult{Data: data}, nil)
return nil
})
type VersionListOptions struct {
}
R(&VersionListOptions{}, "version-list", "query all backend service version", func(s *mcclient.ClientSession, args *VersionListOptions) error {
go.mod
+3 -2
View File
@@ -10,6 +10,7 @@ require (
github.com/Azure/go-autorest/autorest v0.9.6
github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/ClickHouse/clickhouse-go v1.4.7 // indirect
github.com/DataDog/dd-trace-go v0.6.1 // indirect
github.com/DataDog/zstd v1.3.4 // indirect
github.com/Masterminds/goutils v1.1.0 // indirect
@@ -154,8 +155,8 @@ require (
yunion.io/x/jsonutils v0.0.0-20211105163012-d846c05a3c9a
yunion.io/x/log v0.0.0-20201210064738-43181789dc74
yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f
yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada
yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1
yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
)
go.sum
+14 -4
View File
@@ -59,6 +59,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/clickhouse-go v1.4.7 h1:NNZQmlW8dVxGn19pF65BmWr0vq8Pj5Iy8ykyBKhFCPw=
github.com/ClickHouse/clickhouse-go v1.4.7/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/DataDog/dd-trace-go v0.6.1 h1:nsZ2lohbSw1CKtfNRu3wPh1jFirv6XSz8vqNpuIYWbM=
github.com/DataDog/dd-trace-go v0.6.1/go.mod h1:SmQTTcC37XMyEm75HV0AWiZIYxDiaNhRi49zorIpW+o=
github.com/DataDog/zstd v1.3.4 h1:LAGHkXuvC6yky+C2CUG2tD7w8QlrUwpue8XwIh0X4AY=
@@ -145,6 +147,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
@@ -162,6 +166,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
@@ -261,6 +267,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -378,6 +385,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -423,6 +431,7 @@ github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae h1:XoMPFIGibcPKgLr
github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI=
github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 h1:ttJD8hTqvrPEUBoAG5hJKbDOJ84u7zmbnZsUL4V9430=
github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8/go.mod h1:VXFH11P7fHn2iPBsfSW1JacR59rttTcafJnwYcI/IdY=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/libvirt/libvirt-go-xml v5.2.0+incompatible h1:ALyVpa0/lFfFaUpb5l0fNBohufmQG6jsOGRCl/nKEi8=
github.com/libvirt/libvirt-go-xml v5.2.0+incompatible/go.mod h1:oBlgD3xOA01ihiK5stbhFzvieyW+jVS6kbbsMVF623A=
github.com/ma314smith/signedxml v0.0.0-20200410192636-c342a2d0ae60 h1:q5rqPuvxdOzg1NC6sls3XSpRokQ5KAmLIYdLW1/kyOo=
@@ -443,6 +452,7 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.7.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-tty v0.0.0-20181127064339-e4f871175a2f h1:4P7Ul+TAnk92vTeVkXs6VLjmf1EhrYtDRa03PCYY6VM=
@@ -941,11 +951,11 @@ yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7/go.mod h1:0vLkNEhlmA64HViPB
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f h1:maply0f3AsvoXqkpTKkb6/tZFbRAT/rdkk1Cw3FkoB8=
yunion.io/x/pkg v0.0.0-20210918114143-ce839f862c5f/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt6m4ZuGtarIjwNfUsg=
yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada h1:ZIhSSwFHfnzPMKSyiN0nkGwvC6RUeuCIAnWuTtDS900=
yunion.io/x/sqlchemy v0.0.0-20210918113031-c1c107f37ada/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1 h1:WxvTLANfa01EFTORGY2GapbdjQkuLaQ/CwMDdBy632U=
yunion.io/x/sqlchemy v0.0.0-20211116231750-5ec40db49db1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
pkg/ansibleserver/service/service.go
+5 -3
View File
@@ -17,9 +17,8 @@ package service
import (
"os"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
"yunion.io/x/onecloud/pkg/ansibleserver/models"
"yunion.io/x/onecloud/pkg/ansibleserver/options"
@@ -43,9 +42,12 @@ func StartService() {
models.InitPlaybookWorker()
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
common_app.ServeForever(app, baseOpts)
pkg/appsrv/workers.go
+4
View File
@@ -281,6 +281,8 @@ type SWorkerManagerStates struct {
MaxWorkerCnt int
ActiveWorkerCnt int
DetachWorkerCnt int
DbWorker bool
AllowOverflow bool
}
func (s SWorkerManagerStates) IsBusy() bool {
@@ -299,6 +301,8 @@ func (wm *SWorkerManager) getState() SWorkerManagerStates {
state.MaxWorkerCnt = wm.workerCount
state.ActiveWorkerCnt = wm.activeWorker.size()
state.DetachWorkerCnt = wm.detachedWorker.size()
state.DbWorker = wm.dbWorker
state.AllowOverflow = wm.ignoreOverflow
return state
}
pkg/cloudcommon/database.go
+18 -1
View File
@@ -36,6 +36,8 @@ import (
const (
MIN_DB_CONN_MAX = 5
ClickhouseDB = sqlchemy.DBName("clickhosue_db")
)
func InitDB(options *common_options.DBOptions) {
@@ -56,11 +58,26 @@ func InitDB(options *common_options.DBOptions) {
if err != nil {
log.Fatalf("Invalid SqlConnection string: %s error: %v", options.SqlConnection, err)
}
log.Infof("database dialect: %s sqlStr: %s", dialect, sqlStr)
dbConn, err := sql.Open(dialect, sqlStr)
if err != nil {
panic(err)
}
sqlchemy.SetDB(dbConn)
backend := sqlchemy.MySQLBackend
if dialect == "sqlite3" {
backend = sqlchemy.SQLiteBackend
}
sqlchemy.SetDBWithNameBackend(dbConn, sqlchemy.DefaultDB, backend)
dialect, sqlStr, err = options.GetClickhouseConnStr()
if err == nil {
// connect to clickcloud
click, err := sql.Open(dialect, sqlStr)
if err != nil {
panic(err)
}
sqlchemy.SetDBWithNameBackend(click, ClickhouseDB, sqlchemy.ClickhouseBackend)
}
switch options.LockmanMethod {
case common_options.LockMethodInMemory, "":
pkg/cloudcommon/db/db_dispatcher.go
+11 -9
View File
@@ -145,7 +145,7 @@ func listItemsQueryByColumn(manager IModelManager, q *sqlchemy.SQuery, userCred
}
} else if len(arrV) > 1 {
for i := range arrV {
arrV[i] = colSpec.ConvertFromString(arrV[i])
arrV[i] = sqlchemy.GetStringValue(colSpec.ConvertFromString(arrV[i]))
}
q = q.In(fn, arrV)
} else if len(arrV) == 1 {
@@ -691,15 +691,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
if orderBy == nil {
orderBy = []string{}
}
if primaryCol != nil && primaryCol.IsNumeric() {
orderBy = append(orderBy, primaryCol.Name())
} else if manager.TableSpec().ColumnSpec("created_at") != nil {
orderBy = append(orderBy, "created_at")
if manager.TableSpec().ColumnSpec("name") != nil {
orderBy = append(orderBy, "name")
}
if primaryCol != nil {
if !q.IsGroupBy() {
if primaryCol != nil && primaryCol.IsNumeric() {
orderBy = append(orderBy, primaryCol.Name())
} else if manager.TableSpec().ColumnSpec("created_at") != nil {
orderBy = append(orderBy, "created_at")
if manager.TableSpec().ColumnSpec("name") != nil {
orderBy = append(orderBy, "name")
}
if primaryCol != nil {
orderBy = append(orderBy, primaryCol.Name())
}
}
}
for _, orderByField := range orderBy {
pkg/cloudcommon/db/i18n.go
+15 -12
View File
@@ -17,9 +17,8 @@ package db
import (
"context"
"github.com/pkg/errors"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/compare"
"yunion.io/x/onecloud/pkg/i18n"
@@ -272,16 +271,20 @@ func (manager *SI18nManager) newFromI18n(ctx context.Context, userCred mcclient.
}
func (self *SI18n) updateFromI18n(ctx context.Context, userCred mcclient.TokenCredential, entry IModelI18nEntry) error {
_, err := Update(self, func() error {
self.KeyValue = entry.GetKeyValue()
self.Cn = entry.Lookup(i18n.I18N_TAG_CHINESE)
self.En = entry.Lookup(i18n.I18N_TAG_ENGLISH)
return nil
})
if err != nil {
log.Infof("updateFromI18n error %s", err)
return err
zh := entry.Lookup(i18n.I18N_TAG_CHINESE)
en := entry.Lookup(i18n.I18N_TAG_ENGLISH)
log.Debugf("updateFromI18n %#v %s %s", self, zh, en)
if self.Cn != zh || self.En != en {
_, err := Update(self, func() error {
self.KeyValue = entry.GetKeyValue()
self.Cn = zh
self.En = en
return nil
})
if err != nil {
log.Infof("updateFromI18n error %s", err)
return err
}
}
return nil
pkg/cloudcommon/db/modelbase.go
+10 -2
View File
@@ -53,11 +53,19 @@ type SModelBaseManager struct {
}
func NewModelBaseManager(model interface{}, tableName string, keyword string, keywordPlural string) SModelBaseManager {
return NewModelBaseManagerWithSplitable(model, tableName, keyword, keywordPlural, "", "", 0, 0)
return NewModelBaseManagerWithDBName(model, tableName, keyword, keywordPlural, sqlchemy.DefaultDB)
}
func NewModelBaseManagerWithDBName(model interface{}, tableName string, keyword string, keywordPlural string, dbName sqlchemy.DBName) SModelBaseManager {
return NewModelBaseManagerWithSplitableDBName(model, tableName, keyword, keywordPlural, "", "", 0, 0, dbName)
}
func NewModelBaseManagerWithSplitable(model interface{}, tableName string, keyword string, keywordPlural string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) SModelBaseManager {
ts := newTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments)
return NewModelBaseManagerWithSplitableDBName(model, tableName, keyword, keywordPlural, indexField, dateField, maxDuration, maxSegments, sqlchemy.DefaultDB)
}
func NewModelBaseManagerWithSplitableDBName(model interface{}, tableName string, keyword string, keywordPlural string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) SModelBaseManager {
ts := newTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments, dbName)
modelMan := SModelBaseManager{tableSpec: ts, keyword: keyword, keywordPlural: keywordPlural}
return modelMan
}
pkg/cloudcommon/db/models.go
+2 -2
View File
@@ -138,8 +138,8 @@ func CheckSync(autoSync bool) bool {
return inSync
}
func EnsureAppInitSyncDB(app *appsrv.Application, opt *common_options.DBOptions, modelInitDBFunc func() error) {
cloudcommon.InitDB(opt)
func EnsureAppSyncDB(app *appsrv.Application, opt *common_options.DBOptions, modelInitDBFunc func() error) {
// cloudcommon.InitDB(opt)
if !CheckSync(opt.AutoSyncTable) {
log.Fatalf("database schema not in sync!")
pkg/cloudcommon/db/opslog.go
+13 -2
View File
@@ -28,6 +28,7 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/reflectutils"
"yunion.io/x/pkg/util/stringutils"
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/apis"
@@ -77,6 +78,7 @@ var _ IModelManager = (*SOpsLogManager)(nil)
var _ IModel = (*SOpsLog)(nil)
var opslogQueryWorkerMan *appsrv.SWorkerManager
var opslogWriteWorkerMan *appsrv.SWorkerManager
func init() {
OpsLog = &SOpsLogManager{NewModelBaseManagerWithSplitable(
@@ -91,7 +93,8 @@ func init() {
)}
OpsLog.SetVirtualObject(OpsLog)
opslogQueryWorkerMan = appsrv.NewWorkerManager("opslog_query_worker", 2, 1024, true)
opslogQueryWorkerMan = appsrv.NewWorkerManager("opslog_query_worker", 2, 512, true)
opslogWriteWorkerMan = appsrv.NewWorkerManager("opslog_write_worker", 1, 2048, true)
}
func (manager *SOpsLogManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) {
@@ -188,12 +191,20 @@ func (manager *SOpsLogManager) LogEvent(model IModel, action string, notes inter
}
}
err := manager.TableSpec().Insert(context.Background(), opslog)
opslogWriteWorkerMan.Run(opslog, nil, nil)
}
func (opslog *SOpsLog) Run() {
err := OpsLog.TableSpec().Insert(context.Background(), opslog)
if err != nil {
log.Errorf("fail to insert opslog: %s", err)
}
}
func (opslog *SOpsLog) Dump() string {
return fmt.Sprintf("[%s] %s %s", timeutils.CompactTime(opslog.OpsTime), opslog.Action, opslog.Notes)
}
func combineNotes(ctx context.Context, m2 IModel, notes jsonutils.JSONObject) *jsonutils.JSONDict {
desc := m2.GetShortDesc(ctx)
if notes != nil {
pkg/cloudcommon/db/resourcebase_test.go
+2
View File
@@ -18,6 +18,7 @@ import (
"testing"
"yunion.io/x/pkg/utils"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/rbacutils"
@@ -32,6 +33,7 @@ func (tk *omniToken) IsAllow(scope rbacutils.TRbacScope, service string, resourc
}
func TestListFields(t *testing.T) {
sqlchemy.SetupMockDatabaseBackend()
man := NewResourceBaseManager(
&SResourceBase{},
"tbl",
pkg/cloudcommon/db/tablespec.go
+4 -2
View File
@@ -55,17 +55,19 @@ type sTableSpec struct {
sqlchemy.ITableSpec
}
func newTableSpec(model interface{}, tableName string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) ITableSpec {
func newTableSpec(model interface{}, tableName string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) ITableSpec {
var itbl sqlchemy.ITableSpec
if len(indexField) > 0 && len(dateField) > 0 {
var err error
itbl, err = splitable.NewSplitTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments)
itbl, err = splitable.NewSplitTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments, dbName)
if err != nil {
log.Errorf("NewSplitTableSpec %s %s", tableName, err)
return nil
} else {
log.Debugf("table %s maxDuration %d hour maxSegements %d", tableName, maxDuration/time.Hour, maxSegments)
}
} else if len(dbName) > 0 {
itbl = sqlchemy.NewTableSpecFromStructWithDBName(model, tableName, dbName)
} else {
itbl = sqlchemy.NewTableSpecFromStruct(model, tableName)
}
pkg/cloudcommon/db/taskman/tasks.go
+4 -1
View File
@@ -592,12 +592,15 @@ func (self *STask) GetRequestContext() appctx.AppContextData {
}
func (self *STask) SaveRequestContext(data *appctx.AppContextData) {
jsonData := jsonutils.Marshal(data)
log.Debugf("SaveRequestContext %s param %s", jsonData, self.Params)
_, err := db.Update(self, func() error {
params := self.Params.CopyExcludes(REQUEST_CONTEXT_KEY)
params.Add(jsonutils.Marshal(data), REQUEST_CONTEXT_KEY)
params.Add(jsonData, REQUEST_CONTEXT_KEY)
self.Params = params
return nil
})
log.Debugf("Params: %s", self.Params)
if err != nil {
log.Errorf("save_request_context fail %s", err)
}
pkg/cloudcommon/options/options.go
+34 -14
View File
@@ -25,6 +25,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"golang.org/x/net/http/httpproxy"
"golang.org/x/text/language"
@@ -38,6 +39,7 @@ import (
"yunion.io/x/structarg"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/atexit"
"yunion.io/x/onecloud/pkg/util/httputils"
)
@@ -121,7 +123,7 @@ type CommonOptions struct {
AdminDomain string `help:"Admin user domain" default:"Default"`
AdminPassword string `help:"Admin password" alias:"admin-passwd"`
AdminProject string `help:"Admin project" default:"system" alias:"admin-tenant-name"`
AdminProjectDomain string `help:"Domain of Admin project" default:"default"`
AdminProjectDomain string `help:"Domain of Admin project" default:"Default"`
AuthTokenCacheSize uint32 `help:"Auth token Cache Size" default:"2048"`
TenantCacheExpireSeconds int `help:"expire seconds of cached tenant/domain info. defailt 15 minutes" default:"900"`
@@ -141,6 +143,8 @@ type HostCommonOptions struct {
type DBOptions struct {
SqlConnection string `help:"SQL connection string" alias:"connection"`
Clickhouse string `help:"Connection string for click house"`
AutoSyncTable bool `help:"Automatically synchronize table changes if differences are detected"`
ExitAfterDBInit bool `help:"Exit program after db initialization" default:"false"`
@@ -173,23 +177,23 @@ type EtcdOptions struct {
EtcdKey string `help:"path to key file for connecting to etcd cluster"`
}
func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) {
func (opt *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) {
var (
cert tls.Certificate
certLoaded bool
capool *x509.CertPool
)
if this.EtcdCert != "" && this.EtcdKey != "" {
if opt.EtcdCert != "" && opt.EtcdKey != "" {
var err error
cert, err = tls.LoadX509KeyPair(this.EtcdCert, this.EtcdKey)
cert, err = tls.LoadX509KeyPair(opt.EtcdCert, opt.EtcdKey)
if err != nil {
return nil, errors.Wrap(err, "load etcd cert and key")
}
certLoaded = true
this.EtcdUseTLS = true
opt.EtcdUseTLS = true
}
if this.EtcdCacert != "" {
data, err := ioutil.ReadFile(this.EtcdCacert)
if opt.EtcdCacert != "" {
data, err := ioutil.ReadFile(opt.EtcdCacert)
if err != nil {
return nil, errors.Wrap(err, "read cacert file")
}
@@ -206,15 +210,15 @@ func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) {
}
capool.AddCert(cacert)
}
this.EtcdUseTLS = true
opt.EtcdUseTLS = true
}
if this.EtcdSkipTLSVerify { // it's false by default, true means user intends to use tls
this.EtcdUseTLS = true
if opt.EtcdSkipTLSVerify { // it's false by default, true means user intends to use tls
opt.EtcdUseTLS = true
}
if this.EtcdUseTLS {
if opt.EtcdUseTLS {
cfg := &tls.Config{
RootCAs: capool,
InsecureSkipVerify: this.EtcdSkipTLSVerify,
InsecureSkipVerify: opt.EtcdSkipTLSVerify,
}
if certLoaded {
cfg.Certificates = []tls.Certificate{cert}
@@ -224,8 +228,24 @@ func (this *EtcdOptions) GetEtcdTLSConfig() (*tls.Config, error) {
return nil, nil
}
func (this *DBOptions) GetDBConnection() (dialect, connstr string, err error) {
return utils.TransSQLAchemyURL(this.SqlConnection)
func (opt *DBOptions) GetDBConnection() (string, string, error) {
if strings.HasPrefix(opt.SqlConnection, "mysql") {
return utils.TransSQLAchemyURL(opt.SqlConnection)
} else {
pos := strings.Index(opt.SqlConnection, "://")
if pos > 0 {
return opt.SqlConnection[:pos], opt.SqlConnection[pos+3:], nil
} else {
return "", "", httperrors.ErrNotSupported
}
}
}
func (opt *DBOptions) GetClickhouseConnStr() (string, string, error) {
if len(opt.Clickhouse) == 0 {
return "", "", errors.ErrNotFound
}
return "clickhouse", opt.Clickhouse, nil
}
func ParseOptions(optStruct interface{}, args []string, configFileName string, serviceType string) {
pkg/cloudevent/service/service.go
+3 -3
View File
@@ -18,9 +18,8 @@ import (
"os"
"time"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/cloudevent"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -49,9 +48,10 @@ func StartService() {
baseOpts := &opts.BaseOptions
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
if !opts.IsSlaveNode {
pkg/cloudid/service/service.go
+5 -1
View File
@@ -19,6 +19,7 @@ import (
"time"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/cloudid"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -47,9 +48,12 @@ func StartService() {
common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
err := saml.InitSAML(app, api.SAML_IDP_PREFIX)
pkg/cloudnet/service/service.go
+5 -3
View File
@@ -17,9 +17,8 @@ package service
import (
"os"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
"yunion.io/x/onecloud/pkg/cloudcommon"
common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
@@ -42,9 +41,12 @@ func StartService() {
baseOpts := &opts.BaseOptions
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
common_app.ServeForever(app, baseOpts)
pkg/cloudproxy/service/service.go
+5 -2
View File
@@ -15,7 +15,7 @@
package service
import (
_ "github.com/go-sql-driver/mysql"
_ "yunion.io/x/sqlchemy/backends"
"yunion.io/x/onecloud/pkg/cloudcommon"
common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
@@ -32,9 +32,12 @@ func StartService() {
)
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
common_app.ServeForever(app, baseOpts)
pkg/compute/models/billingresource.go
+1 -1
View File
@@ -254,7 +254,7 @@ func CheckBillingResourceExpireAt(ctx context.Context, userCred mcclient.TokenCr
bq := BillingResourceCheckManager.Query("resource_id").Equals("resource_type", manager.Keyword()).Equals("advance_days", advanceDay).SubQuery()
q = q.LeftJoin(bq, sqlchemy.Equals(q.Field("id"), bq.Field("resource_id")))
q = q.IsNull("resource_id")
q = q.Filter(sqlchemy.IsNull(bq.Field("resource_id")))
vp := reflect.New(v.Type())
vp.Elem().Set(v)
pkg/compute/models/cloudaccounts.go
+8 -6
View File
@@ -1867,12 +1867,14 @@ func (account *SCloudaccount) markAllProvidersDicconnected(ctx context.Context,
}
func (account *SCloudaccount) markAccountConnected(ctx context.Context, userCred mcclient.TokenCredential) error {
_, err := db.UpdateWithLock(ctx, account, func() error {
account.ErrorCount = 0
return nil
})
if err != nil {
return err
if account.ErrorCount != 0 {
_, err := db.UpdateWithLock(ctx, account, func() error {
account.ErrorCount = 0
return nil
})
if err != nil {
return err
}
}
return account.SetStatus(userCred, api.CLOUD_PROVIDER_CONNECTED, "")
}
pkg/compute/models/cloudregions.go
+1 -1
View File
@@ -522,7 +522,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
return nil
})
if err != nil {
if err != nil && errors.Cause(err) != sqlchemy.ErrNoDataToUpdate {
log.Errorf("syncWithCloudRegion %s", err)
return err
}
pkg/compute/models/purge.go
+11 -7
View File
@@ -390,14 +390,18 @@ func (lb *SLoadbalancer) purge(ctx context.Context, userCred mcclient.TokenCrede
lb.DeletePreventionOff(lb, userCred)
_, err := db.UpdateWithLock(ctx, lb, func() error {
//避免 purge backendgroups 时循环依赖
lb.BackendGroupId = ""
return nil
})
var err error
if err != nil {
return fmt.Errorf("loadbalancer %s(%s): clear up backend group error: %v", lb.Name, lb.Id, err)
if lb.BackendGroupId != "" {
_, err = db.UpdateWithLock(ctx, lb, func() error {
//避免 purge backendgroups 时循环依赖
lb.BackendGroupId = ""
return nil
})
if err != nil {
return fmt.Errorf("loadbalancer %s(%s): clear up backend group error: %v", lb.Name, lb.Id, err)
}
}
err = lb.detachAllNetworks(ctx, userCred)
pkg/compute/service/service.go
+5 -3
View File
@@ -21,10 +21,9 @@ import (
"path/filepath"
"time"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -77,8 +76,11 @@ func StartService() {
app := app_common.InitApp(baseOpts, true)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
options.InitNameSyncResources()
pkg/devtool/service/service.go
+3 -3
View File
@@ -17,9 +17,8 @@ package service
import (
"os"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/devtool"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -51,8 +50,9 @@ func StartService() {
}
app := app_common.InitApp(&opts.BaseOptions, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
models.InitializeCronjobs()
pkg/image/service/service.go
+5 -3
View File
@@ -21,9 +21,8 @@ import (
"strings"
"time"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/image"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -94,9 +93,12 @@ func StartService() {
}
app := app_common.InitApp(baseOpts, true)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
pkg/keystone/models/domains.go
+1
View File
@@ -142,6 +142,7 @@ func (manager *SDomainManager) FetchDomain(domainId string, domainName string) (
if len(domainId) == 0 && len(domainName) == 0 {
domainId = api.DEFAULT_DOMAIN_ID
}
log.Debugf("FetchDomain id %s name %s", domainId, domainName)
if len(domainId) > 0 {
return manager.FetchDomainById(domainId)
} else {
pkg/keystone/service/service.go
+5 -2
View File
@@ -18,7 +18,7 @@ import (
"os"
"time"
_ "github.com/go-sql-driver/mysql"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/identity"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -72,8 +72,11 @@ func StartService() {
app := app_common.InitApp(&opts.BaseOptions, true)
cloudcommon.InitDB(&opts.DBOptions)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, &opts.DBOptions, models.InitDB)
db.EnsureAppSyncDB(app, &opts.DBOptions, models.InitDB)
app_common.InitBaseAuth(&opts.BaseOptions)
pkg/logger/service/service.go
+5 -3
View File
@@ -17,9 +17,8 @@ package service
import (
"os"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/logger"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -47,9 +46,12 @@ func StartService() {
})
app := app_common.InitApp(baseOpts, true)
cloudcommon.InitDB(dbOpts)
initHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
// models.StartNotifyToWebsocketWorker()
pkg/mcclient/modulebase/stats.go
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package modulebase
import (
"io/ioutil"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/mcclient"
)
// path: stats, db_stats, worker_stats
func GetStats(s *mcclient.ClientSession, path string, serviceType string) (jsonutils.JSONObject, error) {
man := NewBaseManager(serviceType, "", "", nil, nil)
resp, err := man.rawBaseUrlRequest(s, "GET", "/"+path, nil, nil)
if err != nil {
return nil, errors.Wrap(err, "man.rawBaseUrlRequest")
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "ioutil.ReadAll")
}
return jsonutils.Parse(body)
}
pkg/monitor/service/service.go
+5 -2
View File
@@ -19,10 +19,10 @@ import (
"os"
"time"
_ "github.com/go-sql-driver/mysql"
"golang.org/x/sync/errgroup"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
"yunion.io/x/onecloud/pkg/cloudcommon"
common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
@@ -56,9 +56,12 @@ func StartService() {
baseOpts := &opts.BaseOptions
app := common_app.InitApp(baseOpts, false)
cloudcommon.InitDB(dbOpts)
InitHandlers(app)
db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
go startServices()
pkg/notify/service/service.go
+5 -3
View File
@@ -18,9 +18,8 @@ import (
"os"
"time"
_ "github.com/go-sql-driver/mysql"
"yunion.io/x/log"
_ "yunion.io/x/sqlchemy/backends"
api "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/cloudcommon"
@@ -52,10 +51,13 @@ func StartService() {
// init handler
applicaion := app.InitApp(baseOpts, true)
cloudcommon.InitDB(dbOpts)
InitHandlers(applicaion)
// init database
db.EnsureAppInitSyncDB(applicaion, dbOpts, models.InitDB)
db.EnsureAppSyncDB(applicaion, dbOpts, models.InitDB)
defer cloudcommon.CloseDB()
err := models.ReceiverManager.StartWatchUserInKeystone()
pkg/scheduledtask/models/scheduled_tasks.go
+1 -1
View File
@@ -551,7 +551,7 @@ func (stm *SScheduledTaskManager) Timer(ctx context.Context, userCred mcclient.T
if timerQueue == nil {
timerQueue = make(chan struct{}, sop.Options.ScheduledTaskQueueSize)
}
log.Infof("queueSize: %s", sop.Options.ScheduledTaskQueueSize)
log.Infof("queueSize: %d", sop.Options.ScheduledTaskQueueSize)
// 60 is for fault tolerance
interval := 60 + 30
timeScope := stm.timeScope(time.Now(), time.Duration(interval)*time.Second)
pkg/scheduledtask/service/service.go
+4 -1
View File
@@ -42,9 +42,12 @@ func StartService() {
})
applicaion := app.InitApp(baseOpts, true)
cloudcommon.InitDB(dbOpts)
InitHandlers(applicaion)
db.EnsureAppInitSyncDB(applicaion, dbOpts, nil)
db.EnsureAppSyncDB(applicaion, dbOpts, nil)
defer cloudcommon.CloseDB()
cron := cronman.InitCronJobManager(true, 4)
pkg/util/splitable/insert.go
+1 -1
View File
@@ -67,7 +67,7 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error {
if lastMeta.StartDate.IsZero() {
indexCol := t.tableSpec.ColumnSpec(t.indexField)
_, err = t.metaSpec.Update(&lastMeta, func() error {
lastMeta.Start = indexCol.(*sqlchemy.SIntegerColumn).AutoIncrementOffset
lastMeta.Start = indexCol.AutoIncrementOffset()
lastMeta.StartDate = lastDate
return nil
})
pkg/util/splitable/splitable.go
+11 -9
View File
@@ -47,6 +47,10 @@ func (t *SSplitTableSpec) Name() string {
return t.tableName
}
func (t *SSplitTableSpec) Database() *sqlchemy.SDatabase {
return t.metaSpec.Database()
}
func (t *SSplitTableSpec) Columns() []sqlchemy.IColumnSpec {
return t.tableSpec.Columns()
}
@@ -116,29 +120,27 @@ func (t *SSplitTableSpec) Fetch(dt interface{}) error {
return sql.ErrNoRows
}
func NewSplitTableSpec(s interface{}, name string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) (*SSplitTableSpec, error) {
spec := sqlchemy.NewTableSpecFromStruct(s, name)
indexCol := spec.ColumnSpec(indexField)
func NewSplitTableSpec(s interface{}, name string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) (*SSplitTableSpec, error) {
spec := sqlchemy.NewTableSpecFromStructWithDBName(s, name, dbName)
/*indexCol := spec.ColumnSpec(indexField)
if indexCol == nil {
return nil, errors.Wrapf(errors.ErrNotFound, "indexField %s not found", indexField)
}
if !indexCol.IsPrimary() {
return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not primary", indexField)
}
if intCol, ok := indexCol.(*sqlchemy.SIntegerColumn); !ok {
return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not integer", indexField)
} else if !intCol.IsAutoIncrement {
if !indexCol.IsAutoIncrement() {
return nil, errors.Wrapf(errors.ErrInvalidStatus, "indexField %s not auto_increment", indexField)
}
dateCol := spec.ColumnSpec(dateField)
if dateCol == nil {
return nil, errors.Wrapf(errors.ErrNotFound, "dateField %s not found", dateField)
}
if _, ok := dateCol.(*sqlchemy.SDateTimeColumn); !ok {
if !dateCol.IsDateTime() {
return nil, errors.Wrapf(errors.ErrInvalidStatus, "dateField %s not datetime column", dateField)
}
}*/
metaSpec := sqlchemy.NewTableSpecFromStruct(&STableMetadata{}, fmt.Sprintf("%s_metadata", name))
metaSpec := sqlchemy.NewTableSpecFromStructWithDBName(&STableMetadata{}, fmt.Sprintf("%s_metadata", name), dbName)
sts := &SSplitTableSpec{
indexField: indexField,
pkg/util/splitable/sync.go
+1 -1
View File
@@ -130,7 +130,7 @@ func (spec *SSplitTableSpec) SyncSQL() []string {
now := time.Now()
meta := STableMetadata{
Table: fmt.Sprintf("%s_%d", spec.tableName, now.Unix()),
Start: indexCol.(*sqlchemy.SIntegerColumn).AutoIncrementOffset,
Start: indexCol.AutoIncrementOffset(),
}
// insert the first meta
sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now))
vendor/github.com/ClickHouse/clickhouse-go/.gitignore
Generated
+28
View File
@@ -0,0 +1,28 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.out
*.exe
*.test
*.prof
coverage.txt
.idea/**
vendor/github.com/ClickHouse/clickhouse-go/.travis.yml
Generated Vendored
+20
View File
@@ -0,0 +1,20 @@
sudo: required
language: go
go:
- 1.15.x
- 1.16.x
go_import_path: github.com/ClickHouse/clickhouse-go
services:
- docker
install:
- export GO111MODULE="on"
- go mod vendor
before_install:
- docker --version
- docker-compose --version
- docker-compose up -d
script:
- ./go.test.sh
after_success:
- bash <(curl -s https://codecov.io/bash)
vendor/github.com/ClickHouse/clickhouse-go/CONTRIBUTING.md
Generated Vendored
+10
View File
@@ -0,0 +1,10 @@
# Contributing notes
## Local setup
The easiest way to run tests is to use Docker Compose:
```
docker-compose up
make
```
vendor/github.com/ClickHouse/clickhouse-go/LICENSE
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017-2020 Kirill Shvakov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
vendor/github.com/ClickHouse/clickhouse-go/Makefile
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
test:
go install -race -v
go test -i -v
go test -race -timeout 30s -v .
coverage:
go test -coverprofile=coverage.out -v .
go tool cover -html=coverage.out
vendor/github.com/ClickHouse/clickhouse-go/README.md
Generated Vendored
+302
View File
@@ -0,0 +1,302 @@
# ClickHouse [![Build Status](https://travis-ci.org/ClickHouse/clickhouse-go.svg?branch=master)](https://travis-ci.org/ClickHouse/clickhouse-go) [![Go Report Card](https://goreportcard.com/badge/github.com/ClickHouse/clickhouse-go)](https://goreportcard.com/report/github.com/ClickHouse/clickhouse-go) [![codecov](https://codecov.io/gh/ClickHouse/clickhouse-go/branch/master/graph/badge.svg)](https://codecov.io/gh/ClickHouse/clickhouse-go)
Golang SQL database driver for [Yandex ClickHouse](https://clickhouse.yandex/)
## Key features
* Uses native ClickHouse tcp client-server protocol
* Compatibility with `database/sql`
* Round Robin load-balancing
* Bulk write support : `begin->prepare->(in loop exec)->commit`
* LZ4 compression support (default to use pure go lz4, switch to use cgo lz4 by turn clz4 build tags on)
* External Tables support
## DSN
* username/password - auth credentials
* database - select the current default database
* read_timeout/write_timeout - timeout in second
* no_delay - disable/enable the Nagle Algorithm for tcp socket (default is 'true' - disable)
* alt_hosts - comma separated list of single address host for load-balancing
* connection_open_strategy - random/in_order (default random).
* random - choose random server from set
* in_order - first live server is choosen in specified order
* time_random - choose random(based on current time) server from set. This option differs from `random` in that randomness is based on current time rather than on amount of previous connections.
* block_size - maximum rows in block (default is 1000000). If the rows are larger then the data will be split into several blocks to send them to the server. If one block was sent to the server, the data will be persisted on the server disk, we can't rollback the transaction. So always keep in mind that the batch size no larger than the block_size if you want atomic batch insert.
* pool_size - maximum amount of preallocated byte chunks used in queries (default is 100). Decrease this if you experience memory problems at the expense of more GC pressure and vice versa.
* debug - enable debug output (boolean value)
* compress - enable lz4 compression (integer value, default is '0')
SSL/TLS parameters:
* secure - establish secure connection (default is false)
* skip_verify - skip certificate verification (default is false)
* tls_config - name of a TLS config with client certificates, registered using `clickhouse.RegisterTLSConfig()`; implies secure to be true, unless explicitly specified
example:
```
tcp://host1:9000?username=user&password=qwerty&database=clicks&read_timeout=10&write_timeout=20&alt_hosts=host2:9000,host3:9000
```
## Supported data types
* UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
* Float32, Float64
* String
* FixedString(N)
* Date
* DateTime
* IPv4
* IPv6
* Enum
* UUID
* Nullable(T)
* [Array(T)](https://clickhouse.yandex/reference_en.html#Array(T)) [godoc](https://godoc.org/github.com/ClickHouse/clickhouse-go#Array)
* Array(Nullable(T))
* Tuple(...T)
## TODO
* Support other compression methods(zstd ...)
## Install
```
go get -u github.com/ClickHouse/clickhouse-go
```
## Example
```go
package main
import (
"database/sql"
"fmt"
"log"
"time"
"github.com/ClickHouse/clickhouse-go"
)
func main() {
connect, err := sql.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true")
if err != nil {
log.Fatal(err)
}
if err := connect.Ping(); err != nil {
if exception, ok := err.(*clickhouse.Exception); ok {
fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
} else {
fmt.Println(err)
}
return
}
_, err = connect.Exec(`
CREATE TABLE IF NOT EXISTS example (
country_code FixedString(2),
os_id UInt8,
browser_id UInt8,
categories Array(Int16),
action_day Date,
action_time DateTime
) engine=Memory
`)
if err != nil {
log.Fatal(err)
}
var (
tx, _ = connect.Begin()
stmt, _ = tx.Prepare("INSERT INTO example (country_code, os_id, browser_id, categories, action_day, action_time) VALUES (?, ?, ?, ?, ?, ?)")
)
defer stmt.Close()
for i := 0; i < 100; i++ {
if _, err := stmt.Exec(
"RU",
10+i,
100+i,
clickhouse.Array([]int16{1, 2, 3}),
time.Now(),
time.Now(),
); err != nil {
log.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
rows, err := connect.Query("SELECT country_code, os_id, browser_id, categories, action_day, action_time FROM example")
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var (
country string
os, browser uint8
categories []int16
actionDay, actionTime time.Time
)
if err := rows.Scan(&country, &os, &browser, &categories, &actionDay, &actionTime); err != nil {
log.Fatal(err)
}
log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s", country, os, browser, categories, actionDay, actionTime)
}
if err := rows.Err(); err != nil {
log.Fatal(err)
}
if _, err := connect.Exec("DROP TABLE example"); err != nil {
log.Fatal(err)
}
}
```
Use [sqlx](https://github.com/jmoiron/sqlx)
```go
package main
import (
"log"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/ClickHouse/clickhouse-go"
)
func main() {
connect, err := sqlx.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true")
if err != nil {
log.Fatal(err)
}
var items []struct {
CountryCode string `db:"country_code"`
OsID uint8 `db:"os_id"`
BrowserID uint8 `db:"browser_id"`
Categories []int16 `db:"categories"`
ActionTime time.Time `db:"action_time"`
}
if err := connect.Select(&items, "SELECT country_code, os_id, browser_id, categories, action_time FROM example"); err != nil {
log.Fatal(err)
}
for _, item := range items {
log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_time: %s", item.CountryCode, item.OsID, item.BrowserID, item.Categories, item.ActionTime)
}
}
```
External tables support
```go
package main
import (
"database/sql"
"database/sql/driver"
"fmt"
"github.com/ClickHouse/clickhouse-go/lib/column"
"log"
"time"
"github.com/ClickHouse/clickhouse-go"
)
func main() {
connect, err := sql.Open("clickhouse", "tcp://127.0.0.1:9000?debug=true")
if err != nil {
log.Fatal(err)
}
if err := connect.Ping(); err != nil {
if exception, ok := err.(*clickhouse.Exception); ok {
fmt.Printf("[%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
} else {
fmt.Println(err)
}
return
}
_, err = connect.Exec(`
CREATE TABLE IF NOT EXISTS example (
country_code FixedString(2),
os_id UInt8,
browser_id UInt8,
categories Array(Int16),
action_day Date,
action_time DateTime
) engine=Memory
`)
if err != nil {
log.Fatal(err)
}
var (
tx, _ = connect.Begin()
stmt, _ = tx.Prepare("INSERT INTO example (country_code, os_id, browser_id, categories, action_day, action_time) VALUES (?, ?, ?, ?, ?, ?)")
)
defer stmt.Close()
for i := 0; i < 100; i++ {
if _, err := stmt.Exec(
"RU",
10+i,
100+i,
clickhouse.Array([]int16{1, 2, 3}),
time.Now(),
time.Now(),
); err != nil {
log.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
col, err := column.Factory("country_code", "String", nil)
if err != nil {
log.Fatal(err)
}
countriesExternalTable := clickhouse.ExternalTable{
Name: "countries",
Values: [][]driver.Value{
{"RU"},
},
Columns: []column.Column{col},
}
rows, err := connect.Query("SELECT country_code, os_id, browser_id, categories, action_day, action_time "+
"FROM example WHERE country_code IN ?", countriesExternalTable)
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var (
country string
os, browser uint8
categories []int16
actionDay, actionTime time.Time
)
if err := rows.Scan(&country, &os, &browser, &categories, &actionDay, &actionTime); err != nil {
log.Fatal(err)
}
log.Printf("country: %s, os: %d, browser: %d, categories: %v, action_day: %s, action_time: %s", country, os, browser, categories, actionDay, actionTime)
}
if err := rows.Err(); err != nil {
log.Fatal(err)
}
if _, err := connect.Exec("DROP TABLE example"); err != nil {
log.Fatal(err)
}
}
```
vendor/github.com/ClickHouse/clickhouse-go/array.go
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
package clickhouse
import (
"time"
)
func Array(v interface{}) interface{} {
return v
}
func ArrayFixedString(len int, v interface{}) interface{} {
return v
}
func ArrayDate(v []time.Time) interface{} {
return v
}
func ArrayDateTime(v []time.Time) interface{} {
return v
}
vendor/github.com/ClickHouse/clickhouse-go/bootstrap.go
Generated Vendored
+245
View File
@@ -0,0 +1,245 @@
package clickhouse
import (
"bufio"
"database/sql"
"database/sql/driver"
"fmt"
"io"
"log"
"net/url"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
"github.com/ClickHouse/clickhouse-go/lib/data"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
)
const (
// DefaultDatabase when connecting to ClickHouse
DefaultDatabase = "default"
// DefaultUsername when connecting to ClickHouse
DefaultUsername = "default"
// DefaultConnTimeout when connecting to ClickHouse
DefaultConnTimeout = 5 * time.Second
// DefaultReadTimeout when reading query results
DefaultReadTimeout = time.Minute
// DefaultWriteTimeout when sending queries
DefaultWriteTimeout = time.Minute
)
var (
unixtime int64
logOutput io.Writer = os.Stdout
hostname, _ = os.Hostname()
poolInit sync.Once
)
func init() {
sql.Register("clickhouse", &bootstrap{})
go func() {
for tick := time.Tick(time.Second); ; {
select {
case <-tick:
atomic.AddInt64(&unixtime, int64(time.Second))
}
}
}()
}
func now() time.Time {
return time.Unix(0, atomic.LoadInt64(&unixtime))
}
type bootstrap struct{}
func (d *bootstrap) Open(dsn string) (driver.Conn, error) {
return Open(dsn)
}
// SetLogOutput allows to change output of the default logger
func SetLogOutput(output io.Writer) {
logOutput = output
}
// Open the connection
func Open(dsn string) (driver.Conn, error) {
clickhouse, err := open(dsn)
if err != nil {
return nil, err
}
return clickhouse, err
}
func open(dsn string) (*clickhouse, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, err
}
var (
hosts = []string{url.Host}
query = url.Query()
secure = false
skipVerify = false
tlsConfigName = query.Get("tls_config")
noDelay = true
compress = false
database = query.Get("database")
username = query.Get("username")
password = query.Get("password")
blockSize = 1000000
connTimeout = DefaultConnTimeout
readTimeout = DefaultReadTimeout
writeTimeout = DefaultWriteTimeout
connOpenStrategy = connOpenRandom
)
if len(database) == 0 {
database = DefaultDatabase
}
if len(username) == 0 {
username = DefaultUsername
}
if v, err := strconv.ParseBool(query.Get("no_delay")); err == nil {
noDelay = v
}
tlsConfig := getTLSConfigClone(tlsConfigName)
if tlsConfigName != "" && tlsConfig == nil {
return nil, fmt.Errorf("invalid tls_config - no config registered under name %s", tlsConfigName)
}
secure = tlsConfig != nil
if v, err := strconv.ParseBool(query.Get("secure")); err == nil {
secure = v
}
if v, err := strconv.ParseBool(query.Get("skip_verify")); err == nil {
skipVerify = v
}
if duration, err := strconv.ParseFloat(query.Get("timeout"), 64); err == nil {
connTimeout = time.Duration(duration * float64(time.Second))
}
if duration, err := strconv.ParseFloat(query.Get("read_timeout"), 64); err == nil {
readTimeout = time.Duration(duration * float64(time.Second))
}
if duration, err := strconv.ParseFloat(query.Get("write_timeout"), 64); err == nil {
writeTimeout = time.Duration(duration * float64(time.Second))
}
if size, err := strconv.ParseInt(query.Get("block_size"), 10, 64); err == nil {
blockSize = int(size)
}
if altHosts := strings.Split(query.Get("alt_hosts"), ","); len(altHosts) != 0 {
for _, host := range altHosts {
if len(host) != 0 {
hosts = append(hosts, host)
}
}
}
switch query.Get("connection_open_strategy") {
case "random":
connOpenStrategy = connOpenRandom
case "in_order":
connOpenStrategy = connOpenInOrder
case "time_random":
connOpenStrategy = connOpenTimeRandom
}
settings, err := makeQuerySettings(query)
if err != nil {
return nil, err
}
if v, err := strconv.ParseBool(query.Get("compress")); err == nil {
compress = v
}
var (
ch = clickhouse{
logf: func(string, ...interface{}) {},
settings: settings,
compress: compress,
blockSize: blockSize,
ServerInfo: data.ServerInfo{
Timezone: time.Local,
},
}
logger = log.New(logOutput, "[clickhouse]", 0)
)
if debug, err := strconv.ParseBool(url.Query().Get("debug")); err == nil && debug {
ch.logf = logger.Printf
}
ch.logf("host(s)=%s, database=%s, username=%s",
strings.Join(hosts, ", "),
database,
username,
)
options := connOptions{
secure: secure,
tlsConfig: tlsConfig,
skipVerify: skipVerify,
hosts: hosts,
connTimeout: connTimeout,
readTimeout: readTimeout,
writeTimeout: writeTimeout,
noDelay: noDelay,
openStrategy: connOpenStrategy,
logf: ch.logf,
}
if ch.conn, err = dial(options); err != nil {
return nil, err
}
logger.SetPrefix(fmt.Sprintf("[clickhouse][connect=%d]", ch.conn.ident))
ch.buffer = bufio.NewWriter(ch.conn)
ch.decoder = binary.NewDecoderWithCompress(ch.conn)
ch.encoder = binary.NewEncoderWithCompress(ch.buffer)
if err := ch.hello(database, username, password); err != nil {
ch.conn.Close()
return nil, err
}
return &ch, nil
}
func (ch *clickhouse) hello(database, username, password string) error {
ch.logf("[hello] -> %s", ch.ClientInfo)
{
ch.encoder.Uvarint(protocol.ClientHello)
if err := ch.ClientInfo.Write(ch.encoder); err != nil {
return err
}
{
ch.encoder.String(database)
ch.encoder.String(username)
ch.encoder.String(password)
}
if err := ch.encoder.Flush(); err != nil {
return err
}
}
{
packet, err := ch.decoder.Uvarint()
if err != nil {
return err
}
switch packet {
case protocol.ServerException:
return ch.exception()
case protocol.ServerHello:
if err := ch.ServerInfo.Read(ch.decoder); err != nil {
return err
}
case protocol.ServerEndOfStream:
ch.logf("[bootstrap] <- end of stream")
return nil
default:
return fmt.Errorf("[hello] unexpected packet [%d] from server", packet)
}
}
ch.logf("[hello] <- %s", ch.ServerInfo)
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse.go
Generated Vendored
+341
View File
@@ -0,0 +1,341 @@
package clickhouse
import (
"bufio"
"context"
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"net"
"reflect"
"regexp"
"sync"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
"github.com/ClickHouse/clickhouse-go/lib/column"
"github.com/ClickHouse/clickhouse-go/lib/data"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
"github.com/ClickHouse/clickhouse-go/lib/types"
)
type (
Date = types.Date
DateTime = types.DateTime
UUID = types.UUID
)
type ExternalTable struct {
Name string
Values [][]driver.Value
Columns []column.Column
}
var (
ErrInsertInNotBatchMode = errors.New("insert statement supported only in the batch mode (use begin/commit)")
ErrLimitDataRequestInTx = errors.New("data request has already been prepared in transaction")
)
var (
splitInsertRe = regexp.MustCompile(`(?i)\sVALUES\s*\(`)
)
type logger func(format string, v ...interface{})
type clickhouse struct {
sync.Mutex
data.ServerInfo
data.ClientInfo
logf logger
conn *connect
block *data.Block
buffer *bufio.Writer
decoder *binary.Decoder
encoder *binary.Encoder
settings *querySettings
compress bool
blockSize int
inTransaction bool
}
func (ch *clickhouse) Prepare(query string) (driver.Stmt, error) {
return ch.prepareContext(context.Background(), query)
}
func (ch *clickhouse) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
return ch.prepareContext(ctx, query)
}
func (ch *clickhouse) prepareContext(ctx context.Context, query string) (driver.Stmt, error) {
ch.logf("[prepare] %s", query)
switch {
case ch.conn.closed:
return nil, driver.ErrBadConn
case ch.block != nil:
return nil, ErrLimitDataRequestInTx
case isInsert(query):
if !ch.inTransaction {
return nil, ErrInsertInNotBatchMode
}
return ch.insert(ctx, query)
}
return &stmt{
ch: ch,
query: query,
numInput: numInput(query),
}, nil
}
func (ch *clickhouse) insert(ctx context.Context, query string) (_ driver.Stmt, err error) {
if err := ch.sendQuery(ctx, splitInsertRe.Split(query, -1)[0]+" VALUES ", nil); err != nil {
return nil, err
}
if ch.block, err = ch.readMeta(); err != nil {
return nil, err
}
return &stmt{
ch: ch,
isInsert: true,
}, nil
}
func (ch *clickhouse) Begin() (driver.Tx, error) {
return ch.beginTx(context.Background(), txOptions{})
}
func (ch *clickhouse) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
return ch.beginTx(ctx, txOptions{
Isolation: int(opts.Isolation),
ReadOnly: opts.ReadOnly,
})
}
type txOptions struct {
Isolation int
ReadOnly bool
}
func (ch *clickhouse) beginTx(ctx context.Context, opts txOptions) (*clickhouse, error) {
ch.logf("[begin] tx=%t, data=%t", ch.inTransaction, ch.block != nil)
switch {
case ch.inTransaction:
return nil, sql.ErrTxDone
case ch.conn.closed:
return nil, driver.ErrBadConn
}
if finish := ch.watchCancel(ctx); finish != nil {
defer finish()
}
ch.block = nil
ch.inTransaction = true
return ch, nil
}
func (ch *clickhouse) Commit() error {
ch.logf("[commit] tx=%t, data=%t", ch.inTransaction, ch.block != nil)
defer func() {
if ch.block != nil {
ch.block.Reset()
ch.block = nil
}
ch.inTransaction = false
}()
switch {
case !ch.inTransaction:
return sql.ErrTxDone
case ch.conn.closed:
return driver.ErrBadConn
}
if ch.block != nil {
if err := ch.writeBlock(ch.block, ""); err != nil {
return err
}
// Send empty block as marker of end of data.
if err := ch.writeBlock(&data.Block{}, ""); err != nil {
return err
}
if err := ch.encoder.Flush(); err != nil {
return err
}
return ch.process()
}
return nil
}
func (ch *clickhouse) Rollback() error {
ch.logf("[rollback] tx=%t, data=%t", ch.inTransaction, ch.block != nil)
if !ch.inTransaction {
return sql.ErrTxDone
}
if ch.block != nil {
ch.block.Reset()
}
ch.block = nil
ch.buffer = nil
ch.inTransaction = false
return ch.conn.Close()
}
func (ch *clickhouse) CheckNamedValue(nv *driver.NamedValue) error {
switch nv.Value.(type) {
case ExternalTable, column.IP, column.UUID:
return nil
case nil, []byte, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64, string, time.Time:
return nil
}
switch v := nv.Value.(type) {
case
[]int, []int8, []int16, []int32, []int64,
[]uint, []uint8, []uint16, []uint32, []uint64,
[]float32, []float64,
[]string:
return nil
case net.IP, *net.IP:
return nil
case driver.Valuer:
value, err := v.Value()
if err != nil {
return err
}
nv.Value = value
default:
switch value := reflect.ValueOf(nv.Value); value.Kind() {
case reflect.Slice:
return nil
case reflect.Bool:
nv.Value = uint8(0)
if value.Bool() {
nv.Value = uint8(1)
}
case reflect.Int8:
nv.Value = int8(value.Int())
case reflect.Int16:
nv.Value = int16(value.Int())
case reflect.Int32:
nv.Value = int32(value.Int())
case reflect.Int64:
nv.Value = value.Int()
case reflect.Uint8:
nv.Value = uint8(value.Uint())
case reflect.Uint16:
nv.Value = uint16(value.Uint())
case reflect.Uint32:
nv.Value = uint32(value.Uint())
case reflect.Uint64:
nv.Value = uint64(value.Uint())
case reflect.Float32:
nv.Value = float32(value.Float())
case reflect.Float64:
nv.Value = float64(value.Float())
case reflect.String:
nv.Value = value.String()
}
}
return nil
}
func (ch *clickhouse) Close() error {
ch.block = nil
return ch.conn.Close()
}
func (ch *clickhouse) process() error {
packet, err := ch.decoder.Uvarint()
if err != nil {
return err
}
for {
switch packet {
case protocol.ServerPong:
ch.logf("[process] <- pong")
return nil
case protocol.ServerException:
ch.logf("[process] <- exception")
return ch.exception()
case protocol.ServerProgress:
progress, err := ch.progress()
if err != nil {
return err
}
ch.logf("[process] <- progress: rows=%d, bytes=%d, total rows=%d",
progress.rows,
progress.bytes,
progress.totalRows,
)
case protocol.ServerProfileInfo:
profileInfo, err := ch.profileInfo()
if err != nil {
return err
}
ch.logf("[process] <- profiling: rows=%d, bytes=%d, blocks=%d", profileInfo.rows, profileInfo.bytes, profileInfo.blocks)
case protocol.ServerData:
block, err := ch.readBlock()
if err != nil {
return err
}
ch.logf("[process] <- data: packet=%d, columns=%d, rows=%d", packet, block.NumColumns, block.NumRows)
case protocol.ServerEndOfStream:
ch.logf("[process] <- end of stream")
return nil
default:
ch.conn.Close()
return fmt.Errorf("[process] unexpected packet [%d] from server", packet)
}
if packet, err = ch.decoder.Uvarint(); err != nil {
return err
}
}
}
func (ch *clickhouse) cancel() error {
ch.logf("[cancel request]")
// even if we fail to write the cancel, we still need to close
err := ch.encoder.Uvarint(protocol.ClientCancel)
if err == nil {
err = ch.encoder.Flush()
}
// return the close error if there was one, otherwise return the write error
if cerr := ch.conn.Close(); cerr != nil {
return cerr
}
return err
}
func (ch *clickhouse) watchCancel(ctx context.Context) func() {
if done := ctx.Done(); done != nil {
finished := make(chan struct{})
go func() {
select {
case <-done:
ch.cancel()
finished <- struct{}{}
ch.logf("[cancel] <- done")
case <-finished:
ch.logf("[cancel] <- finished")
}
}()
return func() {
select {
case <-finished:
case finished <- struct{}{}:
}
}
}
return func() {}
}
func (ch *clickhouse) ExecContext(ctx context.Context, query string,
args []driver.NamedValue) (driver.Result, error) {
finish := ch.watchCancel(ctx)
defer finish()
stmt, err := ch.PrepareContext(ctx, query)
if err != nil {
return nil, err
}
dargs := make([]driver.Value, len(args))
for i, nv := range args {
dargs[i] = nv.Value
}
return stmt.Exec(dargs)
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_exception.go
Generated Vendored
+46
View File
@@ -0,0 +1,46 @@
package clickhouse
import (
"fmt"
"strings"
)
type Exception struct {
Code int32
Name string
Message string
StackTrace string
nested error
}
func (e *Exception) Error() string {
return fmt.Sprintf("code: %d, message: %s", e.Code, e.Message)
}
func (ch *clickhouse) exception() error {
var (
e Exception
err error
hasNested bool
)
if e.Code, err = ch.decoder.Int32(); err != nil {
return err
}
if e.Name, err = ch.decoder.String(); err != nil {
return err
}
if e.Message, err = ch.decoder.String(); err != nil {
return err
}
e.Message = strings.TrimSpace(strings.TrimPrefix(e.Message, e.Name+":"))
if e.StackTrace, err = ch.decoder.String(); err != nil {
return err
}
if hasNested, err = ch.decoder.Bool(); err != nil {
return err
}
if hasNested {
e.nested = ch.exception()
}
return &e
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_ping.go
Generated Vendored
+28
View File
@@ -0,0 +1,28 @@
package clickhouse
import (
"context"
"database/sql/driver"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
)
func (ch *clickhouse) Ping(ctx context.Context) error {
return ch.ping(ctx)
}
func (ch *clickhouse) ping(ctx context.Context) error {
if ch.conn.closed {
return driver.ErrBadConn
}
ch.logf("-> ping")
finish := ch.watchCancel(ctx)
defer finish()
if err := ch.encoder.Uvarint(protocol.ClientPing); err != nil {
return err
}
if err := ch.encoder.Flush(); err != nil {
return err
}
return ch.process()
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_profile_info.go
Generated Vendored
+37
View File
@@ -0,0 +1,37 @@
package clickhouse
type profileInfo struct {
rows uint64
bytes uint64
blocks uint64
appliedLimit bool
rowsBeforeLimit uint64
calculatedRowsBeforeLimit bool
}
func (ch *clickhouse) profileInfo() (*profileInfo, error) {
var (
p profileInfo
err error
)
if p.rows, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.blocks, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.bytes, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.appliedLimit, err = ch.decoder.Bool(); err != nil {
return nil, err
}
if p.rowsBeforeLimit, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.calculatedRowsBeforeLimit, err = ch.decoder.Bool(); err != nil {
return nil, err
}
return &p, nil
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_progress.go
Generated Vendored
+26
View File
@@ -0,0 +1,26 @@
package clickhouse
type progress struct {
rows uint64
bytes uint64
totalRows uint64
}
func (ch *clickhouse) progress() (*progress, error) {
var (
p progress
err error
)
if p.rows, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.bytes, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
if p.totalRows, err = ch.decoder.Uvarint(); err != nil {
return nil, err
}
return &p, nil
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_block.go
Generated Vendored
+19
View File
@@ -0,0 +1,19 @@
package clickhouse
import (
"github.com/ClickHouse/clickhouse-go/lib/data"
)
func (ch *clickhouse) readBlock() (*data.Block, error) {
if _, err := ch.decoder.String(); err != nil { // temporary table
return nil, err
}
ch.decoder.SelectCompress(ch.compress)
var block data.Block
if err := block.Read(&ch.ServerInfo, ch.decoder); err != nil {
return nil, err
}
ch.decoder.SelectCompress(false)
return &block, nil
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_read_meta.go
Generated Vendored
+53
View File
@@ -0,0 +1,53 @@
package clickhouse
import (
"fmt"
"github.com/ClickHouse/clickhouse-go/lib/data"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
)
func (ch *clickhouse) readMeta() (*data.Block, error) {
for {
packet, err := ch.decoder.Uvarint()
if err != nil {
return nil, err
}
switch packet {
case protocol.ServerException:
ch.logf("[read meta] <- exception")
return nil, ch.exception()
case protocol.ServerProgress:
progress, err := ch.progress()
if err != nil {
return nil, err
}
ch.logf("[read meta] <- progress: rows=%d, bytes=%d, total rows=%d",
progress.rows,
progress.bytes,
progress.totalRows,
)
case protocol.ServerProfileInfo:
profileInfo, err := ch.profileInfo()
if err != nil {
return nil, err
}
ch.logf("[read meta] <- profiling: rows=%d, bytes=%d, blocks=%d", profileInfo.rows, profileInfo.bytes, profileInfo.blocks)
case protocol.ServerData:
block, err := ch.readBlock()
if err != nil {
return nil, err
}
ch.logf("[read meta] <- data: packet=%d, columns=%d, rows=%d", packet, block.NumColumns, block.NumRows)
return block, nil
case protocol.ServerEndOfStream:
_, err := ch.readBlock()
ch.logf("[process] <- end of stream")
return nil, err
default:
ch.conn.Close()
return nil, fmt.Errorf("[read meta] unexpected packet [%d] from server", packet)
}
}
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_external_data.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package clickhouse
import "github.com/ClickHouse/clickhouse-go/lib/data"
func (ch *clickhouse) sendExternalTables(externalTables []ExternalTable) error {
ch.logf("[send external tables] count %d", len(externalTables))
if externalTables == nil || len(externalTables) == 0 {
return nil
}
block := &data.Block{}
sentTables := make(map[string]bool, 0)
for _, externalTable := range externalTables {
if _, ok := sentTables[externalTable.Name]; ok {
continue
}
ch.logf("[send external table] name %s", externalTable.Name)
sentTables[externalTable.Name] = true
block.Columns = externalTable.Columns
block.NumColumns = uint64(len(externalTable.Columns))
for _, row := range externalTable.Values {
err := block.AppendRow(row)
if err != nil {
return err
}
}
if err := ch.writeBlock(block, externalTable.Name); err != nil {
return err
}
if err := ch.encoder.Flush(); err != nil {
return err
}
block.Reset()
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_send_query.go
Generated Vendored
+71
View File
@@ -0,0 +1,71 @@
package clickhouse
import (
"context"
"github.com/ClickHouse/clickhouse-go/lib/data"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
)
func (ch *clickhouse) sendQuery(ctx context.Context, query string, externalTables []ExternalTable) error {
ch.logf("[send query] %s", query)
if err := ch.encoder.Uvarint(protocol.ClientQuery); err != nil {
return err
}
var queryID string
queryIDValue := ctx.Value(queryIDKey)
if queryIDValue != nil {
if queryIdStr, ok := queryIDValue.(string); ok {
queryID = queryIdStr
}
}
if err := ch.encoder.String(queryID); err != nil {
return err
}
{ // client info
ch.encoder.Uvarint(1)
ch.encoder.String("")
ch.encoder.String("")
ch.encoder.String("[::ffff:127.0.0.1]:0")
ch.encoder.Uvarint(1) // iface type TCP
ch.encoder.String(hostname)
ch.encoder.String(hostname)
}
if err := ch.ClientInfo.Write(ch.encoder); err != nil {
return err
}
if ch.ServerInfo.Revision >= protocol.DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO {
ch.encoder.String("")
}
// the settings are written as list of contiguous name-value pairs, finished with empty name
if !ch.settings.IsEmpty() {
ch.logf("[query settings] %s", ch.settings.settingsStr)
if err := ch.settings.Serialize(ch.encoder); err != nil {
return err
}
}
// empty string is a marker of the end of the settings
if err := ch.encoder.String(""); err != nil {
return err
}
if err := ch.encoder.Uvarint(protocol.StateComplete); err != nil {
return err
}
compress := protocol.CompressDisable
if ch.compress {
compress = protocol.CompressEnable
}
if err := ch.encoder.Uvarint(compress); err != nil {
return err
}
if err := ch.encoder.String(query); err != nil {
return err
}
if err := ch.sendExternalTables(externalTables); err != nil {
return err
}
if err := ch.writeBlock(&data.Block{}, ""); err != nil {
return err
}
return ch.encoder.Flush()
}
vendor/github.com/ClickHouse/clickhouse-go/clickhouse_write_block.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
package clickhouse
import (
"github.com/ClickHouse/clickhouse-go/lib/data"
"github.com/ClickHouse/clickhouse-go/lib/protocol"
)
func (ch *clickhouse) writeBlock(block *data.Block, tableName string) error {
ch.Lock()
defer ch.Unlock()
if err := ch.encoder.Uvarint(protocol.ClientData); err != nil {
return err
}
if err := ch.encoder.String(tableName); err != nil { // temporary table
return err
}
// implement CityHash v 1.0.2 and add LZ4 compression
/*
From Alexey Milovidov
Насколько я помню, сжимаются блоки с данными Native формата, а всё остальное (всякие номера пакетов и т. п.) передаётся без сжатия.
Сжатые данные устроены так. Они представляют собой набор сжатых фреймов.
Каждый фрейм имеет следующий вид:
чексумма (16 байт),
идентификатор алгоритма сжатия (1 байт),
размер сжатых данных (4 байта, little endian, размер не включает в себя чексумму, но включает в себя остальные 9 байт заголовка),
размер несжатых данных (4 байта, little endian), затем сжатые данные.
Идентификатор алгоритма: 0x82 - lz4, 0x90 - zstd.
Чексумма - CityHash128 из CityHash версии 1.0.2, вычисленный от сжатых данных с учётом 9 байт заголовка.
См. CompressedReadBufferBase, CompressedWriteBuffer,
utils/compressor, TCPHandler.
*/
ch.encoder.SelectCompress(ch.compress)
err := block.Write(&ch.ServerInfo, ch.encoder)
ch.encoder.SelectCompress(false)
return err
}
vendor/github.com/ClickHouse/clickhouse-go/connect.go
Generated Vendored
+191
View File
@@ -0,0 +1,191 @@
package clickhouse
import (
"bufio"
"crypto/tls"
"database/sql/driver"
"net"
"sync/atomic"
"time"
)
var tick int32
type openStrategy int8
func (s openStrategy) String() string {
switch s {
case connOpenInOrder:
return "in_order"
case connOpenTimeRandom:
return "time_random"
}
return "random"
}
const (
connOpenRandom openStrategy = iota + 1
connOpenInOrder
connOpenTimeRandom
)
type connOptions struct {
secure, skipVerify bool
tlsConfig *tls.Config
hosts []string
connTimeout, readTimeout, writeTimeout time.Duration
noDelay bool
openStrategy openStrategy
logf func(string, ...interface{})
}
func dial(options connOptions) (*connect, error) {
var (
err error
abs = func(v int) int {
if v < 0 {
return -1 * v
}
return v
}
conn net.Conn
ident = abs(int(atomic.AddInt32(&tick, 1)))
)
tlsConfig := options.tlsConfig
if options.secure {
if tlsConfig == nil {
tlsConfig = &tls.Config{}
}
tlsConfig.InsecureSkipVerify = options.skipVerify
}
checkedHosts := make(map[int]struct{}, len(options.hosts))
for i := range options.hosts {
var num int
switch options.openStrategy {
case connOpenInOrder:
num = i
case connOpenRandom:
num = (ident + i) % len(options.hosts)
case connOpenTimeRandom:
// select host based on milliseconds
num = int((time.Now().UnixNano()/1000)%1000) % len(options.hosts)
for _, ok := checkedHosts[num]; ok; _, ok = checkedHosts[num] {
num = int(time.Now().UnixNano()) % len(options.hosts)
}
checkedHosts[num] = struct{}{}
}
switch {
case options.secure:
conn, err = tls.DialWithDialer(
&net.Dialer{
Timeout: options.connTimeout,
},
"tcp",
options.hosts[num],
tlsConfig,
)
default:
conn, err = net.DialTimeout("tcp", options.hosts[num], options.connTimeout)
}
if err == nil {
options.logf(
"[dial] secure=%t, skip_verify=%t, strategy=%s, ident=%d, server=%d -> %s",
options.secure,
options.skipVerify,
options.openStrategy,
ident,
num,
conn.RemoteAddr(),
)
if tcp, ok := conn.(*net.TCPConn); ok {
err = tcp.SetNoDelay(options.noDelay) // Disable or enable the Nagle Algorithm for this tcp socket
if err != nil {
return nil, err
}
}
return &connect{
Conn: conn,
logf: options.logf,
ident: ident,
buffer: bufio.NewReader(conn),
readTimeout: options.readTimeout,
writeTimeout: options.writeTimeout,
}, nil
} else {
options.logf(
"[dial err] secure=%t, skip_verify=%t, strategy=%s, ident=%d, addr=%s\n%#v",
options.secure,
options.skipVerify,
options.openStrategy,
ident,
options.hosts[num],
err,
)
}
}
return nil, err
}
type connect struct {
net.Conn
logf func(string, ...interface{})
ident int
buffer *bufio.Reader
closed bool
readTimeout time.Duration
writeTimeout time.Duration
lastReadDeadlineTime time.Time
lastWriteDeadlineTime time.Time
}
func (conn *connect) Read(b []byte) (int, error) {
var (
n int
err error
total int
dstLen = len(b)
)
if currentTime := now(); conn.readTimeout != 0 && currentTime.Sub(conn.lastReadDeadlineTime) > (conn.readTimeout>>2) {
conn.SetReadDeadline(time.Now().Add(conn.readTimeout))
conn.lastReadDeadlineTime = currentTime
}
for total < dstLen {
if n, err = conn.buffer.Read(b[total:]); err != nil {
conn.logf("[connect] read error: %v", err)
conn.Close()
return n, driver.ErrBadConn
}
total += n
}
return total, nil
}
func (conn *connect) Write(b []byte) (int, error) {
var (
n int
err error
total int
srcLen = len(b)
)
if currentTime := now(); conn.writeTimeout != 0 && currentTime.Sub(conn.lastWriteDeadlineTime) > (conn.writeTimeout>>2) {
conn.SetWriteDeadline(time.Now().Add(conn.writeTimeout))
conn.lastWriteDeadlineTime = currentTime
}
for total < srcLen {
if n, err = conn.Conn.Write(b[total:]); err != nil {
conn.logf("[connect] write error: %v", err)
conn.Close()
return n, driver.ErrBadConn
}
total += n
}
return n, nil
}
func (conn *connect) Close() error {
if !conn.closed {
conn.closed = true
return conn.Conn.Close()
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/docker-compose.yml
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
---
version: '3'
services:
clickhouse:
image: yandex/clickhouse-server
ports:
- 127.0.0.1:8123:8123
- 127.0.0.1:9000:9000
- 127.0.0.1:9009:9009
vendor/github.com/ClickHouse/clickhouse-go/go.mod
Generated Vendored
+11
View File
@@ -0,0 +1,11 @@
module github.com/ClickHouse/clickhouse-go
go 1.12
require (
github.com/bkaradzic/go-lz4 v1.0.0
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58
github.com/jmoiron/sqlx v1.2.0
github.com/pierrec/lz4 v2.0.5+incompatible
github.com/stretchr/testify v1.3.0
)
vendor/github.com/ClickHouse/clickhouse-go/go.sum
Generated Vendored
+18
View File
@@ -0,0 +1,18 @@
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
vendor/github.com/ClickHouse/clickhouse-go/go.test.sh
Generated Vendored
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -e
echo "" > coverage.txt
for d in $(go list ./... | grep -v vendor | grep -v examples); do
go test -race -coverprofile=profile.out -covermode=atomic $d
if [ -f profile.out ]; then
cat profile.out >> coverage.txt
rm profile.out
fi
done
vendor/github.com/ClickHouse/clickhouse-go/helpers.go
Generated Vendored
+146
View File
@@ -0,0 +1,146 @@
package clickhouse
import (
"bytes"
"database/sql/driver"
"fmt"
"math"
"reflect"
"regexp"
"strings"
"time"
)
func numInput(query string) int {
var (
count int
args = make(map[string]struct{})
reader = bytes.NewReader([]byte(query))
quote, gravis bool
escape bool
keyword bool
inBetween bool
like = newMatcher("like")
limit = newMatcher("limit")
offset = newMatcher("offset")
between = newMatcher("between")
in = newMatcher("in")
and = newMatcher("and")
from = newMatcher("from")
join = newMatcher("join")
)
for {
if char, _, err := reader.ReadRune(); err == nil {
if escape {
escape = false
continue
}
switch char {
case '\\':
if gravis || quote {
escape = true
}
case '\'':
if !gravis {
quote = !quote
}
case '`':
if !quote {
gravis = !gravis
}
}
if quote || gravis {
continue
}
switch {
case char == '?' && keyword:
count++
case char == '@':
if param := paramParser(reader); len(param) != 0 {
if _, found := args[param]; !found {
args[param] = struct{}{}
count++
}
}
case
char == '=',
char == '<',
char == '>',
char == '(',
char == ',',
char == '[',
char == '%':
keyword = true
default:
if limit.matchRune(char) || offset.matchRune(char) || like.matchRune(char) ||
in.matchRune(char) || from.matchRune(char) || join.matchRune(char) {
keyword = true
} else if between.matchRune(char) {
keyword = true
inBetween = true
} else if inBetween && and.matchRune(char) {
keyword = true
inBetween = false
} else {
keyword = keyword && (char == ' ' || char == '\t' || char == '\n')
}
}
} else {
break
}
}
return count
}
func paramParser(reader *bytes.Reader) string {
var name bytes.Buffer
for {
if char, _, err := reader.ReadRune(); err == nil {
if char == '_' || char >= '0' && char <= '9' || 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' {
name.WriteRune(char)
} else {
reader.UnreadRune()
break
}
} else {
break
}
}
return name.String()
}
var selectRe = regexp.MustCompile(`\s+SELECT\s+`)
func isInsert(query string) bool {
if f := strings.Fields(query); len(f) > 2 {
return strings.EqualFold("INSERT", f[0]) && strings.EqualFold("INTO", f[1]) && !selectRe.MatchString(strings.ToUpper(query))
}
return false
}
func quote(v driver.Value) string {
switch v := reflect.ValueOf(v); v.Kind() {
case reflect.Slice:
values := make([]string, 0, v.Len())
for i := 0; i < v.Len(); i++ {
values = append(values, quote(v.Index(i).Interface()))
}
return strings.Join(values, ", ")
}
switch v := v.(type) {
case string:
return "'" + strings.NewReplacer(`\`, `\\`, `'`, `\'`).Replace(v) + "'"
case time.Time:
return formatTime(v)
}
return fmt.Sprint(v)
}
func formatTime(value time.Time) string {
// toDate() overflows after 65535 days, but toDateTime() only overflows when time.Time overflows (after 9223372036854775807 seconds)
if days := value.Unix() / 24 / 3600; days <= math.MaxUint16 && (value.Hour()+value.Minute()+value.Second()+value.Nanosecond()) == 0 {
return fmt.Sprintf("toDate(%d)", days)
}
return fmt.Sprintf("toDateTime(%d)", value.Unix())
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader.go
Generated Vendored
+107
View File
@@ -0,0 +1,107 @@
// +build !clz4
package binary
import (
"encoding/binary"
"fmt"
"io"
"github.com/ClickHouse/clickhouse-go/lib/lz4"
)
type compressReader struct {
reader io.Reader
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
// lz4 headers
header []byte
}
// NewCompressReader wrap the io.Reader
func NewCompressReader(r io.Reader) *compressReader {
p := &compressReader{
reader: r,
header: make([]byte, HeaderSize),
}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
p.pos = len(p.data)
return p
}
func (cr *compressReader) Read(buf []byte) (n int, err error) {
var bytesRead = 0
n = len(buf)
if cr.pos < len(cr.data) {
copyedSize := copy(buf, cr.data[cr.pos:])
bytesRead += copyedSize
cr.pos += copyedSize
}
for bytesRead < n {
if err = cr.readCompressedData(); err != nil {
return bytesRead, err
}
copyedSize := copy(buf[bytesRead:], cr.data)
bytesRead += copyedSize
cr.pos = copyedSize
}
return n, nil
}
func (cr *compressReader) readCompressedData() (err error) {
cr.pos = 0
var n int
n, err = cr.reader.Read(cr.header)
if err != nil {
return
}
if n != len(cr.header) {
return fmt.Errorf("Lz4 decompression header EOF")
}
compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
if compressedSize > cap(cr.zdata) {
cr.zdata = make([]byte, compressedSize)
}
if decompressedSize > cap(cr.data) {
cr.data = make([]byte, decompressedSize)
}
cr.zdata = cr.zdata[:compressedSize]
cr.data = cr.data[:decompressedSize]
// @TODO checksum
if cr.header[16] == LZ4 {
n, err = cr.reader.Read(cr.zdata)
if err != nil {
return
}
if n != len(cr.zdata) {
return fmt.Errorf("Decompress read size not match")
}
_, err = lz4.Decode(cr.data, cr.zdata)
if err != nil {
return
}
} else {
return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_reader_clz4.go
Generated Vendored
+107
View File
@@ -0,0 +1,107 @@
// +build clz4
package binary
import (
"encoding/binary"
"fmt"
"io"
lz4 "github.com/cloudflare/golz4"
)
type compressReader struct {
reader io.Reader
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
// lz4 headers
header []byte
}
// NewCompressReader wrap the io.Reader
func NewCompressReader(r io.Reader) *compressReader {
p := &compressReader{
reader: r,
header: make([]byte, HeaderSize),
}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(p.data) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
p.pos = len(p.data)
return p
}
func (cr *compressReader) Read(buf []byte) (n int, err error) {
var bytesRead = 0
n = len(buf)
if cr.pos < len(cr.data) {
copyedSize := copy(buf, cr.data[cr.pos:])
bytesRead += copyedSize
cr.pos += copyedSize
}
for bytesRead < n {
if err = cr.readCompressedData(); err != nil {
return bytesRead, err
}
copyedSize := copy(buf[bytesRead:], cr.data)
bytesRead += copyedSize
cr.pos = copyedSize
}
return n, nil
}
func (cr *compressReader) readCompressedData() (err error) {
cr.pos = 0
var n int
n, err = cr.reader.Read(cr.header)
if err != nil {
return
}
if n != len(cr.header) {
return fmt.Errorf("Lz4 decompression header EOF")
}
compressedSize := int(binary.LittleEndian.Uint32(cr.header[17:])) - 9
decompressedSize := int(binary.LittleEndian.Uint32(cr.header[21:]))
if compressedSize > cap(cr.zdata) {
cr.zdata = make([]byte, compressedSize)
}
if decompressedSize > cap(cr.data) {
cr.data = make([]byte, decompressedSize)
}
cr.zdata = cr.zdata[:compressedSize]
cr.data = cr.data[:decompressedSize]
// @TODO checksum
if cr.header[16] == LZ4 {
n, err = cr.reader.Read(cr.zdata)
if err != nil {
return
}
if n != len(cr.zdata) {
return fmt.Errorf("Decompress read size not match")
}
err = lz4.Uncompress(cr.zdata, cr.data)
if err != nil {
return
}
} else {
return fmt.Errorf("Unknown compression method: 0x%02x ", cr.header[16])
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_settings.go
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
package binary
type CompressionMethodByte byte
const (
NONE CompressionMethodByte = 0x02
LZ4 = 0x82
ZSTD = 0x90
)
const (
// ChecksumSize is 128bits for cityhash102 checksum
ChecksumSize = 16
// CompressHeader magic + compressed_size + uncompressed_size
CompressHeaderSize = 1 + 4 + 4
// HeaderSize
HeaderSize = ChecksumSize + CompressHeaderSize
// BlockMaxSize 1MB
BlockMaxSize = 1 << 20
)
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer.go
Generated Vendored
+79
View File
@@ -0,0 +1,79 @@
// +build !clz4
package binary
import (
"encoding/binary"
"io"
"github.com/ClickHouse/clickhouse-go/lib/cityhash102"
"github.com/ClickHouse/clickhouse-go/lib/lz4"
)
type compressWriter struct {
writer io.Writer
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
}
// NewCompressWriter wrap the io.Writer
func NewCompressWriter(w io.Writer) *compressWriter {
p := &compressWriter{writer: w}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(BlockMaxSize) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
return p
}
func (cw *compressWriter) Write(buf []byte) (int, error) {
var n int
for len(buf) > 0 {
// Accumulate the data to be compressed.
m := copy(cw.data[cw.pos:], buf)
cw.pos += m
buf = buf[m:]
if cw.pos == len(cw.data) {
err := cw.Flush()
if err != nil {
return n, err
}
}
n += m
}
return n, nil
}
func (cw *compressWriter) Flush() (err error) {
if cw.pos == 0 {
return
}
// write the headers
compressedSize, err := lz4.Encode(cw.zdata[HeaderSize:], cw.data[:cw.pos])
if err != nil {
return err
}
compressedSize += CompressHeaderSize
// fill the header, compressed_size_32 + uncompressed_size_32
cw.zdata[16] = LZ4
binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
// fill the checksum
checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
if w, ok := cw.writer.(WriteFlusher); ok {
err = w.Flush()
}
cw.pos = 0
return
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/compress_writer_clz4.go
Generated Vendored
+78
View File
@@ -0,0 +1,78 @@
// +build clz4
package binary
import (
"encoding/binary"
"io"
lz4 "github.com/cloudflare/golz4"
"github.com/ClickHouse/clickhouse-go/lib/cityhash102"
)
type compressWriter struct {
writer io.Writer
// data uncompressed
data []byte
// data position
pos int
// data compressed
zdata []byte
}
// NewCompressWriter wrap the io.Writer
func NewCompressWriter(w io.Writer) *compressWriter {
p := &compressWriter{writer: w}
p.data = make([]byte, BlockMaxSize, BlockMaxSize)
zlen := lz4.CompressBound(p.data) + HeaderSize
p.zdata = make([]byte, zlen, zlen)
return p
}
func (cw *compressWriter) Write(buf []byte) (int, error) {
var n int
for len(buf) > 0 {
// Accumulate the data to be compressed.
m := copy(cw.data[cw.pos:], buf)
cw.pos += m
buf = buf[m:]
if cw.pos == len(cw.data) {
err := cw.Flush()
if err != nil {
return n, err
}
}
n += m
}
return n, nil
}
func (cw *compressWriter) Flush() (err error) {
if cw.pos == 0 {
return
}
// write the headers
compressedSize, err := lz4.Compress(cw.data[:cw.pos], cw.zdata[HeaderSize:])
if err != nil {
return err
}
compressedSize += CompressHeaderSize
// fill the header, compressed_size_32 + uncompressed_size_32
cw.zdata[16] = LZ4
binary.LittleEndian.PutUint32(cw.zdata[17:], uint32(compressedSize))
binary.LittleEndian.PutUint32(cw.zdata[21:], uint32(cw.pos))
// fill the checksum
checkSum := cityhash102.CityHash128(cw.zdata[16:], uint32(compressedSize))
binary.LittleEndian.PutUint64(cw.zdata[0:], checkSum.Lower64())
binary.LittleEndian.PutUint64(cw.zdata[8:], checkSum.Higher64())
cw.writer.Write(cw.zdata[:compressedSize+ChecksumSize])
if w, ok := cw.writer.(WriteFlusher); ok {
err = w.Flush()
}
cw.pos = 0
return
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/decoder.go
Generated Vendored
+177
View File
@@ -0,0 +1,177 @@
package binary
import (
"encoding/binary"
"io"
"math"
)
func NewDecoder(input io.Reader) *Decoder {
return &Decoder{
input: input,
}
}
func NewDecoderWithCompress(input io.Reader) *Decoder {
return &Decoder{
input: input,
compressInput: NewCompressReader(input),
}
}
type Decoder struct {
compress bool
input io.Reader
compressInput io.Reader
scratch [binary.MaxVarintLen64]byte
}
func (decoder *Decoder) SelectCompress(compress bool) {
decoder.compress = compress
}
func (decoder *Decoder) Get() io.Reader {
if decoder.compress && decoder.compressInput != nil {
return decoder.compressInput
}
return decoder.input
}
func (decoder *Decoder) Bool() (bool, error) {
v, err := decoder.ReadByte()
if err != nil {
return false, err
}
return v == 1, nil
}
func (decoder *Decoder) Uvarint() (uint64, error) {
return binary.ReadUvarint(decoder)
}
func (decoder *Decoder) Int8() (int8, error) {
v, err := decoder.ReadByte()
if err != nil {
return 0, err
}
return int8(v), nil
}
func (decoder *Decoder) Int16() (int16, error) {
v, err := decoder.UInt16()
if err != nil {
return 0, err
}
return int16(v), nil
}
func (decoder *Decoder) Int32() (int32, error) {
v, err := decoder.UInt32()
if err != nil {
return 0, err
}
return int32(v), nil
}
func (decoder *Decoder) Int64() (int64, error) {
v, err := decoder.UInt64()
if err != nil {
return 0, err
}
return int64(v), nil
}
func (decoder *Decoder) UInt8() (uint8, error) {
v, err := decoder.ReadByte()
if err != nil {
return 0, err
}
return uint8(v), nil
}
func (decoder *Decoder) UInt16() (uint16, error) {
if _, err := decoder.Get().Read(decoder.scratch[:2]); err != nil {
return 0, err
}
return uint16(decoder.scratch[0]) | uint16(decoder.scratch[1])<<8, nil
}
func (decoder *Decoder) UInt32() (uint32, error) {
if _, err := decoder.Get().Read(decoder.scratch[:4]); err != nil {
return 0, err
}
return uint32(decoder.scratch[0]) |
uint32(decoder.scratch[1])<<8 |
uint32(decoder.scratch[2])<<16 |
uint32(decoder.scratch[3])<<24, nil
}
func (decoder *Decoder) UInt64() (uint64, error) {
if _, err := decoder.Get().Read(decoder.scratch[:8]); err != nil {
return 0, err
}
return uint64(decoder.scratch[0]) |
uint64(decoder.scratch[1])<<8 |
uint64(decoder.scratch[2])<<16 |
uint64(decoder.scratch[3])<<24 |
uint64(decoder.scratch[4])<<32 |
uint64(decoder.scratch[5])<<40 |
uint64(decoder.scratch[6])<<48 |
uint64(decoder.scratch[7])<<56, nil
}
func (decoder *Decoder) Float32() (float32, error) {
v, err := decoder.UInt32()
if err != nil {
return 0, err
}
return math.Float32frombits(v), nil
}
func (decoder *Decoder) Float64() (float64, error) {
v, err := decoder.UInt64()
if err != nil {
return 0, err
}
return math.Float64frombits(v), nil
}
func (decoder *Decoder) Fixed(ln int) ([]byte, error) {
if reader, ok := decoder.Get().(FixedReader); ok {
return reader.Fixed(ln)
}
buf := make([]byte, ln)
if _, err := decoder.Get().Read(buf); err != nil {
return nil, err
}
return buf, nil
}
func (decoder *Decoder) String() (string, error) {
strlen, err := decoder.Uvarint()
if err != nil {
return "", err
}
str, err := decoder.Fixed(int(strlen))
if err != nil {
return "", err
}
return string(str), nil
}
func (decoder *Decoder) Decimal128() ([]byte, error) {
bytes := make([]byte, 16)
_, err := decoder.Get().Read(bytes)
return bytes, err
}
func (decoder *Decoder) ReadByte() (byte, error) {
if _, err := decoder.Get().Read(decoder.scratch[:1]); err != nil {
return 0x0, err
}
return decoder.scratch[0], nil
}
type FixedReader interface {
Fixed(ln int) ([]byte, error)
}
vendor/github.com/ClickHouse/clickhouse-go/lib/binary/encoder.go
Generated Vendored
+364
View File
@@ -0,0 +1,364 @@
package binary
import (
"encoding/binary"
"io"
"math"
"reflect"
"unsafe"
)
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
output: w,
}
}
func NewEncoderWithCompress(w io.Writer) *Encoder {
return &Encoder{
output: w,
compressOutput: NewCompressWriter(w),
}
}
type Encoder struct {
compress bool
output io.Writer
compressOutput io.Writer
scratch [binary.MaxVarintLen64]byte
}
func (enc *Encoder) SelectCompress(compress bool) {
if enc.compressOutput == nil {
return
}
if enc.compress && !compress {
enc.Flush()
}
enc.compress = compress
}
func (enc *Encoder) Get() io.Writer {
if enc.compress && enc.compressOutput != nil {
return enc.compressOutput
}
return enc.output
}
func (enc *Encoder) Nullable(isNull bool) error {
nullablePrefix := uint8(0)
if !isNull {
nullablePrefix = uint8(1)
}
if _, err := enc.Get().Write([]byte{nullablePrefix}); err != nil {
return err
}
return nil
}
func (enc *Encoder) Uvarint(v uint64) error {
ln := binary.PutUvarint(enc.scratch[:binary.MaxVarintLen64], v)
if _, err := enc.Get().Write(enc.scratch[0:ln]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UvarintNullable(v *uint64) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Uvarint(0)
}
return enc.Uvarint(*v)
}
func (enc *Encoder) Bool(v bool) error {
if v {
return enc.UInt8(1)
}
return enc.UInt8(0)
}
func (enc *Encoder) BoolNullable(v *bool) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Bool(false)
}
return enc.Bool(*v)
}
func (enc *Encoder) Int8(v int8) error {
return enc.UInt8(uint8(v))
}
func (enc *Encoder) Int8Nullable(v *int8) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Int8(0)
}
return enc.Int8(*v)
}
func (enc *Encoder) Int16(v int16) error {
return enc.UInt16(uint16(v))
}
func (enc *Encoder) Int16Nullable(v *int16) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Int16(0)
}
return enc.Int16(*v)
}
func (enc *Encoder) Int32(v int32) error {
return enc.UInt32(uint32(v))
}
func (enc *Encoder) Int32Nullable(v *int32) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Int32(0)
}
return enc.Int32(*v)
}
func (enc *Encoder) Int64(v int64) error {
return enc.UInt64(uint64(v))
}
func (enc *Encoder) Int64Nullable(v *int64) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Int64(0)
}
return enc.Int64(*v)
}
func (enc *Encoder) UInt8(v uint8) error {
enc.scratch[0] = v
if _, err := enc.Get().Write(enc.scratch[:1]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt8Nullable(v *uint8) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.UInt8(0)
}
return enc.UInt8(*v)
}
func (enc *Encoder) UInt16(v uint16) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
if _, err := enc.Get().Write(enc.scratch[:2]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt16Nullable(v *uint16) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.UInt16(0)
}
return enc.UInt16(*v)
}
func (enc *Encoder) UInt32(v uint32) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
enc.scratch[2] = byte(v >> 16)
enc.scratch[3] = byte(v >> 24)
if _, err := enc.Get().Write(enc.scratch[:4]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt32Nullable(v *uint32) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.UInt32(0)
}
return enc.UInt32(*v)
}
func (enc *Encoder) UInt64(v uint64) error {
enc.scratch[0] = byte(v)
enc.scratch[1] = byte(v >> 8)
enc.scratch[2] = byte(v >> 16)
enc.scratch[3] = byte(v >> 24)
enc.scratch[4] = byte(v >> 32)
enc.scratch[5] = byte(v >> 40)
enc.scratch[6] = byte(v >> 48)
enc.scratch[7] = byte(v >> 56)
if _, err := enc.Get().Write(enc.scratch[:8]); err != nil {
return err
}
return nil
}
func (enc *Encoder) UInt64Nullable(v *uint64) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.UInt64(0)
}
return enc.UInt64(*v)
}
func (enc *Encoder) Float32(v float32) error {
return enc.UInt32(math.Float32bits(v))
}
func (enc *Encoder) Float32Nullable(v *float32) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Float32(0)
}
return enc.Float32(*v)
}
func (enc *Encoder) Float64(v float64) error {
return enc.UInt64(math.Float64bits(v))
}
func (enc *Encoder) Float64Nullable(v *float64) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.Float64(0)
}
return enc.Float64(*v)
}
func (enc *Encoder) String(v string) error {
str := Str2Bytes(v)
if err := enc.Uvarint(uint64(len(str))); err != nil {
return err
}
if _, err := enc.Get().Write(str); err != nil {
return err
}
return nil
}
func (enc *Encoder) StringNullable(v *string) error {
isNil := v == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.String("")
}
return enc.String(*v)
}
func (enc *Encoder) RawString(str []byte) error {
if err := enc.Uvarint(uint64(len(str))); err != nil {
return err
}
if _, err := enc.Get().Write(str); err != nil {
return err
}
return nil
}
func (enc *Encoder) RawStringNullable(str *[]byte) error {
isNil := str == nil
if err := enc.Nullable(isNil); err != nil {
return err
}
if isNil {
return enc.RawString([]byte{})
}
return enc.RawString(*str)
}
func (enc *Encoder) Decimal128(bytes []byte) error {
_, err := enc.Get().Write(bytes)
return err
}
func (enc *Encoder) Write(b []byte) (int, error) {
return enc.Get().Write(b)
}
func (enc *Encoder) Flush() error {
if w, ok := enc.Get().(WriteFlusher); ok {
return w.Flush()
}
return nil
}
type WriteFlusher interface {
Flush() error
}
func Str2Bytes(str string) []byte {
// Copied from https://github.com/m3db/m3/blob/master/src/x/unsafe/string.go#L62
if len(str) == 0 {
return nil
}
// We need to declare a real byte slice so internally the compiler
// knows to use an unsafe.Pointer to keep track of the underlying memory so that
// once the slice's array pointer is updated with the pointer to the string's
// underlying bytes, the compiler won't prematurely GC the memory when the string
// goes out of scope.
var b []byte
byteHeader := (*reflect.SliceHeader)(unsafe.Pointer(&b))
// This makes sure that even if GC relocates the string's underlying
// memory after this assignment, the corresponding unsafe.Pointer in the internal
// slice struct will be updated accordingly to reflect the memory relocation.
byteHeader.Data = (*reflect.StringHeader)(unsafe.Pointer(&str)).Data
// It is important that we access str after we assign the Data
// pointer of the string header to the Data pointer of the slice header to
// make sure the string (and the underlying bytes backing the string) don't get
// GC'ed before the assignment happens.
l := len(str)
byteHeader.Len = l
byteHeader.Cap = l
return b
}
vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/city64.go
Generated Vendored
+45
View File
@@ -0,0 +1,45 @@
package cityhash102
import (
"encoding/binary"
"hash"
)
type City64 struct {
s []byte
}
var _ hash.Hash64 = (*City64)(nil)
var _ hash.Hash = (*City64)(nil)
func New64() hash.Hash64 {
return &City64{}
}
func (this *City64) Sum(b []byte) []byte {
b2 := make([]byte, 8)
binary.BigEndian.PutUint64(b2, this.Sum64())
b = append(b, b2...)
return b
}
func (this *City64) Sum64() uint64 {
return CityHash64(this.s, uint32(len(this.s)))
}
func (this *City64) Reset() {
this.s = this.s[0:0]
}
func (this *City64) BlockSize() int {
return 1
}
func (this *City64) Write(s []byte) (n int, err error) {
this.s = append(this.s, s...)
return len(s), nil
}
func (this *City64) Size() int {
return 8
}
vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/cityhash.go
Generated Vendored
+383
View File
@@ -0,0 +1,383 @@
/*
* Go implementation of Google city hash (MIT license)
* https://code.google.com/p/cityhash/
*
* MIT License http://www.opensource.org/licenses/mit-license.php
*
* I don't even want to pretend to understand the details of city hash.
* I am only reproducing the logic in Go as faithfully as I can.
*
*/
package cityhash102
import (
"encoding/binary"
)
const (
k0 uint64 = 0xc3a5c85c97cb3127
k1 uint64 = 0xb492b66fbe98f273
k2 uint64 = 0x9ae16a3b2f90404f
k3 uint64 = 0xc949d7c7509e6557
kMul uint64 = 0x9ddfea08eb382d69
)
func fetch64(p []byte) uint64 {
return binary.LittleEndian.Uint64(p)
//return uint64InExpectedOrder(unalignedLoad64(p))
}
func fetch32(p []byte) uint32 {
return binary.LittleEndian.Uint32(p)
//return uint32InExpectedOrder(unalignedLoad32(p))
}
func rotate64(val uint64, shift uint32) uint64 {
if shift != 0 {
return ((val >> shift) | (val << (64 - shift)))
}
return val
}
func rotate32(val uint32, shift uint32) uint32 {
if shift != 0 {
return ((val >> shift) | (val << (32 - shift)))
}
return val
}
func swap64(a, b *uint64) {
*a, *b = *b, *a
}
func swap32(a, b *uint32) {
*a, *b = *b, *a
}
func permute3(a, b, c *uint32) {
swap32(a, b)
swap32(a, c)
}
func rotate64ByAtLeast1(val uint64, shift uint32) uint64 {
return (val >> shift) | (val << (64 - shift))
}
func shiftMix(val uint64) uint64 {
return val ^ (val >> 47)
}
type Uint128 [2]uint64
func (this *Uint128) setLower64(l uint64) {
this[0] = l
}
func (this *Uint128) setHigher64(h uint64) {
this[1] = h
}
func (this Uint128) Lower64() uint64 {
return this[0]
}
func (this Uint128) Higher64() uint64 {
return this[1]
}
func (this Uint128) Bytes() []byte {
b := make([]byte, 16)
binary.LittleEndian.PutUint64(b, this[0])
binary.LittleEndian.PutUint64(b[8:], this[1])
return b
}
func hash128to64(x Uint128) uint64 {
// Murmur-inspired hashing.
var a = (x.Lower64() ^ x.Higher64()) * kMul
a ^= (a >> 47)
var b = (x.Higher64() ^ a) * kMul
b ^= (b >> 47)
b *= kMul
return b
}
func hashLen16(u, v uint64) uint64 {
return hash128to64(Uint128{u, v})
}
func hashLen16_3(u, v, mul uint64) uint64 {
// Murmur-inspired hashing.
var a = (u ^ v) * mul
a ^= (a >> 47)
var b = (v ^ a) * mul
b ^= (b >> 47)
b *= mul
return b
}
func hashLen0to16(s []byte, length uint32) uint64 {
if length > 8 {
var a = fetch64(s)
var b = fetch64(s[length-8:])
return hashLen16(a, rotate64ByAtLeast1(b+uint64(length), length)) ^ b
}
if length >= 4 {
var a = fetch32(s)
return hashLen16(uint64(length)+(uint64(a)<<3), uint64(fetch32(s[length-4:])))
}
if length > 0 {
var a uint8 = uint8(s[0])
var b uint8 = uint8(s[length>>1])
var c uint8 = uint8(s[length-1])
var y uint32 = uint32(a) + (uint32(b) << 8)
var z uint32 = length + (uint32(c) << 2)
return shiftMix(uint64(y)*k2^uint64(z)*k3) * k2
}
return k2
}
// This probably works well for 16-byte strings as well, but it may be overkill
func hashLen17to32(s []byte, length uint32) uint64 {
var a = fetch64(s) * k1
var b = fetch64(s[8:])
var c = fetch64(s[length-8:]) * k2
var d = fetch64(s[length-16:]) * k0
return hashLen16(rotate64(a-b, 43)+rotate64(c, 30)+d,
a+rotate64(b^k3, 20)-c+uint64(length))
}
func weakHashLen32WithSeeds(w, x, y, z, a, b uint64) Uint128 {
a += w
b = rotate64(b+a+z, 21)
var c uint64 = a
a += x
a += y
b += rotate64(a, 44)
return Uint128{a + z, b + c}
}
func weakHashLen32WithSeeds_3(s []byte, a, b uint64) Uint128 {
return weakHashLen32WithSeeds(fetch64(s), fetch64(s[8:]), fetch64(s[16:]), fetch64(s[24:]), a, b)
}
func hashLen33to64(s []byte, length uint32) uint64 {
var z uint64 = fetch64(s[24:])
var a uint64 = fetch64(s) + (uint64(length)+fetch64(s[length-16:]))*k0
var b uint64 = rotate64(a+z, 52)
var c uint64 = rotate64(a, 37)
a += fetch64(s[8:])
c += rotate64(a, 7)
a += fetch64(s[16:])
var vf uint64 = a + z
var vs = b + rotate64(a, 31) + c
a = fetch64(s[16:]) + fetch64(s[length-32:])
z = fetch64(s[length-8:])
b = rotate64(a+z, 52)
c = rotate64(a, 37)
a += fetch64(s[length-24:])
c += rotate64(a, 7)
a += fetch64(s[length-16:])
wf := a + z
ws := b + rotate64(a, 31) + c
r := shiftMix((vf+ws)*k2 + (wf+vs)*k0)
return shiftMix(r*k0+vs) * k2
}
func CityHash64(s []byte, length uint32) uint64 {
if length <= 32 {
if length <= 16 {
return hashLen0to16(s, length)
} else {
return hashLen17to32(s, length)
}
} else if length <= 64 {
return hashLen33to64(s, length)
}
var x uint64 = fetch64(s)
var y uint64 = fetch64(s[length-16:]) ^ k1
var z uint64 = fetch64(s[length-56:]) ^ k0
var v Uint128 = weakHashLen32WithSeeds_3(s[length-64:], uint64(length), y)
var w Uint128 = weakHashLen32WithSeeds_3(s[length-32:], uint64(length)*k1, k0)
z += shiftMix(v.Higher64()) * k1
x = rotate64(z+x, 39) * k1
y = rotate64(y, 33) * k1
length = (length - 1) & ^uint32(63)
for {
x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1
y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1
x ^= w.Higher64()
y ^= v.Lower64()
z = rotate64(z^w.Lower64(), 33)
v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64())
w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y)
swap64(&z, &x)
s = s[64:]
length -= 64
if length == 0 {
break
}
}
return hashLen16(hashLen16(v.Lower64(), w.Lower64())+shiftMix(y)*k1+z, hashLen16(v.Higher64(), w.Higher64())+x)
}
func CityHash64WithSeed(s []byte, length uint32, seed uint64) uint64 {
return CityHash64WithSeeds(s, length, k2, seed)
}
func CityHash64WithSeeds(s []byte, length uint32, seed0, seed1 uint64) uint64 {
return hashLen16(CityHash64(s, length)-seed0, seed1)
}
func cityMurmur(s []byte, length uint32, seed Uint128) Uint128 {
var a uint64 = seed.Lower64()
var b uint64 = seed.Higher64()
var c uint64 = 0
var d uint64 = 0
var l int32 = int32(length) - 16
if l <= 0 { // len <= 16
a = shiftMix(a*k1) * k1
c = b*k1 + hashLen0to16(s, length)
if length >= 8 {
d = shiftMix(a + fetch64(s))
} else {
d = shiftMix(a + c)
}
} else { // len > 16
c = hashLen16(fetch64(s[length-8:])+k1, a)
d = hashLen16(b+uint64(length), c+fetch64(s[length-16:]))
a += d
for {
a ^= shiftMix(fetch64(s)*k1) * k1
a *= k1
b ^= a
c ^= shiftMix(fetch64(s[8:])*k1) * k1
c *= k1
d ^= c
s = s[16:]
l -= 16
if l <= 0 {
break
}
}
}
a = hashLen16(a, c)
b = hashLen16(d, b)
return Uint128{a ^ b, hashLen16(b, a)}
}
func CityHash128WithSeed(s []byte, length uint32, seed Uint128) Uint128 {
if length < 128 {
return cityMurmur(s, length, seed)
}
// We expect length >= 128 to be the common case. Keep 56 bytes of state:
// v, w, x, y, and z.
var v, w Uint128
var x uint64 = seed.Lower64()
var y uint64 = seed.Higher64()
var z uint64 = uint64(length) * k1
var pos uint32
var t = s
v.setLower64(rotate64(y^k1, 49)*k1 + fetch64(s))
v.setHigher64(rotate64(v.Lower64(), 42)*k1 + fetch64(s[8:]))
w.setLower64(rotate64(y+z, 35)*k1 + x)
w.setHigher64(rotate64(x+fetch64(s[88:]), 53) * k1)
// This is the same inner loop as CityHash64(), manually unrolled.
for {
x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1
y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1
x ^= w.Higher64()
y ^= v.Lower64()
z = rotate64(z^w.Lower64(), 33)
v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64())
w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y)
swap64(&z, &x)
s = s[64:]
pos += 64
x = rotate64(x+y+v.Lower64()+fetch64(s[16:]), 37) * k1
y = rotate64(y+v.Higher64()+fetch64(s[48:]), 42) * k1
x ^= w.Higher64()
y ^= v.Lower64()
z = rotate64(z^w.Lower64(), 33)
v = weakHashLen32WithSeeds_3(s, v.Higher64()*k1, x+w.Lower64())
w = weakHashLen32WithSeeds_3(s[32:], z+w.Higher64(), y)
swap64(&z, &x)
s = s[64:]
pos += 64
length -= 128
if length < 128 {
break
}
}
y += rotate64(w.Lower64(), 37)*k0 + z
x += rotate64(v.Lower64()+z, 49) * k0
// If 0 < length < 128, hash up to 4 chunks of 32 bytes each from the end of s.
var tailDone uint32
for tailDone = 0; tailDone < length; {
tailDone += 32
y = rotate64(y-x, 42)*k0 + v.Higher64()
//TODO why not use origin_len ?
w.setLower64(w.Lower64() + fetch64(t[pos+length-tailDone+16:]))
x = rotate64(x, 49)*k0 + w.Lower64()
w.setLower64(w.Lower64() + v.Lower64())
v = weakHashLen32WithSeeds_3(t[pos+length-tailDone:], v.Lower64(), v.Higher64())
}
// At this point our 48 bytes of state should contain more than
// enough information for a strong 128-bit hash. We use two
// different 48-byte-to-8-byte hashes to get a 16-byte final result.
x = hashLen16(x, v.Lower64())
y = hashLen16(y, w.Lower64())
return Uint128{hashLen16(x+v.Higher64(), w.Higher64()) + y,
hashLen16(x+w.Higher64(), y+v.Higher64())}
}
func CityHash128(s []byte, length uint32) (result Uint128) {
if length >= 16 {
result = CityHash128WithSeed(s[16:length], length-16, Uint128{fetch64(s) ^ k3, fetch64(s[8:])})
} else if length >= 8 {
result = CityHash128WithSeed(nil, 0, Uint128{fetch64(s) ^ (uint64(length) * k0), fetch64(s[length-8:]) ^ k1})
} else {
result = CityHash128WithSeed(s, length, Uint128{k0, k1})
}
return
}
vendor/github.com/ClickHouse/clickhouse-go/lib/cityhash102/doc.go
Generated Vendored
+5
View File
@@ -0,0 +1,5 @@
/** COPY from https://github.com/zentures/cityhash/
NOTE: The code is modified to be compatible with CityHash128 used in ClickHouse
*/
package cityhash102
vendor/github.com/ClickHouse/clickhouse-go/lib/column/array.go
Generated Vendored
+270
View File
@@ -0,0 +1,270 @@
package column
import (
"errors"
"fmt"
"net"
"reflect"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type columnDecoder func() (interface{}, error)
var unsupportedArrayTypeErrTemp = "unsupported Array type '%s'"
// If you add Nullable type, that can be used in Array(Nullable(T)) add this type to ../codegen/nullable_appender/main.go in structure values.Types.
// Run code generation.
//go:generate go run ../codegen/nullable_appender -package $GOPACKAGE -file nullable_appender.go
type Array struct {
base
depth int
column Column
nullable bool
}
func (array *Array) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
return nil, fmt.Errorf("do not use Read method for Array(T) column")
}
func (array *Array) WriteNull(nulls, encoder *binary.Encoder, v interface{}) error {
if array.nullable {
column, ok := array.column.(*Nullable)
if !ok {
return fmt.Errorf("cannot convert to nullable type")
}
return column.WriteNull(nulls, encoder, v)
}
return fmt.Errorf("write null to not nullable array")
}
func (array *Array) Write(encoder *binary.Encoder, v interface{}) error {
return array.column.Write(encoder, v)
}
func (array *Array) ReadArray(decoder *binary.Decoder, rows int) (_ []interface{}, err error) {
var (
offsets = make([][]uint64, array.depth)
values = make([]interface{}, rows)
)
// Read offsets
lastOffset := uint64(rows)
for i := 0; i < array.depth; i++ {
offset := make([]uint64, lastOffset)
for j := uint64(0); j < lastOffset; j++ {
if offset[j], err = decoder.UInt64(); err != nil {
return nil, err
}
}
offsets[i] = offset
lastOffset = 0
if len(offset) > 0 {
lastOffset = offset[len(offset)-1]
}
}
var cd columnDecoder
switch column := array.column.(type) {
case *Nullable:
nullRows, err := column.ReadNull(decoder, int(lastOffset))
if err != nil {
return nil, err
}
cd = func(rows []interface{}) columnDecoder {
i := 0
return func() (interface{}, error) {
if i > len(rows) {
return nil, errors.New("not enough rows to return while parsing Null column")
}
ret := rows[i]
i++
return ret, nil
}
}(nullRows)
case *Tuple:
tupleRows, err := column.ReadTuple(decoder, int(lastOffset))
if err != nil {
return nil, err
}
// closure to return fully assembled tuple values as if they
// were decoded one at a time
cd = func(rows []interface{}) columnDecoder {
i := 0
return func() (interface{}, error) {
if i > len(rows) {
return nil, errors.New("not enough rows to return while parsing Tuple column")
}
ret := rows[i]
i++
return ret, nil
}
}(tupleRows)
default:
cd = func(decoder *binary.Decoder) columnDecoder {
return func() (interface{}, error) { return array.column.Read(decoder, array.nullable) }
}(decoder)
}
// Read values
for i := 0; i < rows; i++ {
if values[i], err = array.read(cd, offsets, uint64(i), 0); err != nil {
return nil, err
}
}
return values, nil
}
func (array *Array) read(readColumn columnDecoder, offsets [][]uint64, index uint64, level int) (interface{}, error) {
end := offsets[level][index]
start := uint64(0)
if index > 0 {
start = offsets[level][index-1]
}
scanT := array.column.ScanType()
slice := reflect.MakeSlice(array.arrayType(level), 0, int(end-start))
for i := start; i < end; i++ {
var (
value interface{}
err error
)
if level == array.depth-1 {
value, err = readColumn()
} else {
value, err = array.read(readColumn, offsets, i, level+1)
}
if err != nil {
return nil, err
}
if array.nullable && level == array.depth-1 {
f, ok := nullableAppender[scanT.String()]
if !ok {
return nil, fmt.Errorf(unsupportedArrayTypeErrTemp, scanT.String())
}
cSlice, err := f(value, slice)
if err != nil {
return nil, err
}
slice = cSlice
} else {
slice = reflect.Append(slice, reflect.ValueOf(value))
}
}
return slice.Interface(), nil
}
func (array *Array) arrayType(level int) reflect.Type {
t := array.column.ScanType()
for i := 0; i < array.depth-level; i++ {
t = reflect.SliceOf(t)
}
return t
}
func (array *Array) Depth() int {
return array.depth
}
func parseArray(name, chType string, timezone *time.Location) (*Array, error) {
if len(chType) < 11 {
return nil, fmt.Errorf("invalid Array column type: %s", chType)
}
var (
depth int
columnType = chType
)
loop:
for _, str := range strings.Split(chType, "Array(") {
switch {
case len(str) == 0:
depth++
default:
chType = str[:len(str)-depth]
break loop
}
}
column, err := Factory(name, chType, timezone)
if err != nil {
return nil, fmt.Errorf("Array(T): %v", err)
}
var scanType interface{}
switch t := column.ScanType(); t {
case arrayBaseTypes[int8(0)]:
scanType = []int8{}
case arrayBaseTypes[int16(0)]:
scanType = []int16{}
case arrayBaseTypes[int32(0)]:
scanType = []int32{}
case arrayBaseTypes[int64(0)]:
scanType = []int64{}
case arrayBaseTypes[uint8(0)]:
scanType = []uint8{}
case arrayBaseTypes[uint16(0)]:
scanType = []uint16{}
case arrayBaseTypes[uint32(0)]:
scanType = []uint32{}
case arrayBaseTypes[uint64(0)]:
scanType = []uint64{}
case arrayBaseTypes[float32(0)]:
scanType = []float32{}
case arrayBaseTypes[float64(0)]:
scanType = []float64{}
case arrayBaseTypes[string("")]:
scanType = []string{}
case arrayBaseTypes[time.Time{}]:
scanType = []time.Time{}
case arrayBaseTypes[IPv4{}], arrayBaseTypes[IPv6{}]:
scanType = []net.IP{}
case reflect.ValueOf([]interface{}{}).Type():
scanType = [][]interface{}{}
//nullable
case arrayBaseTypes[ptrInt8T]:
scanType = []*int8{}
case arrayBaseTypes[ptrInt16T]:
scanType = []*int16{}
case arrayBaseTypes[ptrInt32T]:
scanType = []*int32{}
case arrayBaseTypes[ptrInt64T]:
scanType = []*int64{}
case arrayBaseTypes[ptrUInt8T]:
scanType = []*uint8{}
case arrayBaseTypes[ptrUInt16T]:
scanType = []*uint16{}
case arrayBaseTypes[ptrUInt32T]:
scanType = []*uint32{}
case arrayBaseTypes[ptrUInt64T]:
scanType = []*uint64{}
case arrayBaseTypes[ptrFloat32]:
scanType = []*float32{}
case arrayBaseTypes[ptrFloat64]:
scanType = []*float64{}
case arrayBaseTypes[ptrString]:
scanType = []*string{}
case arrayBaseTypes[ptrTime]:
scanType = []*time.Time{}
case arrayBaseTypes[ptrIPv4], arrayBaseTypes[ptrIPv6]:
scanType = []*net.IP{}
default:
return nil, fmt.Errorf(unsupportedArrayTypeErrTemp, column.ScanType().Name())
}
return &Array{
base: base{
name: name,
chType: columnType,
valueOf: reflect.ValueOf(scanType),
},
depth: depth,
column: column,
nullable: strings.HasPrefix(column.CHType(), "Nullable"),
}, nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/column.go
Generated Vendored
+200
View File
@@ -0,0 +1,200 @@
package column
import (
"fmt"
"reflect"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Column interface {
Name() string
CHType() string
ScanType() reflect.Type
Read(*binary.Decoder, bool) (interface{}, error)
Write(*binary.Encoder, interface{}) error
defaultValue() interface{}
Depth() int
}
func Factory(name, chType string, timezone *time.Location) (Column, error) {
switch chType {
case "Int8":
return &Int8{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[int8(0)],
},
}, nil
case "Int16":
return &Int16{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[int16(0)],
},
}, nil
case "Int32":
return &Int32{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[int32(0)],
},
}, nil
case "Int64":
return &Int64{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[int64(0)],
},
}, nil
case "UInt8":
return &UInt8{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[uint8(0)],
},
}, nil
case "UInt16":
return &UInt16{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[uint16(0)],
},
}, nil
case "UInt32":
return &UInt32{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[uint32(0)],
},
}, nil
case "UInt64":
return &UInt64{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[uint64(0)],
},
}, nil
case "Float32":
return &Float32{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[float32(0)],
},
}, nil
case "Float64":
return &Float64{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[float64(0)],
},
}, nil
case "String":
return &String{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[string("")],
},
}, nil
case "UUID":
return &UUID{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[string("")],
},
}, nil
case "Date":
_, offset := time.Unix(0, 0).In(timezone).Zone()
return &Date{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[time.Time{}],
},
Timezone: timezone,
offset: int64(offset),
}, nil
case "IPv4":
return &IPv4{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[IPv4{}],
},
}, nil
case "IPv6":
return &IPv6{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[IPv6{}],
},
}, nil
}
switch {
case strings.HasPrefix(chType, "DateTime") && !strings.HasPrefix(chType, "DateTime64"):
return &DateTime{
base: base{
name: name,
chType: "DateTime",
valueOf: columnBaseTypes[time.Time{}],
},
Timezone: timezone,
}, nil
case strings.HasPrefix(chType, "DateTime64"):
return &DateTime64{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[time.Time{}],
},
Timezone: timezone,
}, nil
case strings.HasPrefix(chType, "Array"):
return parseArray(name, chType, timezone)
case strings.HasPrefix(chType, "Nullable"):
return parseNullable(name, chType, timezone)
case strings.HasPrefix(chType, "FixedString"):
return parseFixedString(name, chType)
case strings.HasPrefix(chType, "Enum8"), strings.HasPrefix(chType, "Enum16"):
return parseEnum(name, chType)
case strings.HasPrefix(chType, "Decimal"):
return parseDecimal(name, chType)
case strings.HasPrefix(chType, "SimpleAggregateFunction"):
if nestedType, err := getNestedType(chType, "SimpleAggregateFunction"); err != nil {
return nil, err
} else {
return Factory(name, nestedType, timezone)
}
case strings.HasPrefix(chType, "Tuple"):
return parseTuple(name, chType, timezone)
}
return nil, fmt.Errorf("column: unhandled type %v", chType)
}
func getNestedType(chType string, wrapType string) (string, error) {
prefixLen := len(wrapType) + 1
suffixLen := 1
if len(chType) > prefixLen+suffixLen {
nested := strings.Split(chType[prefixLen:len(chType)-suffixLen], ",")
if len(nested) == 2 {
return strings.TrimSpace(nested[1]), nil
}
}
return "", fmt.Errorf("column: invalid %s type (%s)", wrapType, chType)
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/common.go
Generated Vendored
+115
View File
@@ -0,0 +1,115 @@
package column
import (
"fmt"
"net"
"reflect"
"time"
)
type ErrUnexpectedType struct {
Column Column
T interface{}
}
func (err *ErrUnexpectedType) Error() string {
return fmt.Sprintf("%s: unexpected type %T", err.Column, err.T)
}
var columnBaseTypes = map[interface{}]reflect.Value{
int8(0): reflect.ValueOf(int8(0)),
int16(0): reflect.ValueOf(int16(0)),
int32(0): reflect.ValueOf(int32(0)),
int64(0): reflect.ValueOf(int64(0)),
uint8(0): reflect.ValueOf(uint8(0)),
uint16(0): reflect.ValueOf(uint16(0)),
uint32(0): reflect.ValueOf(uint32(0)),
uint64(0): reflect.ValueOf(uint64(0)),
float32(0): reflect.ValueOf(float32(0)),
float64(0): reflect.ValueOf(float64(0)),
string(""): reflect.ValueOf(string("")),
time.Time{}: reflect.ValueOf(time.Time{}),
IPv4{}: reflect.ValueOf(net.IPv4zero),
IPv6{}: reflect.ValueOf(net.IPv6unspecified),
}
type ptrTo uint8
const (
ptrInt8T ptrTo = iota
ptrInt16T
ptrInt32T
ptrInt64T
ptrUInt8T
ptrUInt16T
ptrUInt32T
ptrUInt64T
ptrFloat32
ptrFloat64
ptrString
ptrTime
ptrIPv4
ptrIPv6
)
var arrayBaseTypes = map[interface{}]reflect.Type{
int8(0): reflect.ValueOf(int8(0)).Type(),
int16(0): reflect.ValueOf(int16(0)).Type(),
int32(0): reflect.ValueOf(int32(0)).Type(),
int64(0): reflect.ValueOf(int64(0)).Type(),
uint8(0): reflect.ValueOf(uint8(0)).Type(),
uint16(0): reflect.ValueOf(uint16(0)).Type(),
uint32(0): reflect.ValueOf(uint32(0)).Type(),
uint64(0): reflect.ValueOf(uint64(0)).Type(),
float32(0): reflect.ValueOf(float32(0)).Type(),
float64(0): reflect.ValueOf(float64(0)).Type(),
string(""): reflect.ValueOf(string("")).Type(),
time.Time{}: reflect.ValueOf(time.Time{}).Type(),
IPv4{}: reflect.ValueOf(net.IPv4zero).Type(),
IPv6{}: reflect.ValueOf(net.IPv6unspecified).Type(),
// nullable
ptrInt8T: reflect.PtrTo(reflect.ValueOf(int8(0)).Type()),
ptrInt16T: reflect.PtrTo(reflect.ValueOf(int16(0)).Type()),
ptrInt32T: reflect.PtrTo(reflect.ValueOf(int32(0)).Type()),
ptrInt64T: reflect.PtrTo(reflect.ValueOf(int64(0)).Type()),
ptrUInt8T: reflect.PtrTo(reflect.ValueOf(uint8(0)).Type()),
ptrUInt16T: reflect.PtrTo(reflect.ValueOf(uint16(0)).Type()),
ptrUInt32T: reflect.PtrTo(reflect.ValueOf(uint32(0)).Type()),
ptrUInt64T: reflect.PtrTo(reflect.ValueOf(uint64(0)).Type()),
ptrFloat32: reflect.PtrTo(reflect.ValueOf(float32(0)).Type()),
ptrFloat64: reflect.PtrTo(reflect.ValueOf(float64(0)).Type()),
ptrString: reflect.PtrTo(reflect.ValueOf(string("")).Type()),
ptrTime: reflect.PtrTo(reflect.ValueOf(time.Time{}).Type()),
ptrIPv4: reflect.PtrTo(reflect.ValueOf(net.IPv4zero).Type()),
ptrIPv6: reflect.PtrTo(reflect.ValueOf(net.IPv6unspecified).Type()),
}
type base struct {
name, chType string
valueOf reflect.Value
}
func (base *base) Name() string {
return base.name
}
func (base *base) CHType() string {
return base.chType
}
func (base *base) ScanType() reflect.Type {
return base.valueOf.Type()
}
func (base *base) defaultValue() interface{} {
return base.valueOf.Interface()
}
func (base *base) String() string {
return fmt.Sprintf("%s (%s)", base.name, base.chType)
}
func (base *base) Depth() int {
return 0
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/date.go
Generated Vendored
+84
View File
@@ -0,0 +1,84 @@
package column
import (
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Date struct {
base
Timezone *time.Location
offset int64
}
func (dt *Date) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
sec, err := decoder.Int16()
if err != nil {
return nil, err
}
return time.Unix(int64(sec)*24*3600-dt.offset, 0).In(dt.Timezone), nil
}
func (dt *Date) Write(encoder *binary.Encoder, v interface{}) error {
var timestamp int64
switch value := v.(type) {
case time.Time:
_, offset := value.Zone()
timestamp = value.Unix() + int64(offset)
case int16:
return encoder.Int16(value)
case int32:
timestamp = int64(value) + dt.offset
case uint32:
timestamp = int64(value) + dt.offset
case uint64:
timestamp = int64(value) + dt.offset
case int64:
timestamp = value + dt.offset
case string:
var err error
timestamp, err = dt.parse(value)
if err != nil {
return err
}
// this relies on Nullable never sending nil values through
case *time.Time:
_, offset := value.Zone()
timestamp = (*value).Unix() + int64(offset)
case *int16:
return encoder.Int16(*value)
case *int32:
timestamp = int64(*value) + dt.offset
case *int64:
timestamp = *value + dt.offset
case *string:
var err error
timestamp, err = dt.parse(*value)
if err != nil {
return err
}
default:
return &ErrUnexpectedType{
T: v,
Column: dt,
}
}
return encoder.Int16(int16(timestamp / 24 / 3600))
}
func (dt *Date) parse(value string) (int64, error) {
tv, err := time.Parse("2006-01-02", value)
if err != nil {
return 0, err
}
return time.Date(
time.Time(tv).Year(),
time.Time(tv).Month(),
time.Time(tv).Day(),
0, 0, 0, 0, time.UTC,
).Unix(), nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime.go
Generated Vendored
+87
View File
@@ -0,0 +1,87 @@
package column
import (
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type DateTime struct {
base
Timezone *time.Location
}
func (dt *DateTime) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
sec, err := decoder.Int32()
if err != nil {
return nil, err
}
return time.Unix(int64(sec), 0).In(dt.Timezone), nil
}
func (dt *DateTime) Write(encoder *binary.Encoder, v interface{}) error {
var timestamp int64
switch value := v.(type) {
case time.Time:
if !value.IsZero() {
timestamp = value.Unix()
}
case int16:
timestamp = int64(value)
case int32:
timestamp = int64(value)
case uint32:
timestamp = int64(value)
case uint64:
timestamp = int64(value)
case int64:
timestamp = value
case string:
var err error
timestamp, err = dt.parse(value)
if err != nil {
return err
}
case *time.Time:
if value != nil && !(*value).IsZero() {
timestamp = (*value).Unix()
}
case *int16:
timestamp = int64(*value)
case *int32:
timestamp = int64(*value)
case *int64:
timestamp = *value
case *string:
var err error
timestamp, err = dt.parse(*value)
if err != nil {
return err
}
default:
return &ErrUnexpectedType{
T: v,
Column: dt,
}
}
return encoder.Int32(int32(timestamp))
}
func (dt *DateTime) parse(value string) (int64, error) {
tv, err := time.Parse("2006-01-02 15:04:05", value)
if err != nil {
return 0, err
}
return time.Date(
time.Time(tv).Year(),
time.Time(tv).Month(),
time.Time(tv).Day(),
time.Time(tv).Hour(),
time.Time(tv).Minute(),
time.Time(tv).Second(),
0, time.Local, //use local timzone when insert into clickhouse
).Unix(), nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/datetime64.go
Generated Vendored
+100
View File
@@ -0,0 +1,100 @@
package column
import (
"math"
"strconv"
"strings"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type DateTime64 struct {
base
Timezone *time.Location
}
func (dt *DateTime64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
value, err := decoder.Int64()
if err != nil {
return nil, err
}
precision, err := dt.getPrecision()
if err != nil {
return nil, err
}
var nano int64
if precision < 19 {
nano = value * int64(math.Pow10(9-precision))
}
sec := nano / int64(10e8)
nsec := nano - sec*10e8
return time.Unix(sec, nsec).In(dt.Timezone), nil
}
func (dt *DateTime64) Write(encoder *binary.Encoder, v interface{}) error {
var timestamp int64
switch value := v.(type) {
case time.Time:
if !value.IsZero() {
timestamp = value.UnixNano()
}
case uint64:
timestamp = int64(value)
case int64:
timestamp = value
case string:
var err error
timestamp, err = dt.parse(value)
if err != nil {
return err
}
case *time.Time:
if value != nil && !(*value).IsZero() {
timestamp = (*value).UnixNano()
}
case *int64:
timestamp = *value
case *string:
var err error
timestamp, err = dt.parse(*value)
if err != nil {
return err
}
default:
return &ErrUnexpectedType{
T: v,
Column: dt,
}
}
precision, err := dt.getPrecision()
if err != nil {
return err
}
timestamp = timestamp / int64(math.Pow10(9-precision))
return encoder.Int64(timestamp)
}
func (dt *DateTime64) parse(value string) (int64, error) {
tv, err := time.Parse("2006-01-02 15:04:05.999", value)
if err != nil {
return 0, err
}
return tv.UnixNano(), nil
}
func (dt *DateTime64) getPrecision() (int, error) {
dtParams := dt.base.chType[11 : len(dt.base.chType)-1]
precision, err := strconv.Atoi(strings.Split(dtParams, ",")[0])
if err != nil {
return 0, err
}
return precision, nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/decimal.go
Generated Vendored
+377
View File
@@ -0,0 +1,377 @@
package column
import (
b "encoding/binary"
"errors"
"fmt"
"math"
"reflect"
"strconv"
"strings"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
// Table of powers of 10 for fast casting from floating types to decimal type
// representations.
var factors10 = []float64{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13,
1e14, 1e15, 1e16, 1e17, 1e18,
}
// Decimal represents Decimal(P, S) ClickHouse. Decimal is represented as
// integral. Also floating-point types are supported for query parameters.
//
// Since there is no support for int128 in Golang, decimals with precision 19
// through 38 are represented as 16 little-endian bytes.
type Decimal struct {
base
nobits int // its domain is {32, 64}
precision int
scale int
}
func (d *Decimal) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
switch d.nobits {
case 32:
return decoder.Int32()
case 64:
return decoder.Int64()
case 128:
return decoder.Decimal128()
default:
return nil, errors.New("unachievable execution path")
}
}
func (d *Decimal) Write(encoder *binary.Encoder, v interface{}) error {
switch d.nobits {
case 32:
return d.write32(encoder, v)
case 64:
return d.write64(encoder, v)
case 128:
return d.write128(encoder, v)
default:
return errors.New("unachievable execution path")
}
}
func (d *Decimal) float2int32(floating float64) int32 {
fixed := int32(floating * factors10[d.scale])
return fixed
}
func (d *Decimal) float2int64(floating float64) int64 {
fixed := int64(floating * factors10[d.scale])
return fixed
}
func (d *Decimal) write32(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int8:
return encoder.Int32(int32(v))
case int16:
return encoder.Int32(int32(v))
case int32:
return encoder.Int32(int32(v))
case int64:
if v > math.MaxInt32 || v < math.MinInt32 {
return errors.New("overflow when narrowing type conversion from int64 to int32")
}
return encoder.Int32(int32(v))
case uint8:
return encoder.Int32(int32(v))
case uint16:
return encoder.Int32(int32(v))
case uint32:
if v > math.MaxInt32 {
return errors.New("overflow when narrowing type conversion from uint32 to int32")
}
return encoder.Int32(int32(v))
case uint64:
if v > math.MaxInt32 {
return errors.New("overflow when narrowing type conversion from uint64 to int32")
}
return encoder.Int32(int32(v))
case float32:
fixed := d.float2int32(float64(v))
return encoder.Int32(fixed)
case float64:
fixed := d.float2int32(float64(v))
return encoder.Int32(fixed)
// this relies on Nullable never sending nil values through
case *int8:
return encoder.Int32(int32(*v))
case *int16:
return encoder.Int32(int32(*v))
case *int32:
return encoder.Int32(int32(*v))
case *int64:
if *v > math.MaxInt32 || *v < math.MinInt32 {
return errors.New("overflow when narrowing type conversion from int64 to int32")
}
return encoder.Int32(int32(*v))
case *uint8:
return encoder.Int32(int32(*v))
case *uint16:
return encoder.Int32(int32(*v))
case *uint32:
if *v > math.MaxInt32 {
return errors.New("overflow when narrowing type conversion from uint34 to int32")
}
return encoder.Int32(int32(*v))
case *uint64:
if *v > math.MaxInt32 {
return errors.New("overflow when narrowing type conversion from uint64 to int32")
}
return encoder.Int32(int32(*v))
case *float32:
fixed := d.float2int32(float64(*v))
return encoder.Int32(fixed)
case *float64:
fixed := d.float2int32(float64(*v))
return encoder.Int32(fixed)
}
return &ErrUnexpectedType{
T: v,
Column: d,
}
}
func (d *Decimal) write64(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int:
return encoder.Int64(int64(v))
case int8:
return encoder.Int64(int64(v))
case int16:
return encoder.Int64(int64(v))
case int32:
return encoder.Int64(int64(v))
case int64:
return encoder.Int64(int64(v))
case uint8:
return encoder.Int64(int64(v))
case uint16:
return encoder.Int64(int64(v))
case uint32:
return encoder.Int64(int64(v))
case uint64:
if v > math.MaxInt64 {
return errors.New("overflow when narrowing type conversion from uint64 to int64")
}
return encoder.Int64(int64(v))
case float32:
fixed := d.float2int64(float64(v))
return encoder.Int64(fixed)
case float64:
fixed := d.float2int64(float64(v))
return encoder.Int64(fixed)
// this relies on Nullable never sending nil values through
case *int:
return encoder.Int64(int64(*v))
case *int8:
return encoder.Int64(int64(*v))
case *int16:
return encoder.Int64(int64(*v))
case *int32:
return encoder.Int64(int64(*v))
case *int64:
return encoder.Int64(int64(*v))
case *uint8:
return encoder.Int64(int64(*v))
case *uint16:
return encoder.Int64(int64(*v))
case *uint32:
return encoder.Int64(int64(*v))
case *uint64:
if *v > math.MaxInt64 {
return errors.New("overflow when narrowing type conversion from uint64 to int64")
}
return encoder.Int64(int64(*v))
case *float32:
fixed := d.float2int64(float64(*v))
return encoder.Int64(fixed)
case *float64:
fixed := d.float2int64(float64(*v))
return encoder.Int64(fixed)
}
return &ErrUnexpectedType{
T: v,
Column: d,
}
}
// Turns an int64 into 16 little-endian bytes.
func int64ToDecimal128(v int64) []byte {
bytes := make([]byte, 16)
b.LittleEndian.PutUint64(bytes[:8], uint64(v))
sign := 0
if v < 0 {
sign = -1
}
b.LittleEndian.PutUint64(bytes[8:], uint64(sign))
return bytes
}
// Turns a uint64 into 16 little-endian bytes.
func uint64ToDecimal128(v uint64) []byte {
bytes := make([]byte, 16)
b.LittleEndian.PutUint64(bytes[:8], uint64(v))
return bytes
}
func (d *Decimal) write128(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int:
return encoder.Decimal128(int64ToDecimal128(int64(v)))
case int8:
return encoder.Decimal128(int64ToDecimal128(int64(v)))
case int16:
return encoder.Decimal128(int64ToDecimal128(int64(v)))
case int32:
return encoder.Decimal128(int64ToDecimal128(int64(v)))
case int64:
return encoder.Decimal128(int64ToDecimal128(v))
case uint8:
return encoder.Decimal128(uint64ToDecimal128(uint64(v)))
case uint16:
return encoder.Decimal128(uint64ToDecimal128(uint64(v)))
case uint32:
return encoder.Decimal128(uint64ToDecimal128(uint64(v)))
case uint64:
return encoder.Decimal128(uint64ToDecimal128(v))
case float32:
fixed := d.float2int64(float64(v))
return encoder.Decimal128(int64ToDecimal128(fixed))
case float64:
fixed := d.float2int64(float64(v))
return encoder.Decimal128(int64ToDecimal128(fixed))
case []byte:
if len(v) != 16 {
return errors.New("expected 16 bytes")
}
return encoder.Decimal128(v)
// this relies on Nullable never sending nil values through
case *int:
return encoder.Decimal128(int64ToDecimal128(int64(*v)))
case *int8:
return encoder.Decimal128(int64ToDecimal128(int64(*v)))
case *int16:
return encoder.Decimal128(int64ToDecimal128(int64(*v)))
case *int32:
return encoder.Decimal128(int64ToDecimal128(int64(*v)))
case *int64:
return encoder.Decimal128(int64ToDecimal128(*v))
case *uint8:
return encoder.Decimal128(uint64ToDecimal128(uint64(*v)))
case *uint16:
return encoder.Decimal128(uint64ToDecimal128(uint64(*v)))
case *uint32:
return encoder.Decimal128(uint64ToDecimal128(uint64(*v)))
case *uint64:
return encoder.Decimal128(uint64ToDecimal128(*v))
case *float32:
fixed := d.float2int64(float64(*v))
return encoder.Decimal128(int64ToDecimal128(fixed))
case *float64:
fixed := d.float2int64(float64(*v))
return encoder.Decimal128(int64ToDecimal128(fixed))
case *[]byte:
if len(*v) != 16 {
return errors.New("expected 16 bytes")
}
return encoder.Decimal128(*v)
}
return &ErrUnexpectedType{
T: v,
Column: d,
}
}
func parseDecimal(name, chType string) (Column, error) {
switch {
case len(chType) < 12:
fallthrough
case !strings.HasPrefix(chType, "Decimal"):
fallthrough
case chType[7] != '(':
fallthrough
case chType[len(chType)-1] != ')':
return nil, fmt.Errorf("invalid Decimal format: '%s'", chType)
}
var params = strings.Split(chType[8:len(chType)-1], ",")
if len(params) != 2 {
return nil, fmt.Errorf("invalid Decimal format: '%s'", chType)
}
params[0] = strings.TrimSpace(params[0])
params[1] = strings.TrimSpace(params[1])
var err error
var decimal = &Decimal{
base: base{
name: name,
chType: chType,
},
}
if decimal.precision, err = strconv.Atoi(params[0]); err != nil {
return nil, fmt.Errorf("'%s' is not Decimal type: %s", chType, err)
} else if decimal.precision < 1 {
return nil, errors.New("wrong precision of Decimal type")
}
if decimal.scale, err = strconv.Atoi(params[1]); err != nil {
return nil, fmt.Errorf("'%s' is not Decimal type: %s", chType, err)
} else if decimal.scale < 0 || decimal.scale > decimal.precision {
return nil, errors.New("wrong scale of Decimal type")
}
switch {
case decimal.precision <= 9:
decimal.nobits = 32
decimal.valueOf = columnBaseTypes[int32(0)]
case decimal.precision <= 18:
decimal.nobits = 64
decimal.valueOf = columnBaseTypes[int64(0)]
case decimal.precision <= 38:
decimal.nobits = 128
decimal.valueOf = reflect.ValueOf([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})
default:
return nil, errors.New("precision of Decimal exceeds max bound")
}
return decimal, nil
}
func (d *Decimal) GetPrecision() int {
return d.precision
}
func (d *Decimal) GetScale() int {
return d.scale
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/enum.go
Generated Vendored
+175
View File
@@ -0,0 +1,175 @@
package column
import (
"fmt"
"strconv"
"strings"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Enum struct {
iv map[string]interface{}
vi map[interface{}]string
base
baseType interface{}
}
func (enum *Enum) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
var (
err error
ident interface{}
)
switch enum.baseType.(type) {
case int16:
if ident, err = decoder.Int16(); err != nil {
return nil, err
}
default:
if ident, err = decoder.Int8(); err != nil {
return nil, err
}
}
if ident, found := enum.vi[ident]; found || isNull {
return ident, nil
}
return nil, fmt.Errorf("invalid Enum value: %v", ident)
}
func (enum *Enum) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case string:
return enum.encodeFromString(v, encoder)
case uint8:
if _, ok := enum.baseType.(int8); ok {
return encoder.Int8(int8(v))
}
case int8:
if _, ok := enum.baseType.(int8); ok {
return encoder.Int8(v)
}
case uint16:
if _, ok := enum.baseType.(int16); ok {
return encoder.Int16(int16(v))
}
case int16:
if _, ok := enum.baseType.(int16); ok {
return encoder.Int16(v)
}
case int64:
switch enum.baseType.(type) {
case int8:
return encoder.Int8(int8(v))
case int16:
return encoder.Int16(int16(v))
}
// nullable enums
case *string:
return enum.encodeFromString(*v, encoder)
case *uint8:
if _, ok := enum.baseType.(int8); ok {
return encoder.Int8(int8(*v))
}
case *int8:
if _, ok := enum.baseType.(int8); ok {
return encoder.Int8(*v)
}
case *uint16:
if _, ok := enum.baseType.(int16); ok {
return encoder.Int16(int16(*v))
}
case *int16:
if _, ok := enum.baseType.(int16); ok {
return encoder.Int16(*v)
}
case *int64:
switch enum.baseType.(type) {
case int8:
return encoder.Int8(int8(*v))
case int16:
return encoder.Int16(int16(*v))
}
}
return &ErrUnexpectedType{
T: v,
Column: enum,
}
}
func (enum *Enum) encodeFromString(v string, encoder *binary.Encoder) error {
ident, found := enum.iv[v]
if !found {
return fmt.Errorf("invalid Enum ident: %s", v)
}
switch ident := ident.(type) {
case int8:
return encoder.Int8(ident)
case int16:
return encoder.Int16(ident)
default:
return &ErrUnexpectedType{
T: ident,
Column: enum,
}
}
}
func (enum *Enum) defaultValue() interface{} {
return enum.baseType
}
func parseEnum(name, chType string) (*Enum, error) {
var (
data string
isEnum16 bool
)
if len(chType) < 8 {
return nil, fmt.Errorf("invalid Enum format: %s", chType)
}
switch {
case strings.HasPrefix(chType, "Enum8"):
data = chType[6:]
case strings.HasPrefix(chType, "Enum16"):
data = chType[7:]
isEnum16 = true
default:
return nil, fmt.Errorf("'%s' is not Enum type", chType)
}
enum := Enum{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[string("")],
},
iv: make(map[string]interface{}),
vi: make(map[interface{}]string),
}
for _, block := range strings.Split(data[:len(data)-1], ",") {
parts := strings.Split(block, "=")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid Enum format: %s", chType)
}
var (
ident = strings.TrimSpace(parts[0])
value, err = strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 16)
)
if err != nil {
return nil, fmt.Errorf("invalid Enum value: %v", chType)
}
{
var (
ident = ident[1 : len(ident)-1]
value interface{} = int16(value)
)
if !isEnum16 {
value = int8(value.(int16))
}
if enum.baseType == nil {
enum.baseType = value
}
enum.iv[ident] = value
enum.vi[value] = ident
}
}
return &enum, nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/fixed_string.go
Generated Vendored
+71
View File
@@ -0,0 +1,71 @@
package column
import (
"encoding"
"fmt"
"reflect"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type FixedString struct {
base
len int
scanType reflect.Type
}
func (str *FixedString) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Fixed(str.len)
if err != nil {
return "", err
}
return string(v), nil
}
func (str *FixedString) Write(encoder *binary.Encoder, v interface{}) error {
var fixedString []byte
switch v := v.(type) {
case string:
fixedString = binary.Str2Bytes(v)
case []byte:
fixedString = v
case encoding.BinaryMarshaler:
bytes, err := v.MarshalBinary()
if err != nil {
return err
}
fixedString = bytes
default:
return &ErrUnexpectedType{
T: v,
Column: str,
}
}
switch {
case len(fixedString) > str.len:
return fmt.Errorf("too large value '%s' (expected %d, got %d)", fixedString, str.len, len(fixedString))
case len(fixedString) < str.len:
tmp := make([]byte, str.len)
copy(tmp, fixedString)
fixedString = tmp
}
if _, err := encoder.Write(fixedString); err != nil {
return err
}
return nil
}
func parseFixedString(name, chType string) (*FixedString, error) {
var strLen int
if _, err := fmt.Sscanf(chType, "FixedString(%d)", &strLen); err != nil {
return nil, err
}
return &FixedString{
base: base{
name: name,
chType: chType,
valueOf: columnBaseTypes[string("")],
},
len: strLen,
}, nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/float32.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Float32 struct{ base }
func (Float32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Float32()
if err != nil {
return float32(0), err
}
return v, nil
}
func (float *Float32) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case float32:
return encoder.Float32(v)
case float64:
return encoder.Float32(float32(v))
// this relies on Nullable never sending nil values through
case *float32:
return encoder.Float32(*v)
case *float64:
return encoder.Float32(float32(*v))
}
return &ErrUnexpectedType{
T: v,
Column: float,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/float64.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Float64 struct{ base }
func (Float64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Float64()
if err != nil {
return float64(0), err
}
return v, nil
}
func (float *Float64) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case float32:
return encoder.Float64(float64(v))
case float64:
return encoder.Float64(v)
// this relies on Nullable never sending nil values through
case *float32:
return encoder.Float64(float64(*v))
case *float64:
return encoder.Float64(*v)
}
return &ErrUnexpectedType{
T: v,
Column: float,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/int16.go
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Int16 struct{ base }
func (Int16) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Int16()
if err != nil {
return int16(0), err
}
return v, nil
}
func (i *Int16) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int16:
return encoder.Int16(v)
case int64:
return encoder.Int16(int16(v))
case int:
return encoder.Int16(int16(v))
// this relies on Nullable never sending nil values through
case *int16:
return encoder.Int16(*v)
case *int64:
return encoder.Int16(int16(*v))
case *int:
return encoder.Int16(int16(*v))
}
return &ErrUnexpectedType{
T: v,
Column: i,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/int32.go
Generated Vendored
+39
View File
@@ -0,0 +1,39 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Int32 struct{ base }
func (Int32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Int32()
if err != nil {
return int32(0), err
}
return v, nil
}
func (i *Int32) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int32:
return encoder.Int32(v)
case int64:
return encoder.Int32(int32(v))
case int:
return encoder.Int32(int32(v))
// this relies on Nullable never sending nil values through
case *int32:
return encoder.Int32(*v)
case *int64:
return encoder.Int32(int32(*v))
case *int:
return encoder.Int32(int32(*v))
}
return &ErrUnexpectedType{
T: v,
Column: i,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/int64.go
Generated Vendored
+40
View File
@@ -0,0 +1,40 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Int64 struct{ base }
func (Int64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Int64()
if err != nil {
return int64(0), err
}
return v, nil
}
func (i *Int64) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int:
return encoder.Int64(int64(v))
case int64:
return encoder.Int64(v)
case []byte:
if _, err := encoder.Write(v); err != nil {
return err
}
return nil
// this relies on Nullable never sending nil values through
case *int:
return encoder.Int64(int64(*v))
case *int64:
return encoder.Int64(*v)
}
return &ErrUnexpectedType{
T: v,
Column: i,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/int8.go
Generated Vendored
+49
View File
@@ -0,0 +1,49 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Int8 struct{ base }
func (Int8) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Int8()
if err != nil {
return int8(0), err
}
return v, nil
}
func (i *Int8) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case int8:
return encoder.Int8(v)
case int64:
return encoder.Int8(int8(v))
case int:
return encoder.Int8(int8(v))
case bool:
if v {
return encoder.Int8(int8(1))
}
return encoder.Int8(int8(0))
// this relies on Nullable never sending nil values through
case *int8:
return encoder.Int8(*v)
case *int64:
return encoder.Int8(int8(*v))
case *int:
return encoder.Int8(int8(*v))
case *bool:
if *v {
return encoder.Int8(int8(1))
}
return encoder.Int8(int8(0))
}
return &ErrUnexpectedType{
T: v,
Column: i,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/ip.go
Generated Vendored
+84
View File
@@ -0,0 +1,84 @@
/*
IP type supporting for clickhouse as FixedString(16)
*/
package column
import (
"database/sql/driver"
"errors"
"net"
"strings"
)
var (
errInvalidScanType = errors.New("Invalid scan types")
errInvalidScanValue = errors.New("Invalid scan value")
)
// IP column type
type IP net.IP
// Value implements the driver.Valuer interface, json field interface
// Alignment on the right side
func (ip IP) Value() (driver.Value, error) {
return ip.MarshalBinary()
}
func (ip IP) MarshalBinary() ([]byte, error) {
if len(ip) < 16 {
var (
buff = make([]byte, 16)
j = 0
)
for i := 16 - len(ip); i < 16; i++ {
buff[i] = ip[j]
j++
}
for i := 0; i < 16-len(ip); i++ {
buff[i] = '\x00'
}
if len(ip) == 4 {
buff[11] = '\xff'
buff[10] = '\xff'
}
return buff, nil
}
return []byte(ip), nil
}
// Scan implements the driver.Valuer interface, json field interface
func (ip *IP) Scan(value interface{}) (err error) {
switch v := value.(type) {
case []byte:
if len(v) == 4 || len(v) == 16 {
*ip = IP(v)
} else {
err = errInvalidScanValue
}
case string:
if v == "" {
err = errInvalidScanValue
return
}
if (len(v) == 4 || len(v) == 16) && !strings.Contains(v, ".") && !strings.Contains(v, ":"){
*ip = IP([]byte(v))
return
}
if strings.Contains(v, ":") {
*ip = IP(net.ParseIP(v))
return
}
*ip = IP(net.ParseIP(v).To4())
case net.IP:
*ip = IP(v)
default:
err = errInvalidScanType
}
return
}
// String implements the fmt.Stringer interface
func (ip IP) String() string {
return net.IP(ip).String()
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv4.go
Generated Vendored
+54
View File
@@ -0,0 +1,54 @@
package column
import (
"net"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type IPv4 struct {
base
}
func (*IPv4) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Fixed(4)
if err != nil {
return nil, err
}
return net.IPv4(v[3], v[2], v[1], v[0]), nil
}
func (ip *IPv4) Write(encoder *binary.Encoder, v interface{}) error {
var netIP net.IP
switch v.(type) {
case string:
netIP = net.ParseIP(v.(string))
case net.IP:
netIP = v.(net.IP)
case *net.IP:
netIP = *(v.(*net.IP))
default:
return &ErrUnexpectedType{
T: v,
Column: ip,
}
}
if netIP == nil {
return &ErrUnexpectedType{
T: v,
Column: ip,
}
}
ip4 := netIP.To4()
if ip4 == nil {
return &ErrUnexpectedType{
T: v,
Column: ip,
}
}
if _, err := encoder.Write([]byte{ip4[3], ip4[2], ip4[1], ip4[0]}); err != nil {
return err
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/ipv6.go
Generated Vendored
+47
View File
@@ -0,0 +1,47 @@
package column
import (
"net"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type IPv6 struct {
base
}
func (*IPv6) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.Fixed(16)
if err != nil {
return nil, err
}
return net.IP(v), nil
}
func (ip *IPv6) Write(encoder *binary.Encoder, v interface{}) error {
var netIP net.IP
switch v.(type) {
case string:
netIP = net.ParseIP(v.(string))
case net.IP:
netIP = v.(net.IP)
case *net.IP:
netIP = *(v.(*net.IP))
default:
return &ErrUnexpectedType{
T: v,
Column: ip,
}
}
if netIP == nil {
return &ErrUnexpectedType{
T: v,
Column: ip,
}
}
if _, err := encoder.Write([]byte(netIP.To16())); err != nil {
return err
}
return nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable.go
Generated Vendored
+96
View File
@@ -0,0 +1,96 @@
package column
import (
"fmt"
"reflect"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type Nullable struct {
base
column Column
}
func (null *Nullable) ScanType() reflect.Type {
return reflect.PtrTo(null.column.ScanType())
}
func (null *Nullable) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
return null.column.Read(decoder, isNull)
}
func (null *Nullable) Write(encoder *binary.Encoder, v interface{}) error {
return nil
}
func (null *Nullable) ReadNull(decoder *binary.Decoder, rows int) (_ []interface{}, err error) {
var (
isNull byte
value interface{}
nulls = make([]byte, rows)
values = make([]interface{}, rows)
)
for i := 0; i < rows; i++ {
if isNull, err = decoder.ReadByte(); err != nil {
return nil, err
}
nulls[i] = isNull
}
for i, isNull := range nulls {
switch value, err = null.column.Read(decoder, isNull != 0); true {
case err != nil:
return nil, err
case isNull == 0:
values[i] = value
default:
values[i] = nil
}
}
return values, nil
}
func (null *Nullable) WriteNull(nulls, encoder *binary.Encoder, v interface{}) error {
if isNil(v) {
if _, err := nulls.Write([]byte{1}); err != nil {
return err
}
return null.column.Write(encoder, null.column.defaultValue())
}
if _, err := nulls.Write([]byte{0}); err != nil {
return err
}
return null.column.Write(encoder, v)
}
func parseNullable(name, chType string, timezone *time.Location) (*Nullable, error) {
if len(chType) < 14 {
return nil, fmt.Errorf("invalid Nullable column type: %s", chType)
}
column, err := Factory(name, chType[9:][:len(chType)-10], timezone)
if err != nil {
return nil, fmt.Errorf("Nullable(T): %v", err)
}
return &Nullable{
base: base{
name: name,
chType: chType,
},
column: column,
}, nil
}
func (null *Nullable) GetColumn() Column {
return null.column
}
func isNil(v interface{}) bool {
if v == nil {
return true
}
switch val := reflect.ValueOf(v); val.Type().Kind() {
case reflect.Array, reflect.Chan, reflect.Map, reflect.Ptr, reflect.Slice:
return val.IsNil()
}
return false
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/nullable_appender.go
Generated Vendored
+173
View File
@@ -0,0 +1,173 @@
// DANGER! This code was autogenerated from template by clickhouse-go/lib/codegen/nullable_appender.
// You shouldn't change it manually.
// For more info check clickhouse-go/lib/codegen/nullable_appender/main.go
package column
import (
"fmt"
"net"
"reflect"
"time"
)
var nullableAppender = map[string]func(v interface{}, slice reflect.Value) (reflect.Value, error){
"*int8": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(int8)
if !ok {
return slice, fmt.Errorf("cannot assert to type int8")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *int8
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*int16": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(int16)
if !ok {
return slice, fmt.Errorf("cannot assert to type int16")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *int16
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*int32": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(int32)
if !ok {
return slice, fmt.Errorf("cannot assert to type int32")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *int32
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*int64": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(int64)
if !ok {
return slice, fmt.Errorf("cannot assert to type int64")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *int64
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*uint8": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(uint8)
if !ok {
return slice, fmt.Errorf("cannot assert to type uint8")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *uint8
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*uint16": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(uint16)
if !ok {
return slice, fmt.Errorf("cannot assert to type uint16")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *uint16
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*uint32": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(uint32)
if !ok {
return slice, fmt.Errorf("cannot assert to type uint32")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *uint32
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*uint64": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(uint64)
if !ok {
return slice, fmt.Errorf("cannot assert to type uint64")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *uint64
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*float32": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(float32)
if !ok {
return slice, fmt.Errorf("cannot assert to type float32")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *float32
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*float64": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(float64)
if !ok {
return slice, fmt.Errorf("cannot assert to type float64")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *float64
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*string": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(string)
if !ok {
return slice, fmt.Errorf("cannot assert to type string")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *string
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*time.Time": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(time.Time)
if !ok {
return slice, fmt.Errorf("cannot assert to type time.Time")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *time.Time
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
"*net.IP": func(v interface{}, slice reflect.Value) (reflect.Value, error) {
if v != nil {
v, ok := v.(net.IP)
if !ok {
return slice, fmt.Errorf("cannot assert to type net.IP")
}
return reflect.Append(slice, reflect.ValueOf(&v)), nil
}
var vNil *net.IP
return reflect.Append(slice, reflect.ValueOf(vNil)), nil
},
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/string.go
Generated Vendored
+35
View File
@@ -0,0 +1,35 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type String struct{ base }
func (String) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.String()
if err != nil {
return "", err
}
return v, nil
}
func (str *String) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case string:
return encoder.String(v)
case []byte:
return encoder.RawString(v)
// this relies on Nullable never sending nil values through
case *string:
return encoder.String(*v)
case *[]byte:
return encoder.RawString(*v)
}
return &ErrUnexpectedType{
T: v,
Column: str,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/tuple.go
Generated Vendored
+110
View File
@@ -0,0 +1,110 @@
package column
import (
"fmt"
"github.com/ClickHouse/clickhouse-go/lib/binary"
"reflect"
"strconv"
"time"
)
type Tuple struct {
base
columns []Column
}
func (tuple *Tuple) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
return nil, fmt.Errorf("do not use Read method for Tuple(T) column")
}
func (tuple *Tuple) ReadTuple(decoder *binary.Decoder, rows int) ([]interface{}, error) {
var values = make([][]interface{}, rows)
for _, c := range tuple.columns {
switch column := c.(type) {
case *Array:
cols, err := column.ReadArray(decoder, rows)
if err != nil {
return nil, err
}
for i := 0; i < rows; i++ {
values[i] = append(values[i], cols[i])
}
case *Nullable:
cols, err := column.ReadNull(decoder, rows)
if err != nil {
return nil, err
}
for i := 0; i < rows; i++ {
values[i] = append(values[i], cols[i])
}
case *Tuple:
cols, err := column.ReadTuple(decoder, rows)
if err != nil {
return nil, err
}
for i := 0; i < rows; i++ {
values[i] = append(values[i], cols[i])
}
default:
for i := 0; i < rows; i++ {
value, err := c.Read(decoder, false)
if err != nil {
return nil, err
}
values[i] = append(values[i], value)
}
}
}
var ret = make([]interface{}, rows)
for i := range values {
ret[i] = values[i]
}
return ret, nil
}
func (tuple *Tuple) Write(encoder *binary.Encoder, v interface{}) (err error) {
return fmt.Errorf("unsupported Tuple(T) type [%T]", v)
}
func parseTuple(name, chType string, timezone *time.Location) (Column, error) {
var columnType = chType
chType = chType[6 : len(chType)-1]
var types []string
var last, diff int
for i, b := range chType + "," {
if b == '(' {
diff++
} else if b == ')' {
diff--
} else if b == ',' && diff == 0 {
types = append(types, chType[last:i])
last = i + 2
}
}
var columns = make([]Column, 0, len(types))
for i, chType := range types {
column, err := Factory(name+"."+strconv.Itoa(i+1), chType, timezone)
if err != nil {
return nil, fmt.Errorf("%s: %v", chType, err)
}
columns = append(columns, column)
}
return &Tuple{
base: base{
name: name,
chType: columnType,
valueOf: reflect.ValueOf([]interface{}{}),
},
columns: columns,
}, nil
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint16.go
Generated Vendored
+43
View File
@@ -0,0 +1,43 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type UInt16 struct{ base }
func (UInt16) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.UInt16()
if err != nil {
return uint16(0), err
}
return v, nil
}
func (u *UInt16) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case uint16:
return encoder.UInt16(v)
case int64:
return encoder.UInt16(uint16(v))
case uint64:
return encoder.UInt16(uint16(v))
case int:
return encoder.UInt16(uint16(v))
// this relies on Nullable never sending nil values through
case *uint16:
return encoder.UInt16(*v)
case *int64:
return encoder.UInt16(uint16(*v))
case *uint64:
return encoder.UInt16(uint16(*v))
case *int:
return encoder.UInt16(uint16(*v))
}
return &ErrUnexpectedType{
T: v,
Column: u,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint32.go
Generated Vendored
+43
View File
@@ -0,0 +1,43 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type UInt32 struct{ base }
func (UInt32) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.UInt32()
if err != nil {
return uint32(0), err
}
return v, nil
}
func (u *UInt32) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case uint32:
return encoder.UInt32(v)
case uint64:
return encoder.UInt32(uint32(v))
case int64:
return encoder.UInt32(uint32(v))
case int:
return encoder.UInt32(uint32(v))
// this relies on Nullable never sending nil values through
case *uint64:
return encoder.UInt32(uint32(*v))
case *uint32:
return encoder.UInt32(*v)
case *int64:
return encoder.UInt32(uint32(*v))
case *int:
return encoder.UInt32(uint32(*v))
}
return &ErrUnexpectedType{
T: v,
Column: u,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint64.go
Generated Vendored
+44
View File
@@ -0,0 +1,44 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type UInt64 struct{ base }
func (UInt64) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.UInt64()
if err != nil {
return uint64(0), err
}
return v, nil
}
func (u *UInt64) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case []byte:
if _, err := encoder.Write(v); err != nil {
return err
}
return nil
case uint64:
return encoder.UInt64(v)
case int64:
return encoder.UInt64(uint64(v))
case int:
return encoder.UInt64(uint64(v))
// this relies on Nullable never sending nil values through
case *uint64:
return encoder.UInt64(*v)
case *int64:
return encoder.UInt64(uint64(*v))
case *int:
return encoder.UInt64(uint64(*v))
}
return &ErrUnexpectedType{
T: v,
Column: u,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/uint8.go
Generated Vendored
+47
View File
@@ -0,0 +1,47 @@
package column
import (
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
type UInt8 struct{ base }
func (UInt8) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
v, err := decoder.UInt8()
if err != nil {
return uint8(0), err
}
return v, nil
}
func (u *UInt8) Write(encoder *binary.Encoder, v interface{}) error {
switch v := v.(type) {
case bool:
return encoder.Bool(v)
case uint8:
return encoder.UInt8(v)
case int64:
return encoder.UInt8(uint8(v))
case uint64:
return encoder.UInt8(uint8(v))
case int:
return encoder.UInt8(uint8(v))
// this relies on Nullable never sending nil values through
case *bool:
return encoder.Bool(*v)
case *uint8:
return encoder.UInt8(*v)
case *int64:
return encoder.UInt8(uint8(*v))
case *uint64:
return encoder.UInt8(uint8(*v))
case *int:
return encoder.UInt8(uint8(*v))
}
return &ErrUnexpectedType{
T: v,
Column: u,
}
}
vendor/github.com/ClickHouse/clickhouse-go/lib/column/uuid.go
Generated Vendored
+139
View File
@@ -0,0 +1,139 @@
package column
import (
"encoding/hex"
"errors"
"fmt"
"reflect"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
const (
UUIDLen = 16
NullUUID = "00000000-0000-0000-0000-000000000000"
)
var ErrInvalidUUIDFormat = errors.New("invalid UUID format")
type UUID struct {
base
scanType reflect.Type
}
func (*UUID) Read(decoder *binary.Decoder, isNull bool) (interface{}, error) {
src, err := decoder.Fixed(UUIDLen)
if err != nil {
return "", err
}
src = swap(src)
var uuid [36]byte
{
hex.Encode(uuid[:], src[:4])
uuid[8] = '-'
hex.Encode(uuid[9:13], src[4:6])
uuid[13] = '-'
hex.Encode(uuid[14:18], src[6:8])
uuid[18] = '-'
hex.Encode(uuid[19:23], src[8:10])
uuid[23] = '-'
hex.Encode(uuid[24:], src[10:])
}
return string(uuid[:]), nil
}
func (u *UUID) Write(encoder *binary.Encoder, v interface{}) (err error) {
var uuid []byte
switch v := v.(type) {
case string:
if uuid, err = uuid2bytes(v); err != nil {
return err
}
case []byte:
if len(v) != UUIDLen {
return fmt.Errorf("invalid raw UUID len '%s' (expected %d, got %d)", uuid, UUIDLen, len(uuid))
}
uuid = make([]byte, 16)
copy(uuid, v)
default:
return &ErrUnexpectedType{
T: v,
Column: u,
}
}
uuid = swap(uuid)
if _, err := encoder.Write(uuid); err != nil {
return err
}
return nil
}
func swap(src []byte) []byte {
_ = src[15]
src[0], src[7] = src[7], src[0]
src[1], src[6] = src[6], src[1]
src[2], src[5] = src[5], src[2]
src[3], src[4] = src[4], src[3]
src[8], src[15] = src[15], src[8]
src[9], src[14] = src[14], src[9]
src[10], src[13] = src[13], src[10]
src[11], src[12] = src[12], src[11]
return src
}
func uuid2bytes(str string) ([]byte, error) {
var uuid [16]byte
strLength := len(str)
if strLength == 0 {
str = NullUUID
} else if strLength != 36 {
return nil, ErrInvalidUUIDFormat
}
if str[8] != '-' || str[13] != '-' || str[18] != '-' || str[23] != '-' {
return nil, ErrInvalidUUIDFormat
}
for i, x := range [16]int{
0, 2, 4, 6,
9, 11, 14, 16,
19, 21, 24, 26,
28, 30, 32, 34,
} {
if v, ok := xtob(str[x], str[x+1]); !ok {
return nil, ErrInvalidUUIDFormat
} else {
uuid[i] = v
}
}
return uuid[:], nil
}
// xvalues returns the value of a byte as a hexadecimal digit or 255.
var xvalues = [256]byte{
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
}
// xtob converts hex characters x1 and x2 into a byte.
func xtob(x1, x2 byte) (byte, bool) {
b1 := xvalues[x1]
b2 := xvalues[x2]
return (b1 << 4) | b2, b1 != 255 && b2 != 255
}
vendor/github.com/ClickHouse/clickhouse-go/lib/data/block.go
Generated Vendored
+323
View File
@@ -0,0 +1,323 @@
package data
import (
"bytes"
"database/sql/driver"
"fmt"
"io"
"reflect"
"strings"
"github.com/ClickHouse/clickhouse-go/lib/binary"
"github.com/ClickHouse/clickhouse-go/lib/column"
)
type offset [][]int
type Block struct {
Values [][]interface{}
Columns []column.Column
NumRows uint64
NumColumns uint64
offsets []offset
buffers []*buffer
info blockInfo
}
func (block *Block) Copy() *Block {
return &Block{
Columns: block.Columns,
NumColumns: block.NumColumns,
info: block.info,
}
}
func (block *Block) ColumnNames() []string {
names := make([]string, 0, len(block.Columns))
for _, column := range block.Columns {
names = append(names, column.Name())
}
return names
}
func (block *Block) Read(serverInfo *ServerInfo, decoder *binary.Decoder) (err error) {
if serverInfo.Revision > 0 {
if err = block.info.read(decoder); err != nil {
return err
}
}
if block.NumColumns, err = decoder.Uvarint(); err != nil {
return err
}
if block.NumRows, err = decoder.Uvarint(); err != nil {
return err
}
block.Values = make([][]interface{}, block.NumColumns)
if block.NumRows > 10 {
for i := 0; i < int(block.NumColumns); i++ {
block.Values[i] = make([]interface{}, 0, block.NumRows)
}
}
for i := 0; i < int(block.NumColumns); i++ {
var (
value interface{}
columnName string
columnType string
)
if columnName, err = decoder.String(); err != nil {
return err
}
if columnType, err = decoder.String(); err != nil {
return err
}
c, err := column.Factory(columnName, columnType, serverInfo.Timezone)
if err != nil {
return err
}
block.Columns = append(block.Columns, c)
switch column := c.(type) {
case *column.Array:
if block.Values[i], err = column.ReadArray(decoder, int(block.NumRows)); err != nil {
return err
}
case *column.Nullable:
if block.Values[i], err = column.ReadNull(decoder, int(block.NumRows)); err != nil {
return err
}
case *column.Tuple:
if block.Values[i], err = column.ReadTuple(decoder, int(block.NumRows)); err != nil {
return err
}
default:
for row := 0; row < int(block.NumRows); row++ {
if value, err = column.Read(decoder, false); err != nil {
return err
}
block.Values[i] = append(block.Values[i], value)
}
}
}
return nil
}
func (block *Block) writeArray(col column.Column, value Value, num, level int) error {
if level > col.Depth() {
arrColumn, ok := col.(*column.Array)
if ok && strings.Contains(col.CHType(), "Nullable") {
return arrColumn.WriteNull(block.buffers[num].Offset, block.buffers[num].Column, value.Interface())
}
return col.Write(block.buffers[num].Column, value.Interface())
}
switch {
case value.Kind() == reflect.Slice:
if len(block.offsets[num]) < level {
block.offsets[num] = append(block.offsets[num], []int{value.Len()})
} else {
block.offsets[num][level-1] = append(
block.offsets[num][level-1],
block.offsets[num][level-1][len(block.offsets[num][level-1])-1]+value.Len(),
)
}
for i := 0; i < value.Len(); i++ {
if err := block.writeArray(col, value.Index(i), num, level+1); err != nil {
return err
}
}
default:
if err := col.Write(block.buffers[num].Column, value.Interface()); err != nil {
return err
}
}
return nil
}
func (block *Block) AppendRow(args []driver.Value) error {
if len(block.Columns) != len(args) {
return fmt.Errorf("block: expected %d arguments (columns: %s), got %d", len(block.Columns), strings.Join(block.ColumnNames(), ", "), len(args))
}
block.Reserve()
{
block.NumRows++
}
for num, c := range block.Columns {
switch column := c.(type) {
case *column.Array:
value := reflect.ValueOf(args[num])
if value.Kind() != reflect.Slice {
return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface())
}
if err := block.writeArray(c, newValue(value), num, 1); err != nil {
return err
}
case *column.Nullable:
if err := column.WriteNull(block.buffers[num].Offset, block.buffers[num].Column, args[num]); err != nil {
return err
}
default:
if err := column.Write(block.buffers[num].Column, args[num]); err != nil {
return err
}
}
}
return nil
}
func (block *Block) Reserve() {
if len(block.buffers) == 0 {
block.buffers = make([]*buffer, len(block.Columns))
block.offsets = make([]offset, len(block.Columns))
for i := 0; i < len(block.Columns); i++ {
var (
offsetBuffer = new(bytes.Buffer)
columnBuffer = new(bytes.Buffer)
)
block.buffers[i] = &buffer{
Offset: binary.NewEncoder(offsetBuffer),
Column: binary.NewEncoder(columnBuffer),
offsetBuffer: offsetBuffer,
columnBuffer: columnBuffer,
}
}
}
}
func (block *Block) Reset() {
block.NumRows = 0
block.NumColumns = 0
block.Values = block.Values[:0]
block.Columns = block.Columns[:0]
block.info.reset()
for _, buffer := range block.buffers {
buffer.reset()
}
{
block.offsets = nil
block.buffers = nil
}
}
func (block *Block) Write(serverInfo *ServerInfo, encoder *binary.Encoder) error {
if serverInfo.Revision > 0 {
if err := block.info.write(encoder); err != nil {
return err
}
}
if err := encoder.Uvarint(block.NumColumns); err != nil {
return err
}
encoder.Uvarint(block.NumRows)
defer func() {
block.NumRows = 0
for i := range block.offsets {
block.offsets[i] = offset{}
}
}()
for i, column := range block.Columns {
encoder.String(column.Name())
encoder.String(column.CHType())
if len(block.buffers) == len(block.Columns) {
for _, offsets := range block.offsets[i] {
for _, offset := range offsets {
if err := encoder.UInt64(uint64(offset)); err != nil {
return err
}
}
}
if _, err := block.buffers[i].WriteTo(encoder); err != nil {
return err
}
}
}
return nil
}
type blockInfo struct {
num1 uint64
isOverflows bool
num2 uint64
bucketNum int32
num3 uint64
}
func (info *blockInfo) reset() {
info.num1 = 0
info.isOverflows = false
info.num2 = 0
info.bucketNum = 0
info.num3 = 0
}
func (info *blockInfo) read(decoder *binary.Decoder) error {
var err error
if info.num1, err = decoder.Uvarint(); err != nil {
return err
}
if info.isOverflows, err = decoder.Bool(); err != nil {
return err
}
if info.num2, err = decoder.Uvarint(); err != nil {
return err
}
if info.bucketNum, err = decoder.Int32(); err != nil {
return err
}
if info.num3, err = decoder.Uvarint(); err != nil {
return err
}
return nil
}
func (info *blockInfo) write(encoder *binary.Encoder) error {
if err := encoder.Uvarint(1); err != nil {
return err
}
if err := encoder.Bool(info.isOverflows); err != nil {
return err
}
if err := encoder.Uvarint(2); err != nil {
return err
}
if info.bucketNum == 0 {
info.bucketNum = -1
}
if err := encoder.Int32(info.bucketNum); err != nil {
return err
}
if err := encoder.Uvarint(0); err != nil {
return err
}
return nil
}
type buffer struct {
Offset *binary.Encoder
Column *binary.Encoder
offsetBuffer *bytes.Buffer
columnBuffer *bytes.Buffer
}
func (buf *buffer) WriteTo(w io.Writer) (int64, error) {
var size int64
{
ln, err := buf.offsetBuffer.WriteTo(w)
if err != nil {
return size, err
}
size += ln
}
{
ln, err := buf.columnBuffer.WriteTo(w)
if err != nil {
return size, err
}
size += ln
}
return size, nil
}
func (buf *buffer) reset() {
buf.offsetBuffer.Reset()
buf.columnBuffer.Reset()
}
vendor/github.com/ClickHouse/clickhouse-go/lib/data/block_write_column.go
Generated Vendored
+200
View File
@@ -0,0 +1,200 @@
package data
import (
"fmt"
"github.com/ClickHouse/clickhouse-go/lib/column"
"net"
"reflect"
"time"
"github.com/ClickHouse/clickhouse-go/lib/binary"
)
func (block *Block) WriteDate(c int, v time.Time) error {
_, offset := v.Zone()
nday := (v.Unix() + int64(offset)) / 24 / 3600
return block.buffers[c].Column.UInt16(uint16(nday))
}
func (block *Block) WriteDateNullable(c int, v *time.Time) error {
if v == nil {
return block.buffers[c].Column.UInt16Nullable(nil)
}
unixTime := uint16((*v).Unix() / 24 / 3600)
return block.buffers[c].Column.UInt16Nullable(&unixTime)
}
func (block *Block) WriteDateTime(c int, v time.Time) error {
return block.buffers[c].Column.UInt32(uint32(v.Unix()))
}
func (block *Block) WriteBool(c int, v bool) error {
if v {
return block.buffers[c].Column.UInt8(1)
}
return block.buffers[c].Column.UInt8(0)
}
func (block *Block) WriteDateTimeNullable(c int, v *time.Time) error {
if v == nil {
return block.buffers[c].Column.UInt32Nullable(nil)
}
unixTime := uint32((*v).Unix())
return block.buffers[c].Column.UInt32Nullable(&unixTime)
}
func (block *Block) WriteInt8(c int, v int8) error {
return block.buffers[c].Column.Int8(v)
}
func (block *Block) WriteInt8Nullable(c int, v *int8) error {
return block.buffers[c].Column.Int8Nullable(v)
}
func (block *Block) WriteInt16(c int, v int16) error {
return block.buffers[c].Column.Int16(v)
}
func (block *Block) WriteInt16Nullable(c int, v *int16) error {
return block.buffers[c].Column.Int16Nullable(v)
}
func (block *Block) WriteInt32(c int, v int32) error {
return block.buffers[c].Column.Int32(v)
}
func (block *Block) WriteInt32Nullable(c int, v *int32) error {
return block.buffers[c].Column.Int32Nullable(v)
}
func (block *Block) WriteInt64(c int, v int64) error {
return block.buffers[c].Column.Int64(v)
}
func (block *Block) WriteInt64Nullable(c int, v *int64) error {
return block.buffers[c].Column.Int64Nullable(v)
}
func (block *Block) WriteUInt8(c int, v uint8) error {
return block.buffers[c].Column.UInt8(v)
}
func (block *Block) WriteUInt8Nullable(c int, v *uint8) error {
return block.buffers[c].Column.UInt8Nullable(v)
}
func (block *Block) WriteUInt16(c int, v uint16) error {
return block.buffers[c].Column.UInt16(v)
}
func (block *Block) WriteUInt16Nullable(c int, v *uint16) error {
return block.buffers[c].Column.UInt16Nullable(v)
}
func (block *Block) WriteUInt32(c int, v uint32) error {
return block.buffers[c].Column.UInt32(v)
}
func (block *Block) WriteUInt32Nullable(c int, v *uint32) error {
return block.buffers[c].Column.UInt32Nullable(v)
}
func (block *Block) WriteUInt64(c int, v uint64) error {
return block.buffers[c].Column.UInt64(v)
}
func (block *Block) WriteUInt64Nullable(c int, v *uint64) error {
return block.buffers[c].Column.UInt64Nullable(v)
}
func (block *Block) WriteFloat32(c int, v float32) error {
return block.buffers[c].Column.Float32(v)
}
func (block *Block) WriteFloat32Nullable(c int, v *float32) error {
return block.buffers[c].Column.Float32Nullable(v)
}
func (block *Block) WriteFloat64(c int, v float64) error {
return block.buffers[c].Column.Float64(v)
}
func (block *Block) WriteFloat64Nullable(c int, v *float64) error {
return block.buffers[c].Column.Float64Nullable(v)
}
func (block *Block) WriteBytes(c int, v []byte) error {
if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil {
return err
}
if _, err := block.buffers[c].Column.Write(v); err != nil {
return err
}
return nil
}
func (block *Block) WriteBytesNullable(c int, v *[]byte) error {
isNil := v == nil
if err := block.buffers[c].Column.Nullable(isNil); err != nil {
return err
}
if isNil {
return block.WriteBytes(c, []byte{})
}
return block.WriteBytes(c, *v)
}
func (block *Block) WriteString(c int, v string) error {
if err := block.buffers[c].Column.Uvarint(uint64(len(v))); err != nil {
return err
}
if _, err := block.buffers[c].Column.Write(binary.Str2Bytes(v)); err != nil {
return err
}
return nil
}
func (block *Block) WriteStringNullable(c int, v *string) error {
isNil := v == nil
if err := block.buffers[c].Column.Nullable(isNil); err != nil {
return err
}
if isNil {
return block.WriteString(c, "")
}
return block.WriteString(c, *v)
}
func (block *Block) WriteFixedString(c int, v []byte) error {
return block.Columns[c].Write(block.buffers[c].Column, v)
}
func (block *Block) WriteIP(c int, v net.IP) error {
return block.Columns[c].Write(block.buffers[c].Column, v)
}
func (block *Block) WriteArray(c int, v interface{}) error {
return block.WriteArrayWithValue(c, newValue(reflect.ValueOf(v)))
}
func (block *Block) WriteArrayWithValue(c int, value Value) error {
if value.Kind() != reflect.Slice {
return fmt.Errorf("unsupported Array(T) type [%T]", value.Interface())
}
return block.writeArray(block.Columns[c], value, c, 1)
}
func (block *Block) WriteFixedStringNullable(c int, v *[]byte) error {
writer := block.Columns[c].(*column.Nullable)
return writer.WriteNull(block.buffers[c].Offset, block.buffers[c].Column, v)
}
func (block *Block) WriteArrayNullable(c int, v *interface{}) error {
if err := block.buffers[c].Column.Nullable(v == nil); err != nil {
return err
}
if v == nil {
return block.Columns[c].Write(block.buffers[c].Column, []string{})
}
return block.WriteArray(c, *v)
}

Some files were not shown because too many files have changed in this diff Show More