From bffb41d0ec1ed0cf4dc6df7c87905f958a4a8687 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 17 Apr 2019 07:14:38 +0000 Subject: [PATCH 1/2] =?UTF-8?q?vendor:=20=E6=9B=B4=E6=96=B0yunion.io/x/sql?= =?UTF-8?q?chemy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gopkg.lock | 4 ++-- vendor/yunion.io/x/sqlchemy/index.go | 2 -- vendor/yunion.io/x/sqlchemy/query.go | 28 ++++++++++++---------------- vendor/yunion.io/x/sqlchemy/sync.go | 1 + 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index ef2494ff04..dd9ad79bdf 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1762,11 +1762,11 @@ [[projects]] branch = "master" - digest = "1:ea481ba82e96a2a5c70d259fc0766ea18d7b6f59c74d3cac38a456e8c6c3e828" + digest = "1:29f0a99ba079309ef8c82def5b9c452d0871837780ee702d7ae7c1ebe47478b0" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "bf5e3b0d446dae4e9bf66cc82bfbb8c7d6b56dfd" + revision = "5e43c9cfbb8eb2870882b87d3ab429062d69be2b" [[projects]] branch = "master" diff --git a/vendor/yunion.io/x/sqlchemy/index.go b/vendor/yunion.io/x/sqlchemy/index.go index 798fe79106..fbc3a75d45 100644 --- a/vendor/yunion.io/x/sqlchemy/index.go +++ b/vendor/yunion.io/x/sqlchemy/index.go @@ -2,7 +2,6 @@ package sqlchemy import ( "fmt" - "sort" "strings" ) @@ -51,7 +50,6 @@ func (index *STableIndex) QuotedColumns() []string { } func (ts *STableSpec) AddIndex(unique bool, cols ...string) bool { - sort.Sort(TColumnNames(cols)) for i := 0; i < len(ts.indexes); i += 1 { if ts.indexes[i].IsIdentical(cols...) { return false diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 3bebe434b0..f56946fc4e 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -272,15 +272,13 @@ func queryString(tq *SQuery) string { } buf.WriteString(" FROM ") buf.WriteString(fmt.Sprintf("%s AS `%s`", tq.from.Expression(), tq.from.Alias())) - if tq.joins != nil && len(tq.joins) > 0 { - for _, join := range tq.joins { - buf.WriteByte(' ') - buf.WriteString(string(join.jointype)) - buf.WriteByte(' ') - buf.WriteString(fmt.Sprintf("%s AS `%s`", join.from.Expression(), join.from.Alias())) - buf.WriteString(" ON ") - buf.WriteString(join.condition.WhereClause()) - } + for _, join := range tq.joins { + buf.WriteByte(' ') + buf.WriteString(string(join.jointype)) + buf.WriteByte(' ') + buf.WriteString(fmt.Sprintf("%s AS `%s`", join.from.Expression(), join.from.Alias())) + buf.WriteString(" ON ") + buf.WriteString(join.condition.WhereClause()) } if tq.where != nil { buf.WriteString(" WHERE ") @@ -345,13 +343,11 @@ func (tq *SQuery) Variables() []interface{} { fromvars = tq.from.Variables() vars = append(vars, fromvars...) } - if tq.joins != nil && len(tq.joins) > 0 { - for _, join := range tq.joins { - fromvars = join.from.Variables() - vars = append(vars, fromvars...) - fromvars = join.condition.Variables() - vars = append(vars, fromvars...) - } + for _, join := range tq.joins { + fromvars = join.from.Variables() + vars = append(vars, fromvars...) + fromvars = join.condition.Variables() + vars = append(vars, fromvars...) } if tq.where != nil { fromvars = tq.where.Variables() diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 2e801f9066..8847fc1b6d 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -232,6 +232,7 @@ func (ts *STableSpec) SyncSQL() []string { for _, idx := range removeIndexes { sql := fmt.Sprintf("DROP INDEX `%s` ON `%s`", idx.name, ts.name) + ret = append(ret, sql) log.Infof(sql) } From 3b2677acd1b583dd1dd881e62c6fcc582cdab23e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 17 Apr 2019 07:15:25 +0000 Subject: [PATCH 2/2] =?UTF-8?q?tasks:=20=E6=94=B9=E5=96=84=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 索引简化，obj_id的区分度非常高，且在QueryTasksOfObject()等相关的查询中是确定出现的 - obj_name可认为被obj_id唯一确定 - created_at若出现可假设被MySQL Index Merge优化 - stage的值是可枚举的 - Join taskobjects_tbl时，避免使用子查询查全表 - 使用Union合并两个查询结果 --- pkg/cloudcommon/db/taskman/tasks.go | 84 +++++++++++++++-------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 179bdb4ffa..cb30f0f16e 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -65,7 +65,6 @@ var TaskManager *STaskManager func init() { TaskManager = &STaskManager{SResourceBaseManager: db.NewResourceBaseManager(STask{}, "tasks_tbl", "task", "tasks")} - TaskManager.TableSpec().AddIndex(true, "created_at", "stage", "obj_id", "obj_name") } type STask struct { @@ -73,10 +72,10 @@ type STask struct { Id string `width:"36" charset:"ascii" primary:"true" list:"user"` // Column(VARCHAR(36, charset='ascii'), primary_key=True, default=get_uuid) - ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user"` // Column(VARCHAR(128, charset='utf8'), nullable=False) - ObjId string `width:"128" charset:"ascii" nullable:"false" list:"user"` // Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False) - TaskName string `width:"64" charset:"ascii" nullable:"false" list:"user"` // Column(VARCHAR(64, charset='ascii'), nullable=False) - UserCred mcclient.TokenCredential `width:"1024" charset:"ascii" nullable:"false" get:"user"` // Column(VARCHAR(1024, charset='ascii'), nullable=False) + ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user"` // Column(VARCHAR(128, charset='utf8'), nullable=False) + ObjId string `width:"128" charset:"ascii" nullable:"false" list:"user" index:"true"` // Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False) + TaskName string `width:"64" charset:"ascii" nullable:"false" list:"user"` // Column(VARCHAR(64, charset='ascii'), nullable=False) + UserCred mcclient.TokenCredential `width:"1024" charset:"ascii" nullable:"false" get:"user"` // Column(VARCHAR(1024, charset='ascii'), nullable=False) // OwnerCred string `width:"512" charset:"ascii" nullable:"true"` // Column(VARCHAR(512, charset='ascii'), nullable=True) Params *jsonutils.JSONDict `charset:"ascii" length:"medium" nullable:"false" get:"user"` // Column(MEDIUMTEXT(charset='ascii'), nullable=False) @@ -706,48 +705,53 @@ func (task *STask) GetStartTime() time.Time { } func (manager *STaskManager) QueryTasksOfObject(obj db.IStandaloneModel, since time.Time, isOpen *bool) *sqlchemy.SQuery { - subq1 := manager.Query("id") - subq1 = subq1.Equals("obj_name", obj.Keyword()) - subq1 = subq1.Equals("obj_id", obj.GetId()) - if !since.IsZero() { - subq1 = subq1.GE("created_at", since) - } - if isOpen != nil { - if *isOpen { - subq1 = subq1.Filter(sqlchemy.NOT( - sqlchemy.In(subq1.Field("stage"), []string{"complete", "failed"}), - )) - } else if !*isOpen { - subq1 = subq1.In("stage", []string{"complete", "failed"}) + subq1 := manager.Query() + { + subq1 = subq1.Equals("obj_id", obj.GetId()) + subq1 = subq1.Equals("obj_name", obj.Keyword()) + if !since.IsZero() { + subq1 = subq1.GE("created_at", since) + } + if isOpen != nil { + if *isOpen { + subq1 = subq1.Filter(sqlchemy.NOT( + sqlchemy.In(subq1.Field("stage"), []string{"complete", "failed"}), + )) + } else if !*isOpen { + subq1 = subq1.In("stage", []string{"complete", "failed"}) + } } } - taskObjs := TaskObjectManager.Query().SubQuery() - subq2 := manager.Query("id").Distinct() - subq2 = subq2.Join(taskObjs, sqlchemy.Equals(taskObjs.Field("task_id"), subq2.Field("id"))) - subq2 = subq2.Filter(sqlchemy.Equals(subq2.Field("obj_id"), MULTI_OBJECTS_ID)) - subq2 = subq2.Filter(sqlchemy.Equals(subq2.Field("obj_name"), obj.Keyword())) - subq2 = subq2.Filter(sqlchemy.Equals(taskObjs.Field("obj_id"), obj.GetId())) - if !since.IsZero() { - subq2 = subq2.Filter(sqlchemy.GE(subq2.Field("created_at"), since)) - } - if isOpen != nil { - if *isOpen { - subq2 = subq2.Filter(sqlchemy.NOT( - sqlchemy.In(subq2.Field("stage"), []string{"complete", "failed"}), - )) - } else if !*isOpen { - subq2 = subq2.In("stage", []string{"complete", "failed"}) + subq2 := manager.Query() + { + taskObjs := TaskObjectManager.TableSpec().Instance() + subq2 = subq2.Join(taskObjs, sqlchemy.AND( + sqlchemy.Equals(taskObjs.Field("task_id"), subq2.Field("id")), + sqlchemy.Equals(taskObjs.Field("obj_id"), obj.GetId()), + )) + subq2 = subq2.Filter(sqlchemy.Equals(subq2.Field("obj_id"), MULTI_OBJECTS_ID)) + subq2 = subq2.Filter(sqlchemy.Equals(subq2.Field("obj_name"), obj.Keyword())) + if !since.IsZero() { + subq2 = subq2.Filter(sqlchemy.GE(subq2.Field("created_at"), since)) + } + if isOpen != nil { + if *isOpen { + subq2 = subq2.Filter(sqlchemy.NOT( + sqlchemy.In(subq2.Field("stage"), []string{"complete", "failed"}), + )) + } else if !*isOpen { + subq2 = subq2.In("stage", []string{"complete", "failed"}) + } } } - q := manager.Query() - q = q.Filter(sqlchemy.OR( - sqlchemy.In(q.Field("id"), subq1.SubQuery()), - sqlchemy.In(q.Field("id"), subq2.SubQuery()), - )) - q = q.Desc("created_at") + // subq1 and subq2 do not intersect for the fact that they have + // different condition on tasks_tbl.obj_id field + uq := sqlchemy.Union(subq1, subq2) + uq = uq.Desc("created_at") + q := uq.SubQuery().Query() return q }