From 44f9b4c818e8e24449ad732cb6d65fff732009b7 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 13 Mar 2019 13:54:11 +0800 Subject: [PATCH 01/26] fix cgroups --- pkg/hostman/guestman/qemu-kvm.go | 33 +++++------------------------ pkg/util/cgrouputils/cgrouputils.go | 30 +++++++++++--------------- 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index c2e9364916..b30030d91b 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -995,35 +995,12 @@ func (s *SKVMGuestInstance) setCgroupIo() { } func (s *SKVMGuestInstance) setCgroupCpu() { - cpu, _ := s.Desc.Int("cpu") - cgrouputils.CgroupSet(strconv.Itoa(s.cgroupPid), int(cpu)) + var ( + cpu, _ = s.Desc.Int("cpu") + cpuWeight = 1024 + ) - // TODO XXX - /* - var ( - cpuWeight = 1024 - cpuPeriod = 0 - cpuQuota = 0 - appTags = s.getApptags() - meta, _ = s.Desc.Get("metadata") - ) - - if meta != nil { - if meta.Contains("__cpu_weight") { - cpuWeight, _ = meta.Int("__cpu_weight") - } - if meta.Contains("__cpu_period") { - cpuPeriod, _ = meta.Int("__cpu_period") - } else { - cpuPeriod = -1 - } - if meta.Contains("__cpu_quota") { - cpuQuota, _ = meta.Int("__cpu_quota") - } else { - cpuQuota = -1 - } - } - */ + cgrouputils.CgroupSet(strconv.Itoa(s.cgroupPid), int(cpu)*cpuWeight) } func (s *SKVMGuestInstance) CreateFromDesc(desc jsonutils.JSONObject) error { diff --git a/pkg/util/cgrouputils/cgrouputils.go b/pkg/util/cgrouputils/cgrouputils.go index 1a71767992..79fbd9ae5c 100644 --- a/pkg/util/cgrouputils/cgrouputils.go +++ b/pkg/util/cgrouputils/cgrouputils.go @@ -30,6 +30,7 @@ var ( ) type ICGroupTask interface { + InitTask(hand ICGroupTask, coreNum int, pid string) SetPid(string) SetWeight(coreNum int) SetHand(hand ICGroupTask) @@ -108,19 +109,9 @@ func GetRootParam(module, name, pid string) string { } func SetRootParam(module, name, value, pid string) bool { - if param := GetRootParam(module, name, pid); param != value { - fi, err := os.Open(GetTaskParamPath(module, name, pid)) - if err == nil { - _, err = fi.Write([]byte(value)) - if err != nil { - err = fi.Close() - } else { - log.Errorln(err) - } - } else { - log.Errorln(err) - } - + param := GetRootParam(module, name, pid) + if param != value { + err := ioutil.WriteFile(GetTaskParamPath(module, name, pid), []byte(value), 0644) if err != nil { if len(pid) == 0 { pid = "root" @@ -168,6 +159,12 @@ func (c *CGroupTask) SetPid(pid string) { c.pid = pid } +func (c *CGroupTask) InitTask(hand ICGroupTask, coreNum int, pid string) { + c.SetHand(hand) + c.SetWeight(coreNum) + c.SetPid(pid) +} + func (c *CGroupTask) Module() string { return "" } @@ -536,9 +533,7 @@ func CgroupSet(pid string, coreNum int) bool { &CGroupMemoryTask{&CGroupTask{}}, } for _, hand := range tasks { - hand.SetHand(hand) - hand.SetPid(pid) - hand.SetWeight(coreNum) + hand.InitTask(hand, coreNum, pid) if !hand.SetTask() { return false } @@ -563,8 +558,7 @@ func CgroupDestroy(pid string) bool { &CGroupIOHardlimitTask{CGroupIOTask: &CGroupIOTask{&CGroupTask{}}}, } for _, hand := range tasks { - hand.SetHand(hand) - hand.SetPid(pid) + hand.InitTask(hand, 0, pid) if !hand.RemoveTask() { return false } From 703ce13f67198ef7f7825e0fd6e4e4c3417e6738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 11 Mar 2019 16:01:32 +0800 Subject: [PATCH 02/26] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/metadata.go | 35 +++++++++++++++ pkg/cloudcommon/db/db_dispatcher.go | 4 +- pkg/cloudcommon/db/{ => metadata}/metadata.go | 44 +++++++++++++------ pkg/cloudcommon/db/standalone.go | 32 +++++++++----- pkg/compute/service/handlers.go | 3 ++ pkg/mcclient/modules/mod_metadatas.go | 12 +++++ 6 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 cmd/climc/shell/metadata.go rename pkg/cloudcommon/db/{ => metadata}/metadata.go (76%) create mode 100644 pkg/mcclient/modules/mod_metadatas.go diff --git a/cmd/climc/shell/metadata.go b/cmd/climc/shell/metadata.go new file mode 100644 index 0000000000..03cc22cd2a --- /dev/null +++ b/cmd/climc/shell/metadata.go @@ -0,0 +1,35 @@ +package shell + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + type MetadataListOptions struct { + Resources []string `help:"list of resource e.g server、disk"` + IsSys bool `help:"list sys metadata"` + } + + R(&MetadataListOptions{}, "metadata-list", "List metadatas", func(s *mcclient.ClientSession, opts *MetadataListOptions) error { + params := jsonutils.NewDict() + resources := jsonutils.NewArray() + for _, resource := range opts.Resources { + resources.Add(jsonutils.NewString(resource)) + } + if resources.Length() > 0 { + params.Add(resources, "resources") + } + if opts.IsSys { + params.Add(jsonutils.JSONTrue, "is_sys") + } + result, err := modules.Metadatas.List(s, params) + if err != nil { + return err + } + printList(result, []string{}) + return nil + }) +} diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 4c9df13e94..cdbb4dd92c 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -334,6 +334,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To return nil, err } results := make([]jsonutils.JSONObject, 0) + q.DebugQuery() rows, err := q.Rows() if err != nil { return nil, err @@ -452,7 +453,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok if err == nil && colSpec != nil && colSpec.IsNumeric() { orderBy = []string{"id"} } else { - orderBy = []string{"created_at"} + orderBy = []string{"id"} + //orderBy = []string{"created_at"} } } order := sqlchemy.SQL_ORDER_DESC diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata/metadata.go similarity index 76% rename from pkg/cloudcommon/db/metadata.go rename to pkg/cloudcommon/db/metadata/metadata.go index f49c80fa0d..c2f50069d7 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata/metadata.go @@ -1,4 +1,4 @@ -package db +package metadata import ( "context" @@ -9,7 +9,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/stringutils" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -19,22 +21,22 @@ const ( ) type SMetadataManager struct { - SModelBaseManager + db.SModelBaseManager } type SMetadata struct { - SModelBase + db.SModelBase - Id string `width:"128" charset:"ascii" primary:"true"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True) - Key string `width:"64" charset:"ascii" primary:"true"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) - Value string `charset:"utf8"` // = Column(TEXT(charset='utf8'), nullable=True) - UpdatedAt time.Time `nullable:"false" updated_at:"true"` // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow) + Id string `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True) + Key string `width:"64" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) + Value string `charset:"utf8" list:"user" get:"user"` // = Column(TEXT(charset='utf8'), nullable=True) + UpdatedAt time.Time `nullable:"false" updated_at:"true"` // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow) } var Metadata *SMetadataManager func init() { - Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")} + Metadata = &SMetadataManager{SModelBaseManager: db.NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")} } func (m *SMetadata) GetId() string { @@ -53,6 +55,22 @@ func GetObjectIdstr(model IModel) string { return fmt.Sprintf("%s::%s", model.GetModelManager().Keyword(), model.GetId()) } +func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return true +} + +func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + // queryDict, ok := query.(*jsonutils.JSONDict) + // if !ok { + // return nil, fmt.Errorf("invalid querystring format") + // } + resources := jsonutils.GetQueryStringArray(query, "resrouces") + if len(resources) > 0 { + //q = models.DiskManager.Query() + } + return q, nil +} + /* @classmethod def get_object_idstr(cls, obj, keygen_func): idstr = None @@ -64,7 +82,7 @@ if idstr is None: raise Exception('get_object_idstr: failed to generate obj ID') return idstr */ -func (manager *SMetadataManager) GetStringValue(model IModel, key string, userCred mcclient.TokenCredential) string { +func (manager *SMetadataManager) GetStringValue(model db.IModel, key string, userCred mcclient.TokenCredential) string { if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, model, "metadata")) { return "" } @@ -77,7 +95,7 @@ func (manager *SMetadataManager) GetStringValue(model IModel, key string, userCr return "" } -func (manager *SMetadataManager) GetJsonValue(model IModel, key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { +func (manager *SMetadataManager) GetJsonValue(model db.IModel, key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, model, "metadata")) { return nil } @@ -115,7 +133,7 @@ func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, us changes := make([]sMetadataChange, 0) for _, rec := range records { if len(rec.Value) > 0 { - _, err := Update(&rec, func() error { + _, err := db.Update(&rec, func() error { rec.Value = "" return nil }) @@ -163,7 +181,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m return err } } else { - _, err := Update(&record, func() error { + _, err := db.Update(&record, func() error { record.Value = valStr return nil }) @@ -194,7 +212,7 @@ func (manager *SMetadataManager) GetAll(obj IModel, keys []string, userCred mccl for _, rec := range records { if len(rec.Value) > 0 { if strings.HasPrefix(rec.Key, SYSTEM_ADMIN_PREFIX) { - if userCred != nil && IsAdminAllowGetSpec(userCred, obj, "metadata") { + if userCred != nil && db.IsAdminAllowGetSpec(userCred, obj, "metadata") { key := rec.Key[len(SYSTEM_ADMIN_PREFIX):] ret[key] = rec.Value } diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 46320a863b..8befa91185 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -11,6 +12,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db/metadata" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" @@ -143,23 +145,23 @@ func (model *SStandaloneResourceBase) GetShortDesc(ctx context.Context) *jsonuti * userCred: optional */ func (model *SStandaloneResourceBase) GetMetadata(key string, userCred mcclient.TokenCredential) string { - return Metadata.GetStringValue(model, key, userCred) + return metadata.Metadata.GetStringValue(model, key, userCred) } func (model *SStandaloneResourceBase) GetMetadataJson(key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { - return Metadata.GetJsonValue(model, key, userCred) + return metadata.Metadata.GetJsonValue(model, key, userCred) } func (model *SStandaloneResourceBase) SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error { - if Metadata.IsSystemAdminKey(key) && !IsAdminAllowPerform(userCred, model, "metadata") { + if metadata.Metadata.IsSystemAdminKey(key) && !IsAdminAllowPerform(userCred, model, "metadata") { return httperrors.NewNotSufficientPrivilegeError("cannot set system key") } - return Metadata.SetValue(ctx, model, key, value, userCred) + return metadata.Metadata.SetValue(ctx, model, key, value, userCred) } func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { for k := range dictstore { - if Metadata.IsSystemAdminKey(k) && !IsAdminAllowPerform(userCred, model, "metadata") { + if metadata.Metadata.IsSystemAdminKey(k) && !IsAdminAllowPerform(userCred, model, "metadata") { return httperrors.NewNotSufficientPrivilegeError("not allow to set system key %s", k) } } @@ -167,15 +169,15 @@ func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictst } func (model *SStandaloneResourceBase) RemoveMetadata(ctx context.Context, key string, userCred mcclient.TokenCredential) error { - return Metadata.SetValue(ctx, model, key, "", userCred) + return metadata.Metadata.SetValue(ctx, model, key, "", userCred) } func (model *SStandaloneResourceBase) RemoveAllMetadata(ctx context.Context, userCred mcclient.TokenCredential) error { - return Metadata.RemoveAll(ctx, model, userCred) + return metadata.Metadata.RemoveAll(ctx, model, userCred) } func (model *SStandaloneResourceBase) GetAllMetadata(userCred mcclient.TokenCredential) (map[string]string, error) { - return Metadata.GetAll(model, nil, userCred) + return metadata.Metadata.GetAll(model, nil, userCred) } func (model *SStandaloneResourceBase) AllowGetDetailsMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -184,7 +186,7 @@ func (model *SStandaloneResourceBase) AllowGetDetailsMetadata(ctx context.Contex func (model *SStandaloneResourceBase) GetDetailsMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { fields := jsonutils.GetQueryStringArray(query, "field") - val, err := Metadata.GetAll(model, fields, userCred) + val, err := metadata.Metadata.GetAll(model, fields, userCred) if err != nil { return nil, err } @@ -206,17 +208,27 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC } dictStore := make(map[string]interface{}) for k, v := range dictMap { + if err := model.ValidateMetadataKey(k); err != nil { + return nil, err + } dictStore[k], _ = v.GetString() } err = model.SetAllMetadata(ctx, dictStore, userCred) return nil, err } +func (model *SStandaloneResourceBase) ValidateMetadataKey(key string) error { + if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "ext:") || strings.Contains(key, ":=") { + return httperrors.NewInputParameterError("key cannot start with _ or ext: and not contain :=") + } + return nil +} + func (model *SStandaloneResourceBase) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := model.SResourceBase.GetCustomizeColumns(ctx, userCred, query) withMeta, _ := query.GetString("with_meta") if utils.ToBool(withMeta) { - jsonMeta, err := Metadata.GetAll(model, nil, userCred) + jsonMeta, err := metadata.Metadata.GetAll(model, nil, userCred) if err == nil { extra.Add(jsonutils.Marshal(jsonMeta), "metadata") } else { diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 0ebeea59c1..5f4aee8bb9 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -39,6 +39,9 @@ func InitHandlers(app *appsrv.Application) { db.RegisterModelManager(manager) } + metadatahandler := db.NewModelHandler(db.Metadata) + dispatcher.AddModelDispatcher("", app, metadatahandler) + for _, manager := range []db.IModelManager{ db.OpsLog, models.CloudaccountManager, diff --git a/pkg/mcclient/modules/mod_metadatas.go b/pkg/mcclient/modules/mod_metadatas.go new file mode 100644 index 0000000000..ffc3217474 --- /dev/null +++ b/pkg/mcclient/modules/mod_metadatas.go @@ -0,0 +1,12 @@ +package modules + +var ( + Metadatas ResourceManager +) + +func init() { + Metadatas = NewComputeManager("metadata", "metadata", + []string{"id", "key", "value"}, + []string{}) + registerCompute(&Metadatas) +} From 2d0cc14835b9c7b744c814266b82c47aa4616376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 11 Mar 2019 17:53:23 +0800 Subject: [PATCH 03/26] =?UTF-8?q?=E6=94=AF=E6=8C=81server=20disk=20?= =?UTF-8?q?=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/metadata.go | 23 ++---- cmd/climc/shell/servers.go | 17 +++++ pkg/cloudcommon/db/db_dispatcher.go | 6 +- pkg/cloudcommon/db/{metadata => }/metadata.go | 68 +++++++++++++----- pkg/cloudcommon/db/standalone.go | 71 +++++++++++++++---- pkg/mcclient/options/base.go | 11 +++ pkg/mcclient/options/metadata.go | 7 ++ 7 files changed, 151 insertions(+), 52 deletions(-) rename pkg/cloudcommon/db/{metadata => }/metadata.go (69%) create mode 100644 pkg/mcclient/options/metadata.go diff --git a/cmd/climc/shell/metadata.go b/cmd/climc/shell/metadata.go index 03cc22cd2a..be57c5b386 100644 --- a/cmd/climc/shell/metadata.go +++ b/cmd/climc/shell/metadata.go @@ -1,29 +1,16 @@ package shell import ( - "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" ) func init() { - type MetadataListOptions struct { - Resources []string `help:"list of resource e.g server、disk"` - IsSys bool `help:"list sys metadata"` - } - - R(&MetadataListOptions{}, "metadata-list", "List metadatas", func(s *mcclient.ClientSession, opts *MetadataListOptions) error { - params := jsonutils.NewDict() - resources := jsonutils.NewArray() - for _, resource := range opts.Resources { - resources.Add(jsonutils.NewString(resource)) - } - if resources.Length() > 0 { - params.Add(resources, "resources") - } - if opts.IsSys { - params.Add(jsonutils.JSONTrue, "is_sys") + R(&options.MetadataListOptions{}, "metadata-list", "List metadatas", func(s *mcclient.ClientSession, opts *options.MetadataListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err } result, err := modules.Metadatas.List(s, params) if err != nil { diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index f88cf61154..7acdcbf424 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -85,6 +85,23 @@ func init() { return nil }) + type ServerSetMetadataOptions struct { + ID string `help:"ID or name of server" json:"-"` + KEY string `help:"Metadata key"` + VALUE string `help:"Metadata Value"` + } + + R(&ServerSetMetadataOptions{}, "server-set-metadata", "Set metadata of a server", func(s *mcclient.ClientSession, opts *ServerSetMetadataOptions) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(opts.VALUE), opts.KEY) + result, err := modules.Servers.PerformAction(s, opts.ID, "metadata", params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&options.ServerCreateOptions{}, "server-create", "Create a server", func(s *mcclient.ClientSession, opts *options.ServerCreateOptions) error { params, err := opts.Params() if err != nil { diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index cdbb4dd92c..1375de94d1 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -334,7 +334,6 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To return nil, err } results := make([]jsonutils.JSONObject, 0) - q.DebugQuery() rows, err := q.Rows() if err != nil { return nil, err @@ -453,8 +452,9 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok if err == nil && colSpec != nil && colSpec.IsNumeric() { orderBy = []string{"id"} } else { - orderBy = []string{"id"} - //orderBy = []string{"created_at"} + if manager.TableSpec().ColumnSpec("created_at") != nil { + orderBy = []string{"created_at"} + } } } order := sqlchemy.SQL_ORDER_DESC diff --git a/pkg/cloudcommon/db/metadata/metadata.go b/pkg/cloudcommon/db/metadata.go similarity index 69% rename from pkg/cloudcommon/db/metadata/metadata.go rename to pkg/cloudcommon/db/metadata.go index c2f50069d7..692c494504 100644 --- a/pkg/cloudcommon/db/metadata/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -1,4 +1,4 @@ -package metadata +package db import ( "context" @@ -11,8 +11,8 @@ import ( "yunion.io/x/pkg/util/stringutils" "yunion.io/x/sqlchemy" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -21,22 +21,29 @@ const ( ) type SMetadataManager struct { - db.SModelBaseManager + SModelBaseManager } type SMetadata struct { - db.SModelBase + SModelBase Id string `width:"128" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(128, charset='ascii'), primary_key=True) - Key string `width:"64" charset:"ascii" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) + Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) Value string `charset:"utf8" list:"user" get:"user"` // = Column(TEXT(charset='utf8'), nullable=True) UpdatedAt time.Time `nullable:"false" updated_at:"true"` // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow) } var Metadata *SMetadataManager +var ResourceMap map[string]*SVirtualResourceBaseManager func init() { - Metadata = &SMetadataManager{SModelBaseManager: db.NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")} + Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")} + ResourceMap = map[string]*SVirtualResourceBaseManager{ + "disk": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "disks_tbl", "disk", "disks")}, + "server": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "guests_tbl", "server", "servers")}, + "eip": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "elasticips_tbl", "eip", "eips")}, + "snapshot": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "snapshots_tbl", "snpashot", "snpashots")}, + } } func (m *SMetadata) GetId() string { @@ -60,13 +67,38 @@ func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mc } func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { - // queryDict, ok := query.(*jsonutils.JSONDict) - // if !ok { - // return nil, fmt.Errorf("invalid querystring format") - // } - resources := jsonutils.GetQueryStringArray(query, "resrouces") - if len(resources) > 0 { - //q = models.DiskManager.Query() + resources := jsonutils.GetQueryStringArray(query, "resources") + if len(resources) == 0 { + for resource := range ResourceMap { + resources = append(resources, resource) + } + } + conditions := []sqlchemy.ICondition{} + admin := jsonutils.QueryBoolean(query, "admin", false) + for _, resource := range resources { + if manager, ok := ResourceMap[resource]; ok { + resourceView := manager.Query().SubQuery() + field := sqlchemy.CONCAT(manager.Keyword(), fmt.Sprintf("%s::", manager.Keyword()), resourceView.Field("id")) + sq := resourceView.Query(field) + if !admin || !IsAdminAllowList(userCred, manager) { + ownerId := manager.GetOwnerId(userCred) + if len(ownerId) > 0 { + sq = manager.FilterByOwner(sq, ownerId) + } + } + conditions = append(conditions, sqlchemy.In(q.Field("id"), sq)) + } else { + return nil, httperrors.NewInputParameterError("Not support resource %s tag filter", resource) + } + } + if len(conditions) > 0 { + q = q.Filter(sqlchemy.OR(conditions...)) + } + if !jsonutils.QueryBoolean(query, "with_sys", false) { + q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), "_"))) + } + if !jsonutils.QueryBoolean(query, "with_cloud", false) { + q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), "ext:"))) } return q, nil } @@ -82,7 +114,7 @@ if idstr is None: raise Exception('get_object_idstr: failed to generate obj ID') return idstr */ -func (manager *SMetadataManager) GetStringValue(model db.IModel, key string, userCred mcclient.TokenCredential) string { +func (manager *SMetadataManager) GetStringValue(model IModel, key string, userCred mcclient.TokenCredential) string { if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, model, "metadata")) { return "" } @@ -95,7 +127,7 @@ func (manager *SMetadataManager) GetStringValue(model db.IModel, key string, use return "" } -func (manager *SMetadataManager) GetJsonValue(model db.IModel, key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { +func (manager *SMetadataManager) GetJsonValue(model IModel, key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, model, "metadata")) { return nil } @@ -133,7 +165,7 @@ func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, us changes := make([]sMetadataChange, 0) for _, rec := range records { if len(rec.Value) > 0 { - _, err := db.Update(&rec, func() error { + _, err := Update(&rec, func() error { rec.Value = "" return nil }) @@ -181,7 +213,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m return err } } else { - _, err := db.Update(&record, func() error { + _, err := Update(&record, func() error { record.Value = valStr return nil }) @@ -212,7 +244,7 @@ func (manager *SMetadataManager) GetAll(obj IModel, keys []string, userCred mccl for _, rec := range records { if len(rec.Value) > 0 { if strings.HasPrefix(rec.Key, SYSTEM_ADMIN_PREFIX) { - if userCred != nil && db.IsAdminAllowGetSpec(userCred, obj, "metadata") { + if userCred != nil && IsAdminAllowGetSpec(userCred, obj, "metadata") { key := rec.Key[len(SYSTEM_ADMIN_PREFIX):] ret[key] = rec.Value } diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 8befa91185..6c8767fb3c 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "fmt" "strings" "yunion.io/x/jsonutils" @@ -12,7 +13,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" - "yunion.io/x/onecloud/pkg/cloudcommon/db/metadata" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" @@ -116,6 +116,51 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex q = q.Filter(sqlchemy.IsFalse(q.Field("is_emulated"))) } + tags := map[string]string{} + if query.Contains("tags") { + idx := 0 + for { + key, _ := query.GetString("tags", fmt.Sprintf("%d", idx), "key") + if len(key) == 0 { + break + } + value, _ := query.GetString("tags", fmt.Sprintf("%d", idx), "value") + tags[key] = value + idx++ + } + } + + if len(tags) > 0 { + metadataView := Metadata.Query("id") + idx := 0 + for k, v := range tags { + if idx == 0 { + metadataView = metadataView.Equals("key", k).Equals("value", v) + } else { + subMetataView := Metadata.Query().Equals("key", k).Equals("value", v).SubQuery() + metadataView.Join(subMetataView, sqlchemy.Equals(metadataView.Field("id"), subMetataView.Field("id"))) + } + idx++ + } + metadataView = metadataView.Filter(sqlchemy.Like(metadataView.Field("id"), manager.Keyword()+"::%")).Distinct() + resourceIds := []string{} + rows, err := metadataView.Rows() + if err != nil { + log.Errorf("query metadata ids error: %v", err) + return nil, err + } + defer rows.Close() + for rows.Next() { + var metadataID string + err = rows.Scan(&metadataID) + if err != nil { + log.Errorf("get metadata id scan error: %v", err) + return nil, err + } + resourceIds = append(resourceIds, strings.Replace(metadataID, manager.Keyword()+"::", "", -1)) + } + q = q.Filter(sqlchemy.In(q.Field("id"), resourceIds)) + } return q, nil } @@ -145,23 +190,23 @@ func (model *SStandaloneResourceBase) GetShortDesc(ctx context.Context) *jsonuti * userCred: optional */ func (model *SStandaloneResourceBase) GetMetadata(key string, userCred mcclient.TokenCredential) string { - return metadata.Metadata.GetStringValue(model, key, userCred) + return Metadata.GetStringValue(model, key, userCred) } func (model *SStandaloneResourceBase) GetMetadataJson(key string, userCred mcclient.TokenCredential) jsonutils.JSONObject { - return metadata.Metadata.GetJsonValue(model, key, userCred) + return Metadata.GetJsonValue(model, key, userCred) } func (model *SStandaloneResourceBase) SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error { - if metadata.Metadata.IsSystemAdminKey(key) && !IsAdminAllowPerform(userCred, model, "metadata") { + if Metadata.IsSystemAdminKey(key) && !IsAdminAllowPerform(userCred, model, "metadata") { return httperrors.NewNotSufficientPrivilegeError("cannot set system key") } - return metadata.Metadata.SetValue(ctx, model, key, value, userCred) + return Metadata.SetValue(ctx, model, key, value, userCred) } func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { for k := range dictstore { - if metadata.Metadata.IsSystemAdminKey(k) && !IsAdminAllowPerform(userCred, model, "metadata") { + if Metadata.IsSystemAdminKey(k) && !IsAdminAllowPerform(userCred, model, "metadata") { return httperrors.NewNotSufficientPrivilegeError("not allow to set system key %s", k) } } @@ -169,15 +214,15 @@ func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictst } func (model *SStandaloneResourceBase) RemoveMetadata(ctx context.Context, key string, userCred mcclient.TokenCredential) error { - return metadata.Metadata.SetValue(ctx, model, key, "", userCred) + return Metadata.SetValue(ctx, model, key, "", userCred) } func (model *SStandaloneResourceBase) RemoveAllMetadata(ctx context.Context, userCred mcclient.TokenCredential) error { - return metadata.Metadata.RemoveAll(ctx, model, userCred) + return Metadata.RemoveAll(ctx, model, userCred) } func (model *SStandaloneResourceBase) GetAllMetadata(userCred mcclient.TokenCredential) (map[string]string, error) { - return metadata.Metadata.GetAll(model, nil, userCred) + return Metadata.GetAll(model, nil, userCred) } func (model *SStandaloneResourceBase) AllowGetDetailsMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -186,7 +231,7 @@ func (model *SStandaloneResourceBase) AllowGetDetailsMetadata(ctx context.Contex func (model *SStandaloneResourceBase) GetDetailsMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { fields := jsonutils.GetQueryStringArray(query, "field") - val, err := metadata.Metadata.GetAll(model, fields, userCred) + val, err := Metadata.GetAll(model, fields, userCred) if err != nil { return nil, err } @@ -218,8 +263,8 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC } func (model *SStandaloneResourceBase) ValidateMetadataKey(key string) error { - if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "ext:") || strings.Contains(key, ":=") { - return httperrors.NewInputParameterError("key cannot start with _ or ext: and not contain :=") + if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "ext:") || strings.ContainsAny(key, `:=#&?$/\`) { + return httperrors.NewInputParameterError(`key cannot start with _ or ext: and not contain :=#&?$/\`) } return nil } @@ -228,7 +273,7 @@ func (model *SStandaloneResourceBase) GetCustomizeColumns(ctx context.Context, u extra := model.SResourceBase.GetCustomizeColumns(ctx, userCred, query) withMeta, _ := query.GetString("with_meta") if utils.ToBool(withMeta) { - jsonMeta, err := metadata.Metadata.GetAll(model, nil, userCred) + jsonMeta, err := Metadata.GetAll(model, nil, userCred) if err == nil { extra.Add(jsonutils.Marshal(jsonMeta), "metadata") } else { diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 07900fe7b4..488c502e1d 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -186,6 +186,8 @@ type BaseListOptions struct { ExportFile string `help:"Export to file" metavar:"" json:"-"` ExportKeys string `help:"Export field keys"` ExportTexts string `help:"Export field displayname texts" json:"-"` + TagsKey []string `help:"Tag key" json:"-"` + TagsValue []string `help:"Tag key" json:"-"` Manager string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"` Account string `help:"List objects belonging to the cloud account" json:"account,omitempty"` @@ -217,5 +219,14 @@ func (opts *BaseListOptions) Params() (*jsonutils.JSONDict, error) { params.Set("admin", jsonutils.JSONTrue) } } + if len(opts.TagsKey) > 0 { + if len(opts.TagsKey) != len(opts.TagsValue) { + return nil, fmt.Errorf("tags key length not equal value length") + } + for i := 0; i < len(opts.TagsKey); i++ { + params.Add(jsonutils.NewString(opts.TagsKey[i]), fmt.Sprintf("tags.%d.key", i)) + params.Add(jsonutils.NewString(opts.TagsValue[i]), fmt.Sprintf("tags.%d.value", i)) + } + } return params, nil } diff --git a/pkg/mcclient/options/metadata.go b/pkg/mcclient/options/metadata.go new file mode 100644 index 0000000000..24d0f70cf6 --- /dev/null +++ b/pkg/mcclient/options/metadata.go @@ -0,0 +1,7 @@ +package options + +type MetadataListOptions struct { + Resources []string `help:"list of resource e.g server、disk、eip、snapshot, empty will show all metadata"` + WithSys *bool `help:"With sys metadata"` + WithCloud *bool `help:"With cloud metadata"` +} From 9bcaf1ec844a621d47634fed55a84d6f51dfb7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 13 Mar 2019 12:07:08 +0800 Subject: [PATCH 04/26] =?UTF-8?q?metadata=E8=BF=94=E5=9B=9E=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E4=BD=BF=E7=94=A8=E5=A4=8D=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/metadata.go | 6 ++++-- pkg/cloudcommon/db/standalone.go | 2 +- pkg/mcclient/modules/mod_metadatas.go | 2 +- pkg/mcclient/options/base.go | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 692c494504..4d23b2068f 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -37,7 +37,7 @@ var Metadata *SMetadataManager var ResourceMap map[string]*SVirtualResourceBaseManager func init() { - Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadata")} + Metadata = &SMetadataManager{SModelBaseManager: NewModelBaseManager(SMetadata{}, "metadata_tbl", "metadata", "metadatas")} ResourceMap = map[string]*SVirtualResourceBaseManager{ "disk": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "disks_tbl", "disk", "disks")}, "server": {SStatusStandaloneResourceBaseManager: NewStatusStandaloneResourceBaseManager(SVirtualResourceBase{}, "guests_tbl", "server", "servers")}, @@ -78,7 +78,8 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy for _, resource := range resources { if manager, ok := ResourceMap[resource]; ok { resourceView := manager.Query().SubQuery() - field := sqlchemy.CONCAT(manager.Keyword(), fmt.Sprintf("%s::", manager.Keyword()), resourceView.Field("id")) + prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", manager.Keyword())) + field := sqlchemy.CONCAT(manager.Keyword(), prefix, resourceView.Field("id")) sq := resourceView.Query(field) if !admin || !IsAdminAllowList(userCred, manager) { ownerId := manager.GetOwnerId(userCred) @@ -94,6 +95,7 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy if len(conditions) > 0 { q = q.Filter(sqlchemy.OR(conditions...)) } + q.DebugQuery() if !jsonutils.QueryBoolean(query, "with_sys", false) { q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), "_"))) } diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 6c8767fb3c..7123088815 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -157,7 +157,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex log.Errorf("get metadata id scan error: %v", err) return nil, err } - resourceIds = append(resourceIds, strings.Replace(metadataID, manager.Keyword()+"::", "", -1)) + resourceIds = append(resourceIds, strings.TrimLeft(metadataID, manager.Keyword()+"::")) } q = q.Filter(sqlchemy.In(q.Field("id"), resourceIds)) } diff --git a/pkg/mcclient/modules/mod_metadatas.go b/pkg/mcclient/modules/mod_metadatas.go index ffc3217474..06bcec6974 100644 --- a/pkg/mcclient/modules/mod_metadatas.go +++ b/pkg/mcclient/modules/mod_metadatas.go @@ -5,7 +5,7 @@ var ( ) func init() { - Metadatas = NewComputeManager("metadata", "metadata", + Metadatas = NewComputeManager("metadata", "metadatas", []string{"id", "key", "value"}, []string{}) registerCompute(&Metadatas) diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 488c502e1d..e52733a73c 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -219,10 +219,10 @@ func (opts *BaseListOptions) Params() (*jsonutils.JSONDict, error) { params.Set("admin", jsonutils.JSONTrue) } } + if len(opts.TagsKey) != len(opts.TagsValue) { + return nil, fmt.Errorf("tags key length not equal value length") + } if len(opts.TagsKey) > 0 { - if len(opts.TagsKey) != len(opts.TagsValue) { - return nil, fmt.Errorf("tags key length not equal value length") - } for i := 0; i < len(opts.TagsKey); i++ { params.Add(jsonutils.NewString(opts.TagsKey[i]), fmt.Sprintf("tags.%d.key", i)) params.Add(jsonutils.NewString(opts.TagsValue[i]), fmt.Sprintf("tags.%d.value", i)) From 46d5120a35fa0a3eac6cd7e3e37326979a3b80e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 13 Mar 2019 16:03:29 +0800 Subject: [PATCH 05/26] =?UTF-8?q?=E5=8F=96=E6=B6=88key=E4=BB=A5=5F?= =?UTF-8?q?=E5=BC=80=E5=A4=B4key=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/metadata.go | 12 ++++++++---- pkg/cloudcommon/db/standalone.go | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 4d23b2068f..16f29e801f 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -17,7 +17,8 @@ import ( ) const ( - SYSTEM_ADMIN_PREFIX = "__sys_" + SYSTEM_ADMIN_PREFIX = "_" + CLOUD_TAG_PREFIX = "ext:" ) type SMetadataManager struct { @@ -95,12 +96,11 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy if len(conditions) > 0 { q = q.Filter(sqlchemy.OR(conditions...)) } - q.DebugQuery() if !jsonutils.QueryBoolean(query, "with_sys", false) { - q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), "_"))) + q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), SYSTEM_ADMIN_PREFIX))) } if !jsonutils.QueryBoolean(query, "with_cloud", false) { - q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), "ext:"))) + q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX))) } return q, nil } @@ -194,6 +194,10 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m changes := make([]sMetadataChange, 0) for key, value := range store { + if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, obj, "metadata")) { + return httperrors.NewForbiddenError("Ordinary users can't set the tags that begin with an underscore") + } + valStr := stringutils.Interface2String(value) valStrLower := strings.ToLower(valStr) if valStrLower == "none" || valStrLower == "null" { diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 7123088815..09fcd2e85a 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -263,8 +263,8 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC } func (model *SStandaloneResourceBase) ValidateMetadataKey(key string) error { - if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "ext:") || strings.ContainsAny(key, `:=#&?$/\`) { - return httperrors.NewInputParameterError(`key cannot start with _ or ext: and not contain :=#&?$/\`) + if strings.HasPrefix(key, CLOUD_TAG_PREFIX) || strings.ContainsAny(key, `:=#&?$/\`) { + return httperrors.NewInputParameterError(`key cannot start with %s and not contain :=#&?$/\`, CLOUD_TAG_PREFIX) } return nil } From 5d5cd9833b722ae7264f57192a136a46da6633a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Thu, 14 Mar 2019 15:19:37 +0800 Subject: [PATCH 06/26] =?UTF-8?q?=E5=85=81=E8=AE=B8=E5=8F=AA=E4=BC=A0key?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/standalone.go | 26 +++++++++++++++++++++----- pkg/mcclient/options/base.go | 7 +++---- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 09fcd2e85a..45bbfd5716 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -105,6 +105,11 @@ func (manager *SStandaloneResourceBaseManager) FetchByExternalId(idStr string) ( } } +type STagValue struct { + value string + exist bool +} + func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { q, err := manager.SResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { @@ -116,7 +121,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex q = q.Filter(sqlchemy.IsFalse(q.Field("is_emulated"))) } - tags := map[string]string{} + tags := map[string]STagValue{} if query.Contains("tags") { idx := 0 for { @@ -124,7 +129,11 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex if len(key) == 0 { break } - value, _ := query.GetString("tags", fmt.Sprintf("%d", idx), "value") + value := STagValue{exist: false} + if query.Contains("tags", fmt.Sprintf("%d", idx), "value") { + value.value, _ = query.GetString("tags", fmt.Sprintf("%d", idx), "value") + value.exist = true + } tags[key] = value idx++ } @@ -135,10 +144,17 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex idx := 0 for k, v := range tags { if idx == 0 { - metadataView = metadataView.Equals("key", k).Equals("value", v) + metadataView = metadataView.Equals("key", k) + if v.exist { + metadataView = metadataView.Equals("value", v.value) + } } else { - subMetataView := Metadata.Query().Equals("key", k).Equals("value", v).SubQuery() - metadataView.Join(subMetataView, sqlchemy.Equals(metadataView.Field("id"), subMetataView.Field("id"))) + subMetataView := Metadata.Query().Equals("key", k) + if v.exist { + subMetataView = subMetataView.Equals("value", v.value) + } + sq := subMetataView.SubQuery() + metadataView.Join(sq, sqlchemy.Equals(metadataView.Field("id"), sq.Field("id"))) } idx++ } diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index e52733a73c..af622c7347 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -219,13 +219,12 @@ func (opts *BaseListOptions) Params() (*jsonutils.JSONDict, error) { params.Set("admin", jsonutils.JSONTrue) } } - if len(opts.TagsKey) != len(opts.TagsValue) { - return nil, fmt.Errorf("tags key length not equal value length") - } if len(opts.TagsKey) > 0 { for i := 0; i < len(opts.TagsKey); i++ { params.Add(jsonutils.NewString(opts.TagsKey[i]), fmt.Sprintf("tags.%d.key", i)) - params.Add(jsonutils.NewString(opts.TagsValue[i]), fmt.Sprintf("tags.%d.value", i)) + if len(opts.TagsValue) > i { + params.Add(jsonutils.NewString(opts.TagsValue[i]), fmt.Sprintf("tags.%d.value", i)) + } } } return params, nil From ec764d2d1d8ebf1686eeff9e94469dfc1b317485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Thu, 14 Mar 2019 18:56:18 +0800 Subject: [PATCH 07/26] =?UTF-8?q?=E5=85=BC=E5=AE=B9disk=20json=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index ec0fc857f5..a6d6f5ff46 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -895,21 +895,21 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") } - dataDiskDefs := make([]string, 0) + dataDiskDefs := []jsonutils.JSONObject{} if sku != nil && sku.AttachedDiskCount > 0 { for i := 0; i < sku.AttachedDiskCount; i += 1 { - dataDiskDefs = append(dataDiskDefs, fmt.Sprintf("%dg:%s", sku.AttachedDiskSizeGB, sku.AttachedDiskType)) + dataDisk := jsonutils.Marshal(map[string]interface{}{ + "size_mb": sku.AttachedDiskSizeGB * 1024, + "backend": strings.ToLower(sku.AttachedDiskType), + }) + dataDiskDefs = append(dataDiskDefs, dataDisk) } } // start from data disk jsonArray := jsonutils.GetArrayOfPrefix(data, "disk") for idx := 1; idx < len(jsonArray); idx += 1 { // data.Contains(fmt.Sprintf("disk.%d", idx)) - diskJson, err := jsonArray[idx].GetString() // data.GetString(fmt.Sprintf("disk.%d", idx)) - if err != nil { - return nil, httperrors.NewInputParameterError("invalid disk description %s", err) - } - dataDiskDefs = append(dataDiskDefs, diskJson) + dataDiskDefs = append(dataDiskDefs, jsonArray[idx]) } rootDiskConfig, err := parseDiskInfo(ctx, userCred, jsonArray[0]) @@ -926,7 +926,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m data.Set("disk.0", jsonutils.Marshal(rootDiskConfig)) for i := 0; i < len(dataDiskDefs); i += 1 { - diskConfig, err := parseDiskInfo(ctx, userCred, jsonutils.NewString(dataDiskDefs[i])) + diskConfig, err := parseDiskInfo(ctx, userCred, dataDiskDefs[i]) if err != nil { return nil, httperrors.NewInputParameterError("parse disk description error %s", err) } From e0ee2b8c59961f4e6140480dd6444c002073c703 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Thu, 14 Mar 2019 18:49:25 +0800 Subject: [PATCH 08/26] fix package dependence --- cmd/lbagent/main.go | 7 ++++--- pkg/baremetal/options/options.go | 6 ++---- pkg/baremetal/pxe/dhcp.go | 9 +++++---- pkg/baremetal/service/service.go | 11 ++++++----- pkg/cloudcommon/{ => app}/app.go | 10 +++++----- pkg/cloudcommon/{ => app}/auth.go | 5 +++-- pkg/cloudcommon/app/doc.go | 1 + pkg/cloudcommon/consts/hosttypes/doc.go | 1 + pkg/cloudcommon/consts/hosttypes/hosttypes.go | 18 ++++++++++++++++++ pkg/cloudcommon/consts/networktypes/doc.go | 1 + .../consts/networktypes/network_consts.go | 9 +++++++++ .../{ => consts}/storagetypes/doc.go | 2 +- .../{ => consts}/storagetypes/storagetypes.go | 11 +++++++++-- pkg/cloudcommon/database.go | 3 ++- pkg/cloudcommon/options/doc.go | 1 + pkg/cloudcommon/{ => options}/options.go | 2 +- pkg/cloudir/options/options.go | 4 ++-- pkg/cloudir/service/service.go | 12 +++++++----- pkg/cloutpost/options/options.go | 4 ++-- pkg/cloutpost/service/service.go | 12 +++++++----- pkg/compute/options/options.go | 7 +++---- pkg/compute/service/service.go | 12 +++++++----- pkg/hostimage/host_image_service.go | 10 ++++++---- pkg/hostman/guestman/qemu-kvm.go | 2 +- pkg/hostman/host_services.go | 13 +++++++------ pkg/hostman/hostinfo/hostinfo.go | 2 +- pkg/hostman/options/options.go | 4 ++-- pkg/hostman/storageman/core.go | 2 +- pkg/hostman/storageman/disk_local.go | 2 +- pkg/hostman/storageman/disk_nfs.go | 2 +- pkg/hostman/storageman/disk_rbd.go | 2 +- .../storageman/imagecachemanager_rbd.go | 2 +- pkg/hostman/storageman/storage_local.go | 2 +- pkg/hostman/storageman/storage_nfs.go | 5 ++--- pkg/hostman/storageman/storage_rbd.go | 5 ++--- pkg/image/options/options.go | 6 +++--- pkg/image/service/service.go | 11 +++++++---- pkg/lbagent/options.go | 4 ++-- pkg/logger/options/options.go | 6 +++--- pkg/logger/service/service.go | 11 +++++++---- pkg/util/qemuimg/qemuimg.go | 4 ++-- pkg/webconsole/options/options.go | 6 ++---- pkg/webconsole/service/service.go | 9 +++++---- pkg/yunionconf/options/options.go | 6 +++--- pkg/yunionconf/service/service.go | 11 +++++++---- 45 files changed, 165 insertions(+), 110 deletions(-) rename pkg/cloudcommon/{ => app}/app.go (78%) rename pkg/cloudcommon/{ => app}/auth.go (88%) create mode 100644 pkg/cloudcommon/app/doc.go create mode 100644 pkg/cloudcommon/consts/hosttypes/doc.go create mode 100644 pkg/cloudcommon/consts/hosttypes/hosttypes.go create mode 100644 pkg/cloudcommon/consts/networktypes/doc.go create mode 100644 pkg/cloudcommon/consts/networktypes/network_consts.go rename pkg/cloudcommon/{ => consts}/storagetypes/doc.go (71%) rename pkg/cloudcommon/{ => consts}/storagetypes/storagetypes.go (64%) create mode 100644 pkg/cloudcommon/options/doc.go rename pkg/cloudcommon/{ => options}/options.go (99%) diff --git a/cmd/lbagent/main.go b/cmd/lbagent/main.go index f2842002e2..623eb2229f 100644 --- a/cmd/lbagent/main.go +++ b/cmd/lbagent/main.go @@ -9,7 +9,8 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/lbagent" ) @@ -17,8 +18,8 @@ func main() { opts := &lbagent.Options{} commonOpts := &opts.CommonOptions { - cloudcommon.ParseOptions(opts, os.Args, "lbagent.conf", "lbagent") - cloudcommon.InitAuth(commonOpts, func() { + common_options.ParseOptions(opts, os.Args, "lbagent.conf", "lbagent") + app_common.InitAuth(commonOpts, func() { log.Infof("auth finished ok") }) } diff --git a/pkg/baremetal/options/options.go b/pkg/baremetal/options/options.go index 5fa018a65c..9eada3b168 100644 --- a/pkg/baremetal/options/options.go +++ b/pkg/baremetal/options/options.go @@ -1,11 +1,9 @@ package options -import ( - "yunion.io/x/onecloud/pkg/cloudcommon" -) +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" type BaremetalOptions struct { - cloudcommon.CommonOptions + common_options.CommonOptions ListenInterface string `help:"Master net interface of baremetal server" default:"br0"` AccessAddress string `help:"Management IP address of baremetal server, only need to use when multiple address bind to ListenInterface"` diff --git a/pkg/baremetal/pxe/dhcp.go b/pkg/baremetal/pxe/dhcp.go index 784baec4b4..71dbd650b1 100644 --- a/pkg/baremetal/pxe/dhcp.go +++ b/pkg/baremetal/pxe/dhcp.go @@ -10,8 +10,9 @@ import ( "yunion.io/x/log" o "yunion.io/x/onecloud/pkg/baremetal/options" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/hosttypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/networktypes" "yunion.io/x/onecloud/pkg/cloudcommon/types" - "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/dhcp" ) @@ -165,7 +166,7 @@ func (h *DHCPHandler) fetchConfig() (*dhcp.ResponseConfig, error) { h.baremetalInstance = bmInstance ipmiNic := h.baremetalInstance.GetIPMINic(h.ClientMac) if ipmiNic != nil && ipmiNic.Mac == h.ClientMac.String() { - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, models.NETWORK_TYPE_IPMI) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, networktypes.NETWORK_TYPE_IPMI) if err != nil { return nil, err } @@ -215,7 +216,7 @@ func (h *DHCPHandler) findNetworkConf(filterUseIp bool) (*types.SNetworkConfig, func (h *DHCPHandler) findBaremetalsOfAnyMac(isBaremetal bool) (*modules.ListResult, error) { session := h.baremetalManager.GetClientSession() params := jsonutils.NewDict() - params.Add(jsonutils.NewString(models.HOST_TYPE_BAREMETAL), "host_type") + params.Add(jsonutils.NewString(hosttypes.HOST_TYPE_BAREMETAL), "host_type") params.Add(jsonutils.NewString(h.ClientMac.String()), "any_mac") if isBaremetal { params.Add(jsonutils.JSONTrue, "is_baremetal") @@ -292,7 +293,7 @@ func (h *DHCPHandler) doInitBaremetalAdminNetif(desc jsonutils.JSONObject) error if err != nil { return err } - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, models.NETWORK_TYPE_PXE) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, networktypes.NETWORK_TYPE_PXE) return err } diff --git a/pkg/baremetal/service/service.go b/pkg/baremetal/service/service.go index a0596ed205..cfd4759578 100644 --- a/pkg/baremetal/service/service.go +++ b/pkg/baremetal/service/service.go @@ -9,7 +9,8 @@ import ( "yunion.io/x/onecloud/pkg/baremetal/handler" o "yunion.io/x/onecloud/pkg/baremetal/options" "yunion.io/x/onecloud/pkg/baremetal/tasks" - "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudcommon/service" ) @@ -22,13 +23,13 @@ func New() *BaremetalService { } func (s *BaremetalService) StartService() { - cloudcommon.ParseOptions(&o.Options, os.Args, "baremetal.conf", "baremetal") - cloudcommon.InitAuth(&o.Options.CommonOptions, s.startAgent) + common_options.ParseOptions(&o.Options, os.Args, "baremetal.conf", "baremetal") + app_common.InitAuth(&o.Options.CommonOptions, s.startAgent) - app := cloudcommon.InitApp(&o.Options.CommonOptions, false) + app := app_common.InitApp(&o.Options.CommonOptions, false) handler.InitHandlers(app) - cloudcommon.ServeForeverWithCleanup(app, &o.Options.CommonOptions, func() { + app_common.ServeForeverWithCleanup(app, &o.Options.CommonOptions, func() { tasks.OnStop() }) } diff --git a/pkg/cloudcommon/app.go b/pkg/cloudcommon/app/app.go similarity index 78% rename from pkg/cloudcommon/app.go rename to pkg/cloudcommon/app/app.go index 6463dfbbea..d7fa497d14 100644 --- a/pkg/cloudcommon/app.go +++ b/pkg/cloudcommon/app/app.go @@ -1,4 +1,4 @@ -package cloudcommon +package app import ( "net" @@ -8,10 +8,11 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/appsrv" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/util/seclib2" ) -func InitApp(options *CommonOptions, dbAccess bool) *appsrv.Application { +func InitApp(options *common_options.CommonOptions, dbAccess bool) *appsrv.Application { // cache := appsrv.NewCache(options.AuthTokenCacheSize) app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess) app.CORSAllowHosts(options.CorsHosts) @@ -23,12 +24,11 @@ func InitApp(options *CommonOptions, dbAccess bool) *appsrv.Application { return app } -func ServeForever(app *appsrv.Application, options *CommonOptions) { +func ServeForever(app *appsrv.Application, options *common_options.CommonOptions) { ServeForeverWithCleanup(app, options, nil) } -func ServeForeverWithCleanup(app *appsrv.Application, options *CommonOptions, onStop func()) { - AppDBInit(app) +func ServeForeverWithCleanup(app *appsrv.Application, options *common_options.CommonOptions, onStop func()) { addr := net.JoinHostPort(options.Address, strconv.Itoa(options.Port)) proto := "http" if options.EnableSsl { diff --git a/pkg/cloudcommon/auth.go b/pkg/cloudcommon/app/auth.go similarity index 88% rename from pkg/cloudcommon/auth.go rename to pkg/cloudcommon/app/auth.go index a68c7a8c31..ba68a741f5 100644 --- a/pkg/cloudcommon/auth.go +++ b/pkg/cloudcommon/app/auth.go @@ -1,4 +1,4 @@ -package cloudcommon +package app import ( "context" @@ -7,11 +7,12 @@ import ( "time" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/mcclient/auth" ) -func InitAuth(options *CommonOptions, authComplete auth.AuthCompletedCallback) { +func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompletedCallback) { if len(options.AuthURL) == 0 { fmt.Println("Missing AuthURL") diff --git a/pkg/cloudcommon/app/doc.go b/pkg/cloudcommon/app/doc.go new file mode 100644 index 0000000000..5e7c05a278 --- /dev/null +++ b/pkg/cloudcommon/app/doc.go @@ -0,0 +1 @@ +package app // import "yunion.io/x/onecloud/pkg/cloudcommon/app" diff --git a/pkg/cloudcommon/consts/hosttypes/doc.go b/pkg/cloudcommon/consts/hosttypes/doc.go new file mode 100644 index 0000000000..6bca1d2a51 --- /dev/null +++ b/pkg/cloudcommon/consts/hosttypes/doc.go @@ -0,0 +1 @@ +package hosttypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/hosttypes" diff --git a/pkg/cloudcommon/consts/hosttypes/hosttypes.go b/pkg/cloudcommon/consts/hosttypes/hosttypes.go new file mode 100644 index 0000000000..381543f122 --- /dev/null +++ b/pkg/cloudcommon/consts/hosttypes/hosttypes.go @@ -0,0 +1,18 @@ +package hosttypes + +const ( + HOST_TYPE_BAREMETAL = "baremetal" + HOST_TYPE_HYPERVISOR = "hypervisor" // KVM + HOST_TYPE_KVM = "kvm" + HOST_TYPE_ESXI = "esxi" // # VMWare vSphere ESXi + HOST_TYPE_KUBELET = "kubelet" // # Kubernetes Kubelet + HOST_TYPE_HYPERV = "hyperv" // # Microsoft Hyper-V + HOST_TYPE_XEN = "xen" // # XenServer + + HOST_TYPE_ALIYUN = "aliyun" + HOST_TYPE_AWS = "aws" + HOST_TYPE_QCLOUD = "qcloud" + HOST_TYPE_AZURE = "azure" + HOST_TYPE_HUAWEI = "huawei" + HOST_TYPE_OPENSTACK = "openstack" +) diff --git a/pkg/cloudcommon/consts/networktypes/doc.go b/pkg/cloudcommon/consts/networktypes/doc.go new file mode 100644 index 0000000000..caf722ad4a --- /dev/null +++ b/pkg/cloudcommon/consts/networktypes/doc.go @@ -0,0 +1 @@ +package networktypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/networktypes" diff --git a/pkg/cloudcommon/consts/networktypes/network_consts.go b/pkg/cloudcommon/consts/networktypes/network_consts.go new file mode 100644 index 0000000000..49cdd7fd7c --- /dev/null +++ b/pkg/cloudcommon/consts/networktypes/network_consts.go @@ -0,0 +1,9 @@ +package networktypes + +const ( + NETWORK_TYPE_GUEST = "guest" + NETWORK_TYPE_BAREMETAL = "baremetal" + NETWORK_TYPE_CONTAINER = "container" + NETWORK_TYPE_PXE = "pxe" + NETWORK_TYPE_IPMI = "ipmi" +) diff --git a/pkg/cloudcommon/storagetypes/doc.go b/pkg/cloudcommon/consts/storagetypes/doc.go similarity index 71% rename from pkg/cloudcommon/storagetypes/doc.go rename to pkg/cloudcommon/consts/storagetypes/doc.go index 2619223c2a..868d183308 100644 --- a/pkg/cloudcommon/storagetypes/doc.go +++ b/pkg/cloudcommon/consts/storagetypes/doc.go @@ -1 +1 @@ -package storagetypes // import "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" +package storagetypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" diff --git a/pkg/cloudcommon/storagetypes/storagetypes.go b/pkg/cloudcommon/consts/storagetypes/storagetypes.go similarity index 64% rename from pkg/cloudcommon/storagetypes/storagetypes.go rename to pkg/cloudcommon/consts/storagetypes/storagetypes.go index 4779914227..51d29d9103 100644 --- a/pkg/cloudcommon/storagetypes/storagetypes.go +++ b/pkg/cloudcommon/consts/storagetypes/storagetypes.go @@ -1,7 +1,7 @@ package storagetypes // TODO: move models/storages.go storage types to this file -var ( +const ( STORAGE_LOCAL = "local" STORAGE_BAREMETAL = "baremetal" STORAGE_SHEEPDOG = "sheepdog" @@ -13,6 +13,13 @@ var ( DISK_TYPE_ROTATE = "rotate" DISK_TYPE_SSD = "ssd" +) - Local = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS} +var Local = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS} + +const ( + STORAGE_ENABLED = "enabled" + // STORAGE_DISABLED = "disabled" + STORAGE_OFFLINE = "offline" + STORAGE_ONLINE = "online" ) diff --git a/pkg/cloudcommon/database.go b/pkg/cloudcommon/database.go index 9dacb845f5..27eb2f12a7 100644 --- a/pkg/cloudcommon/database.go +++ b/pkg/cloudcommon/database.go @@ -13,13 +13,14 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) const ( MIN_DB_CONN_MAX = 5 ) -func InitDB(options *DBOptions) { +func InitDB(options *common_options.DBOptions) { if options.GlobalVirtualResourceNamespace { consts.EnableGlobalVirtualResourceNamespace() } diff --git a/pkg/cloudcommon/options/doc.go b/pkg/cloudcommon/options/doc.go new file mode 100644 index 0000000000..b2d7d12e08 --- /dev/null +++ b/pkg/cloudcommon/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/cloudcommon/options" diff --git a/pkg/cloudcommon/options.go b/pkg/cloudcommon/options/options.go similarity index 99% rename from pkg/cloudcommon/options.go rename to pkg/cloudcommon/options/options.go index d51b4ad6c0..df051dba72 100644 --- a/pkg/cloudcommon/options.go +++ b/pkg/cloudcommon/options/options.go @@ -1,4 +1,4 @@ -package cloudcommon +package options import ( "fmt" diff --git a/pkg/cloudir/options/options.go b/pkg/cloudir/options/options.go index 7ce0d65f46..1c1890da02 100644 --- a/pkg/cloudir/options/options.go +++ b/pkg/cloudir/options/options.go @@ -1,14 +1,14 @@ package options import ( - "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/etcd" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) type SCloudirOptions struct { etcd.SEtcdOptions - cloudcommon.CommonOptions + common_options.CommonOptions } var ( diff --git a/pkg/cloudir/service/service.go b/pkg/cloudir/service/service.go index 97ac69d5cf..0b9f217744 100644 --- a/pkg/cloudir/service/service.go +++ b/pkg/cloudir/service/service.go @@ -6,16 +6,18 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/etcd" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudir/options" ) func StartService() { opts := &options.Options commonOpts := &opts.CommonOptions - cloudcommon.ParseOptions(opts, os.Args, "cloudir.conf", "cloudir") + common_options.ParseOptions(opts, os.Args, "cloudir.conf", "cloudir") - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) @@ -25,11 +27,11 @@ func StartService() { return } - app := cloudcommon.InitApp(commonOpts, false) - + app := app_common.InitApp(commonOpts, false) + cloudcommon.AppDBInit(app) initHandlers(app) - cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + app_common.ServeForeverWithCleanup(app, commonOpts, func() { etcd.CloseDefaultEtcdClient() }) } diff --git a/pkg/cloutpost/options/options.go b/pkg/cloutpost/options/options.go index 058aac2330..4871691580 100644 --- a/pkg/cloutpost/options/options.go +++ b/pkg/cloutpost/options/options.go @@ -1,8 +1,8 @@ package options import ( - "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/etcd" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) type CloudSyncOptions struct { @@ -13,7 +13,7 @@ type CloudSyncOptions struct { etcd.SEtcdOptions - cloudcommon.CommonOptions + common_options.CommonOptions } var ( diff --git a/pkg/cloutpost/service/service.go b/pkg/cloutpost/service/service.go index e2162cff0d..15533530f2 100644 --- a/pkg/cloutpost/service/service.go +++ b/pkg/cloutpost/service/service.go @@ -6,8 +6,10 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/etcd" "yunion.io/x/onecloud/pkg/cloudcommon/etcd/models" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloutpost/options" ) @@ -18,9 +20,9 @@ const ( func StartService() { opts := &options.Options commonOpts := &opts.CommonOptions - cloudcommon.ParseOptions(opts, os.Args, "cloutpost.conf", SERVICE_TYPE) + common_options.ParseOptions(opts, os.Args, "cloutpost.conf", SERVICE_TYPE) - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) @@ -30,8 +32,8 @@ func StartService() { } defer etcd.CloseDefaultEtcdClient() - app := cloudcommon.InitApp(commonOpts, false) - + app := app_common.InitApp(commonOpts, false) + cloudcommon.AppDBInit(app) initHandlers(app) err = models.ServiceRegistryManager.Register( @@ -49,5 +51,5 @@ func StartService() { log.Fatalf("fail to register service %s", err) } - cloudcommon.ServeForever(app, commonOpts) + app_common.ServeForever(app, commonOpts) } diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index e810c40cfc..dd807cbeba 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -1,7 +1,7 @@ package options import ( - "yunion.io/x/onecloud/pkg/cloudcommon" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudcommon/pending_delete" ) @@ -87,9 +87,8 @@ type ComputeOptions struct { DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"24"` SCapabilityOptions - - cloudcommon.CommonOptions - cloudcommon.DBOptions + common_options.CommonOptions + common_options.DBOptions } type SCapabilityOptions struct { diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index 5d1dc07efc..e9b6b08e18 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -9,8 +9,10 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" _ "yunion.io/x/onecloud/pkg/compute/hostdrivers" "yunion.io/x/onecloud/pkg/compute/models" @@ -32,22 +34,22 @@ func StartService() { opts := &options.Options commonOpts := &options.Options.CommonOptions dbOpts := &options.Options.DBOptions - cloudcommon.ParseOptions(opts, os.Args, "region.conf", "compute") + common_options.ParseOptions(opts, os.Args, "region.conf", "compute") if opts.PortV2 > 0 { log.Infof("Port V2 %d is specified, use v2 port", opts.PortV2) commonOpts.Port = opts.PortV2 } - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) cloudcommon.InitDB(dbOpts) defer cloudcommon.CloseDB() - app := cloudcommon.InitApp(commonOpts, true) - + app := app_common.InitApp(commonOpts, true) + cloudcommon.AppDBInit(app) InitHandlers(app) if !db.CheckSync(opts.AutoSyncTable) { @@ -81,5 +83,5 @@ func StartService() { cron.Start() defer cron.Stop() - cloudcommon.ServeForever(app, commonOpts) + app_common.ServeForever(app, commonOpts) } diff --git a/pkg/hostimage/host_image_service.go b/pkg/hostimage/host_image_service.go index 6a82a0d019..8d8d92923a 100644 --- a/pkg/hostimage/host_image_service.go +++ b/pkg/hostimage/host_image_service.go @@ -17,7 +17,9 @@ import ( "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/consts" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -32,14 +34,14 @@ var HostImageOptions SHostImageOptions func StartService() { consts.SetServiceType("host-image") - cloudcommon.ParseOptions(&HostImageOptions, &HostImageOptions.Options, os.Args, "host.conf") + common_options.ParseOptions(&HostImageOptions, &HostImageOptions.Options, os.Args, "host.conf") HostImageOptions.Port += 40000 - cloudcommon.InitAuth(&HostImageOptions.Options, func() { + app_common.InitAuth(&HostImageOptions.Options, func() { log.Infof("Auth complete!!") }) - app := cloudcommon.InitApp(&HostImageOptions.Options) + app := app_common.InitApp(&HostImageOptions.Options) initHandlers(app, "") - cloudcommon.ServeForever(app, &HostImageOptions.Options) + app_common.ServeForever(app, &HostImageOptions.Options) } func initHandlers(app *appsrv.Application, prefix string) { diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index c2e9364916..06a3e254e7 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -18,7 +18,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/guestfs" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostutils" diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index c81c76ad59..7b958a099f 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -6,8 +6,9 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudcommon/service" "yunion.io/x/onecloud/pkg/hostman/diskhandlers" "yunion.io/x/onecloud/pkg/hostman/downloader" @@ -27,10 +28,10 @@ type SHostService struct { } func (host *SHostService) StartService() { - cloudcommon.ParseOptions(&options.HostOptions, os.Args, "host.conf", "host") + common_options.ParseOptions(&options.HostOptions, os.Args, "host.conf", "host") options.HostOptions.EnableRbac = false // disable rbac - app := cloudcommon.InitApp(&options.HostOptions.CommonOptions, false) + app := app_common.InitApp(&options.HostOptions.CommonOptions, false) hostInstance := hostinfo.Instance() if err := hostInstance.Init(); err != nil { log.Fatalf(err.Error()) @@ -41,7 +42,7 @@ func (host *SHostService) StartService() { } guestman.Init(hostInstance, options.HostOptions.ServersPath) - cloudcommon.InitAuth(&options.HostOptions.CommonOptions, func() { + app_common.InitAuth(&options.HostOptions.CommonOptions, func() { log.Infof("Auth complete!!") // ??? Why wait 5 seconds @@ -57,14 +58,14 @@ func (host *SHostService) StartService() { // Init Metadata handler go metadata.StartService( - cloudcommon.InitApp(&options.HostOptions.CommonOptions, false), + app_common.InitApp(&options.HostOptions.CommonOptions, false), options.HostOptions.Address, options.HostOptions.Port+1000) cronManager := cronman.GetCronJobManager(false) cronManager.AddJob2( "CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false) - cloudcommon.ServeForeverWithCleanup(app, &options.HostOptions.CommonOptions, func() { + app_common.ServeForeverWithCleanup(app, &options.HostOptions.CommonOptions, func() { hostinfo.Stop() storageman.Stop() hostmetrics.Stop() diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index f49b2043ae..523459ff9d 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -16,8 +16,8 @@ import ( "yunion.io/x/pkg/utils" bare2 "yunion.io/x/onecloud/pkg/baremetal" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" bare1 "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 725bb7f44b..1f95ef5952 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -1,9 +1,9 @@ package options -import "yunion.io/x/onecloud/pkg/cloudcommon" +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" type SHostOptions struct { - cloudcommon.CommonOptions + common_options.CommonOptions HostType string `help:"Host server type, either hypervisor or kubelet" default:"hypervisor"` ListenInterface string `help:"Master address of host server"` diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index d1c50604a5..316fe55c1b 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index f7570930e0..512ad3654d 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" diff --git a/pkg/hostman/storageman/disk_nfs.go b/pkg/hostman/storageman/disk_nfs.go index 384dd21235..df349d0928 100644 --- a/pkg/hostman/storageman/disk_nfs.go +++ b/pkg/hostman/storageman/disk_nfs.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" ) type SNFSDisk struct { diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index fc59da9a9d..18bd1a6150 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -13,7 +13,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go index 1a0a536afe..14544b533a 100644 --- a/pkg/hostman/storageman/imagecachemanager_rbd.go +++ b/pkg/hostman/storageman/imagecachemanager_rbd.go @@ -10,7 +10,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) diff --git a/pkg/hostman/storageman/storage_local.go b/pkg/hostman/storageman/storage_local.go index f2d15e66e2..c4e2de339c 100644 --- a/pkg/hostman/storageman/storage_local.go +++ b/pkg/hostman/storageman/storage_local.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" diff --git a/pkg/hostman/storageman/storage_nfs.go b/pkg/hostman/storageman/storage_nfs.go index bc387ff869..9e9dc91d0e 100644 --- a/pkg/hostman/storageman/storage_nfs.go +++ b/pkg/hostman/storageman/storage_nfs.go @@ -8,8 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" - "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/fileutils2" @@ -85,7 +84,7 @@ func (s *SNFSStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content := jsonutils.NewDict() content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb()))) content.Set("storage_type", jsonutils.NewString(s.StorageType())) - content.Set("status", jsonutils.NewString(models.STORAGE_ONLINE)) + content.Set("status", jsonutils.NewString(storagetypes.STORAGE_ONLINE)) content.Set("zone", jsonutils.NewString(s.GetZone())) log.Infof("Sync storage info %s", s.StorageId) res, err := modules.Storages.Put( diff --git a/pkg/hostman/storageman/storage_rbd.go b/pkg/hostman/storageman/storage_rbd.go index 03add9ff67..95c61691df 100644 --- a/pkg/hostman/storageman/storage_rbd.go +++ b/pkg/hostman/storageman/storage_rbd.go @@ -16,8 +16,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" - "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -356,7 +355,7 @@ func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content = map[string]interface{}{ "name": s.StorageName, "capacity": capacity, - "status": models.STORAGE_ONLINE, + "status": storagetypes.STORAGE_ONLINE, "zone": s.GetZone(), } return modules.Storages.Put(hostutils.GetComputeSession(context.Background()), s.StorageId, jsonutils.Marshal(content)) diff --git a/pkg/image/options/options.go b/pkg/image/options/options.go index 7f07c220b0..22a312bea7 100644 --- a/pkg/image/options/options.go +++ b/pkg/image/options/options.go @@ -1,14 +1,14 @@ package options import ( - "yunion.io/x/onecloud/pkg/cloudcommon" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/cloudcommon/pending_delete" ) type SImageOptions struct { - cloudcommon.CommonOptions + common_options.CommonOptions - cloudcommon.DBOptions + common_options.DBOptions pending_delete.SPendingDeleteOptions diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go index c98cc441bc..7dd42674c6 100644 --- a/pkg/image/service/service.go +++ b/pkg/image/service/service.go @@ -10,8 +10,10 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/image/models" "yunion.io/x/onecloud/pkg/image/options" _ "yunion.io/x/onecloud/pkg/image/tasks" @@ -27,7 +29,7 @@ func StartService() { opts := &options.Options commonOpts := &opts.CommonOptions dbOpts := &opts.DBOptions - cloudcommon.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE) + common_options.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE) if opts.PortV2 > 0 { log.Infof("Port V2 %d is specified, use v2 port", opts.PortV2) @@ -57,7 +59,7 @@ func StartService() { log.Infof("Target image formats %#v", opts.TargetImageFormats) - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) @@ -69,7 +71,7 @@ func StartService() { cloudcommon.InitDB(dbOpts) - app := cloudcommon.InitApp(commonOpts, true) + app := app_common.InitApp(commonOpts, true) initHandlers(app) if !db.CheckSync(opts.AutoSyncTable) { @@ -85,7 +87,8 @@ func StartService() { cron.Start() - cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + cloudcommon.AppDBInit(app) + app_common.ServeForeverWithCleanup(app, commonOpts, func() { cloudcommon.CloseDB() cron.Stop() diff --git a/pkg/lbagent/options.go b/pkg/lbagent/options.go index 8ac80d5f8e..52804fc348 100644 --- a/pkg/lbagent/options.go +++ b/pkg/lbagent/options.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "yunion.io/x/onecloud/pkg/cloudcommon" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" ) @@ -31,7 +31,7 @@ type LbagentOptions struct { } type Options struct { - cloudcommon.CommonOptions + common_options.CommonOptions LbagentOptions } diff --git a/pkg/logger/options/options.go b/pkg/logger/options/options.go index 13c82de5a7..cd5361ecf8 100644 --- a/pkg/logger/options/options.go +++ b/pkg/logger/options/options.go @@ -1,13 +1,13 @@ package options import ( - "yunion.io/x/onecloud/pkg/cloudcommon" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) type SLoggerOptions struct { - cloudcommon.CommonOptions + common_options.CommonOptions - cloudcommon.DBOptions + common_options.DBOptions } var ( diff --git a/pkg/logger/service/service.go b/pkg/logger/service/service.go index ec2ddccdae..fe82e4f216 100644 --- a/pkg/logger/service/service.go +++ b/pkg/logger/service/service.go @@ -8,8 +8,10 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/logger/options" ) @@ -24,21 +26,22 @@ func StartService() { opts := &options.Options commonOpts := &opts.CommonOptions dbOpts := &opts.DBOptions - cloudcommon.ParseOptions(opts, os.Args, "log.conf", SERVICE_TYPE) + common_options.ParseOptions(opts, os.Args, "log.conf", SERVICE_TYPE) - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) cloudcommon.InitDB(dbOpts) defer cloudcommon.CloseDB() - app := cloudcommon.InitApp(commonOpts, true) + app := app_common.InitApp(commonOpts, true) + cloudcommon.AppDBInit(app) initHandlers(app) if !db.CheckSync(opts.AutoSyncTable) { log.Fatalf("database schema not in sync!") } - cloudcommon.ServeForever(app, commonOpts) + app_common.ServeForever(app, commonOpts) } diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 0bacb008c0..831d8f1a89 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemutils" ) @@ -82,7 +82,7 @@ func (img *SQemuImage) parse() error { } else if strings.HasPrefix(img.Path, "sheepdog") { // sheepdog -> sheepdog[+tcp|+unix]://[host:port]/vdiname[?socket=path][#snapid|#tag] return cloudprovider.ErrNotImplemented - } else if strings.HasPrefix(img.Path, models.STORAGE_RBD) { + } else if strings.HasPrefix(img.Path, storagetypes.STORAGE_RBD) { img.ActualSizeBytes = 0 } else { fileInfo, err := os.Stat(img.Path) diff --git a/pkg/webconsole/options/options.go b/pkg/webconsole/options/options.go index abcdd4a688..b236029933 100644 --- a/pkg/webconsole/options/options.go +++ b/pkg/webconsole/options/options.go @@ -1,15 +1,13 @@ package options -import ( - "yunion.io/x/onecloud/pkg/cloudcommon" -) +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" var ( Options WebConsoleOptions ) type WebConsoleOptions struct { - cloudcommon.CommonOptions + common_options.CommonOptions ApiServer string `help:"API server url to handle websocket connection, usually with public access" default:"http://webconsole.yunion.io"` KubectlPath string `help:"kubectl binary path used to connect k8s cluster" default:"/usr/bin/kubectl"` diff --git a/pkg/webconsole/service/service.go b/pkg/webconsole/service/service.go index 43e165e3ac..bbad6c2f56 100644 --- a/pkg/webconsole/service/service.go +++ b/pkg/webconsole/service/service.go @@ -11,7 +11,8 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/webconsole" o "yunion.io/x/onecloud/pkg/webconsole/options" "yunion.io/x/onecloud/pkg/webconsole/server" @@ -27,7 +28,7 @@ func StartService() { opts := &o.Options commonOpts := &o.Options.CommonOptions - cloudcommon.ParseOptions(opts, os.Args, "webconsole.conf", "webconsole") + common_options.ParseOptions(opts, os.Args, "webconsole.conf", "webconsole") if opts.ApiServer == "" { log.Fatalf("--api-server must specified") @@ -41,7 +42,7 @@ func StartService() { ensureBinExists(binPath) } - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete") }) start() @@ -49,7 +50,7 @@ func StartService() { func start() { commonOpts := &o.Options.CommonOptions - app := cloudcommon.InitApp(commonOpts, false) + app := app_common.InitApp(commonOpts, false) webconsole.InitHandlers(app) root := mux.NewRouter() diff --git a/pkg/yunionconf/options/options.go b/pkg/yunionconf/options/options.go index 1d935e28ba..fab3d201fc 100644 --- a/pkg/yunionconf/options/options.go +++ b/pkg/yunionconf/options/options.go @@ -1,10 +1,10 @@ package options -import "yunion.io/x/onecloud/pkg/cloudcommon" +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" type YunionConfOptions struct { - cloudcommon.CommonOptions - cloudcommon.DBOptions + common_options.CommonOptions + common_options.DBOptions } var ( diff --git a/pkg/yunionconf/service/service.go b/pkg/yunionconf/service/service.go index c822043a40..24c822bc96 100644 --- a/pkg/yunionconf/service/service.go +++ b/pkg/yunionconf/service/service.go @@ -8,8 +8,10 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/yunionconf" "yunion.io/x/onecloud/pkg/yunionconf/models" "yunion.io/x/onecloud/pkg/yunionconf/options" @@ -20,8 +22,8 @@ func StartService() { opts := &options.Options commonOpts := &options.Options.CommonOptions dbOpts := &options.Options.DBOptions - cloudcommon.ParseOptions(opts, os.Args, "yunionconf.conf", "yunionconf") - cloudcommon.InitAuth(commonOpts, func() { + common_options.ParseOptions(opts, os.Args, "yunionconf.conf", "yunionconf") + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) @@ -31,13 +33,14 @@ func StartService() { cloudcommon.InitDB(dbOpts) - app := cloudcommon.InitApp(commonOpts, true) + app := app_common.InitApp(commonOpts, true) yunionconf.InitHandlers(app) + cloudcommon.AppDBInit(app) if db.CheckSync(opts.AutoSyncTable) { err := models.InitDB() if err == nil { - cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + app_common.ServeForeverWithCleanup(app, commonOpts, func() { cloudcommon.CloseDB() }) } else { From b67eb79797040b211d0b2ab4bb2f8988d06c78d4 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 13 Mar 2019 21:07:03 +0800 Subject: [PATCH 09/26] fix cpu set --- pkg/appsrv/appsrv.go | 10 +- pkg/hostman/guestman/guestman.go | 10 +- pkg/hostman/metadata/metadatahandler.go | 2 +- pkg/util/cgrouputils/cgrouputils.go | 20 ++-- pkg/util/cgrouputils/cpuutils.go | 116 ++++++++++++++---------- pkg/util/cgrouputils/cpuutils_test.go | 30 ++++++ 6 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 pkg/util/cgrouputils/cpuutils_test.go diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index ef8cfbf227..4fdae4346b 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -397,7 +397,16 @@ func (app *Application) ListenAndServeWithCleanup(addr string, onStop func()) { func (app *Application) ListenAndServeTLSWithCleanup(addr string, certFile, keyFile string, onStop func()) { s := app.initServer(addr) app.registerCleanShutdown(s, onStop) + app.listenAndServe(s, certFile, keyFile) + app.waitCleanShutdown() +} +func (app *Application) ListenAndServeWithoutCleanup(addr, certFile, keyFile string) { + s := app.initServer(addr) + app.listenAndServe(s, certFile, keyFile) +} + +func (app *Application) listenAndServe(s *http.Server, certFile, keyFile string) { var err error if len(certFile) == 0 && len(keyFile) == 0 { err = s.ListenAndServe() @@ -407,7 +416,6 @@ func (app *Application) ListenAndServeTLSWithCleanup(addr string, certFile, keyF if err != nil && err != http.ErrServerClosed { log.Fatalf("ListAndServer fail: %s", err) } - app.waitCleanShutdown() } func isJsonContentType(r *http.Request) bool { diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 344c60b03d..9b45c9ebde 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path" + "runtime/debug" "strings" "sync" "time" @@ -156,11 +157,18 @@ func (m *SGuestManager) StartCpusetBalancer() { return } go func() { + defer func() { + if r := recover(); r != nil { + debug.PrintStack() + log.Errorf("Cpuset balancer failed %s", r) + } + }() for { + time.Sleep(time.Second * 120) + if options.HostOptions.EnableCpuBinding { m.cpusetBalance() } - time.Sleep(time.Second * 120) } }() } diff --git a/pkg/hostman/metadata/metadatahandler.go b/pkg/hostman/metadata/metadatahandler.go index 520894047f..82ef69935c 100644 --- a/pkg/hostman/metadata/metadatahandler.go +++ b/pkg/hostman/metadata/metadatahandler.go @@ -251,5 +251,5 @@ func StartService(app *appsrv.Application, address string, port int) { addMetadataHandler("", app) addr := net.JoinHostPort(address, strconv.Itoa(port)) log.Infof("Host Metadata Start listen on %s://%s", "http", addr) - app.ListenAndServeWithCleanup(addr, nil) + app.ListenAndServeWithoutCleanup(addr, "", "") } diff --git a/pkg/util/cgrouputils/cgrouputils.go b/pkg/util/cgrouputils/cgrouputils.go index 79fbd9ae5c..1cf340d132 100644 --- a/pkg/util/cgrouputils/cgrouputils.go +++ b/pkg/util/cgrouputils/cgrouputils.go @@ -421,7 +421,9 @@ func (c *CGroupIOTask) init() bool { } func NewCGroupIOTask(pid string, coreNum int) *CGroupIOTask { - return &CGroupIOTask{NewCGroupTask(pid, coreNum)} + task := &CGroupIOTask{NewCGroupTask(pid, coreNum)} + task.SetHand(task) + return task } /** @@ -446,13 +448,15 @@ func (c *CGroupIOHardlimitTask) GetConfig() map[string]string { return config } -func NewCGroupIOHardlimitTask(pid string, mem int, params map[string]int, devId string) CGroupIOHardlimitTask { - return CGroupIOHardlimitTask{ +func NewCGroupIOHardlimitTask(pid string, mem int, params map[string]int, devId string) *CGroupIOHardlimitTask { + task := &CGroupIOHardlimitTask{ CGroupIOTask: NewCGroupIOTask(pid, 0), cpuNum: mem, params: params, devId: devId, } + task.SetHand(task) + return task } /** @@ -477,10 +481,12 @@ func (c *CGroupMemoryTask) GetConfig() map[string]string { return map[string]string{MEMORY_SWAPPINESS: fmt.Sprintf("%d", vm_swappiness)} } -func NewCGroupMemoryTask(pid string, coreNum int) CGroupMemoryTask { - return CGroupMemoryTask{ +func NewCGroupMemoryTask(pid string, coreNum int) *CGroupMemoryTask { + task := &CGroupMemoryTask{ CGroupTask: NewCGroupTask(pid, coreNum), } + task.SetHand(task) + return task } /** @@ -511,10 +517,12 @@ func (c *CGroupCPUSetTask) GetConfig() map[string]string { } func NewCGroupCPUSetTask(pid string, coreNum int, cpuset string) CGroupCPUSetTask { - return CGroupCPUSetTask{ + task := CGroupCPUSetTask{ CGroupTask: NewCGroupTask(pid, coreNum), cpuset: cpuset, } + task.SetHand(&task) + return task } func Init() bool { diff --git a/pkg/util/cgrouputils/cpuutils.go b/pkg/util/cgrouputils/cpuutils.go index 523a9e80d6..e3d820ed3d 100644 --- a/pkg/util/cgrouputils/cpuutils.go +++ b/pkg/util/cgrouputils/cpuutils.go @@ -21,6 +21,8 @@ type CPU struct { func NewCPU() (*CPU, error) { var cpu = new(CPU) + cpu.DieList = make([]*CPUDie, 0) + cpuinfo, err := fileutils2.FileGetContents("/proc/cpuinfo") if err != nil { return nil, err @@ -30,23 +32,30 @@ func NewCPU() (*CPU, error) { for _, line := range strings.Split(cpuinfo, "\n") { parts := strings.Split(line, ":") if len(parts) == 2 { - if parts[0] == "processor" { - val, err := strconv.Atoi(parts[1]) + key := strings.TrimSpace(parts[0]) + val := strings.TrimSpace(parts[1]) + if key == "processor" { + iVal, err := strconv.Atoi(val) if err != nil { return nil, err } - core = NewCPUCore(val) - cpu.AddCore(core) + if core != nil { + cpu.AddCore(core) + } + core = NewCPUCore(iVal) } else if core != nil { - core.setInfo(parts[0], parts[1]) + core.setInfo(key, val) } } } + if core != nil { + cpu.AddCore(core) + } return cpu, nil } func (c *CPU) AddCore(core *CPUCore) { - for len(c.DieList) < core.PhysicalId { + for len(c.DieList) < core.PhysicalId+1 { c.DieList = append(c.DieList, NewCPUDie(len(c.DieList))) } c.DieList[core.PhysicalId].AddCore(core) @@ -176,8 +185,11 @@ func Average(arr []float64) float64 { return total / float64(len(arr)) } -func GetProcessWeight(share int, util float64) float64 { - return float64(share) * (util*0.8 + 30) +func GetProcessWeight(share *int, util float64) float64 { + if share != nil { + return float64(*share) * (util*0.8 + 30) + } + return 0.0 } func NewProcessCPUinfo(pid int) (*ProcessCPUinfo, error) { @@ -185,52 +197,60 @@ func NewProcessCPUinfo(pid int) (*ProcessCPUinfo, error) { cpuinfo.Pid = pid spid := strconv.Itoa(pid) - share := NewCGroupCPUTask(spid, 0).GetParam("cpu.shares") - ishare, err := strconv.Atoi(share) - if err != nil { - log.Errorln(err) - } else { - if ishare != 0 { - cpuinfo.Share = &ishare - proc, err := process.NewProcess(int32(pid)) - if err != nil { - log.Errorln(err) - return nil, err - } - util, err := proc.CPUPercent() - if err != nil { - log.Errorln(err) - return nil, err - } - util /= float64(ishare) - uHistory := FetchHistoryUtil() - - var utils = []float64{} - if _, ok := uHistory[spid]; ok { - utils = uHistory[spid] - } - utils = append(utils, util) - for len(utils) > MAX_HISTORY_UTIL_COUNT { - utils = utils[1:] - } - uHistory[spid] = utils - - cpuinfo.Util = Average(utils) - } - } - - cpuset := NewCGroupCPUSetTask(fmt.Sprintf("%s", pid), 0, "").GetParam("cpuset.cpus") - if len(cpuset) > 0 { - c, err := GetSystemCpu() + cpuTask := NewCGroupCPUTask(spid, 0) + if cpuTask.taskIsExist() { + share := cpuTask.GetParam("cpu.shares") + ishare, err := strconv.Atoi(share) + ishare /= 1024.0 if err != nil { log.Errorln(err) } else { - icpuset := c.GetPhysicalId(ParseCpusetStr(cpuset)) - cpuinfo.Cpuset = &icpuset + cpuinfo.Share = &ishare } } - cpuinfo.Weight = GetProcessWeight(*cpuinfo.Share, cpuinfo.Util) + cpusetTask := NewCGroupCPUSetTask(fmt.Sprintf("%d", pid), 0, "") + if cpusetTask.taskIsExist() { + cpuset := cpusetTask.GetParam("cpuset.cpus") + if len(cpuset) > 0 { + c, err := GetSystemCpu() + if err != nil { + log.Errorln(err) + } else { + icpuset := c.GetPhysicalId(ParseCpusetStr(cpuset)) + cpuinfo.Cpuset = &icpuset + } + } + } + + if cpuinfo.Share != nil { + proc, err := process.NewProcess(int32(pid)) + if err != nil { + log.Errorln(err) + return nil, err + } + util, err := proc.CPUPercent() + if err != nil { + log.Errorln(err) + return nil, err + } + util /= float64(*cpuinfo.Share) + uHistory := FetchHistoryUtil() + + var utils = []float64{} + if _, ok := uHistory[spid]; ok { + utils = uHistory[spid] + } + utils = append(utils, util) + for len(utils) > MAX_HISTORY_UTIL_COUNT { + utils = utils[1:] + } + uHistory[spid] = utils + + cpuinfo.Util = Average(utils) + } + + cpuinfo.Weight = GetProcessWeight(cpuinfo.Share, cpuinfo.Util) return cpuinfo, nil } diff --git a/pkg/util/cgrouputils/cpuutils_test.go b/pkg/util/cgrouputils/cpuutils_test.go new file mode 100644 index 0000000000..f753bfa1cd --- /dev/null +++ b/pkg/util/cgrouputils/cpuutils_test.go @@ -0,0 +1,30 @@ +package cgrouputils + +import ( + "os" + "testing" +) + +func TestCPUUtils(t *testing.T) { + cpu, err := GetSystemCpu() + if err != nil { + t.Errorf("NewCPU error %s", err) + } + + t.Logf("die list %v", cpu.DieList) + t.Logf("physical cpu num %d", cpu.GetPhysicalNum()) + + cpusetStr := cpu.GetCpuset(0) + t.Logf("cpu 0 set str %s", cpusetStr) + physicalNum := cpu.GetPhysicalId(cpusetStr) + if physicalNum != 0 { + t.Errorf("failed get physical id %d from cpuset %s", 0, cpusetStr) + } + + pid := os.Getpid() + proc, err := NewProcessCPUinfo(pid) + if err != nil { + t.Errorf("New process cpu info error %s", err) + } + t.Logf("Shared %v %v %f %f", proc.Share, proc.Cpuset, proc.Util, proc.Weight) +} From ee62f5d1537261e7cddfd301f7994bab607ee569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 15 Mar 2019 14:03:40 +0800 Subject: [PATCH 10/26] =?UTF-8?q?=E9=99=90=E5=88=B6key=E5=BF=85=E9=A1=BB?= =?UTF-8?q?=E6=98=AF=E4=B8=AD=E8=8B=B1=E6=96=87=E3=80=81=E6=95=B0=E5=AD=97?= =?UTF-8?q?=E6=88=96=E8=80=85=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/standalone.go | 7 +++++-- pkg/mcclient/options/base.go | 25 ++++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 45bbfd5716..88702015c3 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "strings" + "unicode" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -279,8 +280,10 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC } func (model *SStandaloneResourceBase) ValidateMetadataKey(key string) error { - if strings.HasPrefix(key, CLOUD_TAG_PREFIX) || strings.ContainsAny(key, `:=#&?$/\`) { - return httperrors.NewInputParameterError(`key cannot start with %s and not contain :=#&?$/\`, CLOUD_TAG_PREFIX) + for _, k := range []rune(key) { + if k != rune('_') && !unicode.IsLetter(k) && !unicode.IsDigit(k) { + return httperrors.NewInputParameterError(`Not support tag key with %s`, string(k)) + } } return nil } diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index af622c7347..f6fdb6cabd 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -3,7 +3,9 @@ package options import ( "fmt" "reflect" + "strings" "time" + "unicode" "yunion.io/x/jsonutils" "yunion.io/x/pkg/gotypes" @@ -186,8 +188,7 @@ type BaseListOptions struct { ExportFile string `help:"Export to file" metavar:"" json:"-"` ExportKeys string `help:"Export field keys"` ExportTexts string `help:"Export field displayname texts" json:"-"` - TagsKey []string `help:"Tag key" json:"-"` - TagsValue []string `help:"Tag key" json:"-"` + Tags []string `help:"Tags info, eg: hypervisor=aliyun、os_type=Linux、os_version"` Manager string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"` Account string `help:"List objects belonging to the cloud account" json:"account,omitempty"` @@ -219,13 +220,23 @@ func (opts *BaseListOptions) Params() (*jsonutils.JSONDict, error) { params.Set("admin", jsonutils.JSONTrue) } } - if len(opts.TagsKey) > 0 { - for i := 0; i < len(opts.TagsKey); i++ { - params.Add(jsonutils.NewString(opts.TagsKey[i]), fmt.Sprintf("tags.%d.key", i)) - if len(opts.TagsValue) > i { - params.Add(jsonutils.NewString(opts.TagsValue[i]), fmt.Sprintf("tags.%d.value", i)) + for idx, tag := range opts.Tags { + tagInfo := strings.Split(tag, "=") + if len(tagInfo) > 2 { + return nil, fmt.Errorf("failed parse tags info %s", tag) + } + if len(tagInfo[0]) == 0 { + return nil, fmt.Errorf("Not support empty key") + } + for _, k := range tagInfo[0] { + if k != rune('_') && !unicode.IsLetter(k) && !unicode.IsDigit(k) { + return nil, fmt.Errorf("Not support tag key with %s", string(k)) } } + params.Add(jsonutils.NewString(tagInfo[0]), fmt.Sprintf("tags.%d.key", idx)) + if len(tagInfo) == 2 { + params.Add(jsonutils.NewString(tagInfo[1]), fmt.Sprintf("tags.%d.value", idx)) + } } return params, nil } From 6c040d4b76e21b5ffa6a53b1ba102ae2a4e730f2 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 15 Mar 2019 16:10:37 +0800 Subject: [PATCH 11/26] skus group by GB --- pkg/compute/models/skus.go | 81 ++++++++++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index e34e90a49e..86a31bf2e1 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "math" "sort" "strconv" "strings" @@ -394,15 +395,46 @@ func (self *SServerSkuManager) AllowGetPropertyInstanceSpecs(ctx context.Context return true } -func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - q := self.Query() - // 未明确指定provider或者public_cloud时，默认查询私有云 - provider, _ := query.GetString("provider") - public_cloud, _ := query.Bool("public_cloud") - if len(provider) > 0 { +// 四舍五入 +func round(n int, step int) int { + q := float64(n) / float64(step) + return int(math.Floor(q+0.5)) * step +} + +// 内存按GB取整 +func roundMem(n int) int { + if n <= 512 { + return 512 + } + + return round(n, 1024) +} + +// step必须是偶数 +func interval(n int, step int) (int, int) { + r := round(n, step) + start := r - step/2 + end := r + step/2 + return start, end +} + +// 计算内存所在区间范围 +func intervalMem(n int) (int, int) { + if n <= 512 { + return 0, 512 + } + return interval(n, 1024) +} + +func providerFilter(q *sqlchemy.SQuery, provider string, public_cloud bool) *sqlchemy.SQuery { + if provider == "all" { + // provider 参数为all时。表示查询所有instance type. + return q + } else if len(provider) > 0 { q = q.Equals("provider", provider) } else if public_cloud { q = q.IsNotEmpty("provider") + q = q.NotIn("provider", []string{CLOUD_PROVIDER_OPENSTACK}) } else { q = q.Filter(sqlchemy.OR( sqlchemy.IsNull(q.Field("provider")), @@ -410,6 +442,15 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use )) } + return q +} + +func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + q := self.Query() + // 未明确指定provider或者public_cloud时，默认查询私有云 + provider, _ := query.GetString("provider") + public_cloud, _ := query.Bool("public_cloud") + q = providerFilter(q, provider, public_cloud) q = excludeSkus(q) // 如果是查询私有云需要忽略zone参数 @@ -452,7 +493,7 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use oc := 0 for i := range skus { nc := skus[i].CpuCoreCount - nm := skus[i].MemorySizeMB + nm := roundMem(skus[i].MemorySizeMB) // 内存按GB取整 if nc > oc { cpus.Add(jsonutils.NewInt(int64(nc))) @@ -615,20 +656,9 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem provider := jsonutils.GetAnyString(query, []string{"provider"}) public_cloud, _ := query.Bool("public_cloud") queryDict := query.(*jsonutils.JSONDict) - if provider == "all" { - // provider 参数为all时。表示查询所有instance type. - queryDict.Remove("provider") - } else if len(provider) > 0 { - q = q.Equals("provider", provider) - } else if public_cloud { - q = q.IsNotEmpty("provider") - } else { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - )) - } - + // 手动处理provider查询 + queryDict.Remove("provider") + q = providerFilter(q, provider, public_cloud) q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { return nil, err @@ -694,6 +724,15 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem q.Equals("prepaid_status", prepaid) } + // 按区间查询内存 + memSizeMB, _ := queryDict.Int("memory_size_mb") + if memSizeMB > 0 { + s, e := intervalMem(int(memSizeMB)) + q.GT("memory_size_mb", s) + q.LE("memory_size_mb", e) + queryDict.Remove("memory_size_mb") + } + q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb")) return q, err } From 6fa0e254efa362fb35c6792035956840c56e73c7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 16 Mar 2019 21:23:43 +0800 Subject: [PATCH 12/26] fix: cloud provider sync time is not set in auto-sync cycles --- pkg/compute/models/cloudproviders.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index f82e20b401..406651598c 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -867,6 +867,9 @@ func (provider *SCloudprovider) GetEnabledCloudproviderRegions() []SCloudprovide } func (provider *SCloudprovider) syncCloudproviderRegions(userCred mcclient.TokenCredential, syncRange *SSyncRange, wg *sync.WaitGroup) { + if wg == nil { + provider.MarkSyncing(userCred) + } cprs := provider.GetEnabledCloudproviderRegions() for i := range cprs { if cprs[i].needSync() { @@ -882,6 +885,9 @@ func (provider *SCloudprovider) syncCloudproviderRegions(userCred mcclient.Token } } } + if wg == nil { + provider.MarkEndSync(userCred) + } } func (provider *SCloudprovider) SyncCallSyncCloudproviderRegions(userCred mcclient.TokenCredential, syncRange *SSyncRange) { From 53ece34f3b424d4430b1198ea06a66456a4a9a12 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Sun, 17 Mar 2019 11:21:17 +0800 Subject: [PATCH 13/26] merge const pkg from cloudcommon to compute --- pkg/baremetal/pxe/dhcp.go | 9 ++++----- pkg/cloudcommon/consts/hosttypes/doc.go | 1 - pkg/cloudcommon/consts/networktypes/doc.go | 1 - pkg/cloudcommon/consts/storagetypes/doc.go | 1 - .../consts/hosttypes => compute/consts}/hosttypes.go | 2 +- .../{loadbalancer_const.go => loadbalancer.go} | 0 .../network_consts.go => compute/consts/networks.go} | 2 +- .../storagetypes.go => compute/consts/storages.go} | 2 +- pkg/hostman/guestman/qemu-kvm.go | 6 +++--- pkg/hostman/hostinfo/hostinfo.go | 10 +++++----- pkg/hostman/storageman/core.go | 12 ++++++------ pkg/hostman/storageman/disk_local.go | 4 ++-- pkg/hostman/storageman/disk_nfs.go | 4 ++-- pkg/hostman/storageman/disk_rbd.go | 4 ++-- pkg/hostman/storageman/imagecachemanager_rbd.go | 4 ++-- pkg/hostman/storageman/storage_local.go | 4 ++-- pkg/hostman/storageman/storage_nfs.go | 8 ++++---- pkg/hostman/storageman/storage_rbd.go | 8 ++++---- pkg/util/qemuimg/qemuimg.go | 4 ++-- 19 files changed, 41 insertions(+), 45 deletions(-) delete mode 100644 pkg/cloudcommon/consts/hosttypes/doc.go delete mode 100644 pkg/cloudcommon/consts/networktypes/doc.go delete mode 100644 pkg/cloudcommon/consts/storagetypes/doc.go rename pkg/{cloudcommon/consts/hosttypes => compute/consts}/hosttypes.go (96%) rename pkg/compute/consts/{loadbalancer_const.go => loadbalancer.go} (100%) rename pkg/{cloudcommon/consts/networktypes/network_consts.go => compute/consts/networks.go} (89%) rename pkg/{cloudcommon/consts/storagetypes/storagetypes.go => compute/consts/storages.go} (96%) diff --git a/pkg/baremetal/pxe/dhcp.go b/pkg/baremetal/pxe/dhcp.go index 71dbd650b1..bca1951992 100644 --- a/pkg/baremetal/pxe/dhcp.go +++ b/pkg/baremetal/pxe/dhcp.go @@ -10,9 +10,8 @@ import ( "yunion.io/x/log" o "yunion.io/x/onecloud/pkg/baremetal/options" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/hosttypes" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/networktypes" "yunion.io/x/onecloud/pkg/cloudcommon/types" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/dhcp" ) @@ -166,7 +165,7 @@ func (h *DHCPHandler) fetchConfig() (*dhcp.ResponseConfig, error) { h.baremetalInstance = bmInstance ipmiNic := h.baremetalInstance.GetIPMINic(h.ClientMac) if ipmiNic != nil && ipmiNic.Mac == h.ClientMac.String() { - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, networktypes.NETWORK_TYPE_IPMI) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, consts.NETWORK_TYPE_IPMI) if err != nil { return nil, err } @@ -216,7 +215,7 @@ func (h *DHCPHandler) findNetworkConf(filterUseIp bool) (*types.SNetworkConfig, func (h *DHCPHandler) findBaremetalsOfAnyMac(isBaremetal bool) (*modules.ListResult, error) { session := h.baremetalManager.GetClientSession() params := jsonutils.NewDict() - params.Add(jsonutils.NewString(hosttypes.HOST_TYPE_BAREMETAL), "host_type") + params.Add(jsonutils.NewString(consts.HOST_TYPE_BAREMETAL), "host_type") params.Add(jsonutils.NewString(h.ClientMac.String()), "any_mac") if isBaremetal { params.Add(jsonutils.JSONTrue, "is_baremetal") @@ -293,7 +292,7 @@ func (h *DHCPHandler) doInitBaremetalAdminNetif(desc jsonutils.JSONObject) error if err != nil { return err } - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, networktypes.NETWORK_TYPE_PXE) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, consts.NETWORK_TYPE_PXE) return err } diff --git a/pkg/cloudcommon/consts/hosttypes/doc.go b/pkg/cloudcommon/consts/hosttypes/doc.go deleted file mode 100644 index 6bca1d2a51..0000000000 --- a/pkg/cloudcommon/consts/hosttypes/doc.go +++ /dev/null @@ -1 +0,0 @@ -package hosttypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/hosttypes" diff --git a/pkg/cloudcommon/consts/networktypes/doc.go b/pkg/cloudcommon/consts/networktypes/doc.go deleted file mode 100644 index caf722ad4a..0000000000 --- a/pkg/cloudcommon/consts/networktypes/doc.go +++ /dev/null @@ -1 +0,0 @@ -package networktypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/networktypes" diff --git a/pkg/cloudcommon/consts/storagetypes/doc.go b/pkg/cloudcommon/consts/storagetypes/doc.go deleted file mode 100644 index 868d183308..0000000000 --- a/pkg/cloudcommon/consts/storagetypes/doc.go +++ /dev/null @@ -1 +0,0 @@ -package storagetypes // import "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" diff --git a/pkg/cloudcommon/consts/hosttypes/hosttypes.go b/pkg/compute/consts/hosttypes.go similarity index 96% rename from pkg/cloudcommon/consts/hosttypes/hosttypes.go rename to pkg/compute/consts/hosttypes.go index 381543f122..8aa2ff88ae 100644 --- a/pkg/cloudcommon/consts/hosttypes/hosttypes.go +++ b/pkg/compute/consts/hosttypes.go @@ -1,4 +1,4 @@ -package hosttypes +package consts const ( HOST_TYPE_BAREMETAL = "baremetal" diff --git a/pkg/compute/consts/loadbalancer_const.go b/pkg/compute/consts/loadbalancer.go similarity index 100% rename from pkg/compute/consts/loadbalancer_const.go rename to pkg/compute/consts/loadbalancer.go diff --git a/pkg/cloudcommon/consts/networktypes/network_consts.go b/pkg/compute/consts/networks.go similarity index 89% rename from pkg/cloudcommon/consts/networktypes/network_consts.go rename to pkg/compute/consts/networks.go index 49cdd7fd7c..f51058c06a 100644 --- a/pkg/cloudcommon/consts/networktypes/network_consts.go +++ b/pkg/compute/consts/networks.go @@ -1,4 +1,4 @@ -package networktypes +package consts const ( NETWORK_TYPE_GUEST = "guest" diff --git a/pkg/cloudcommon/consts/storagetypes/storagetypes.go b/pkg/compute/consts/storages.go similarity index 96% rename from pkg/cloudcommon/consts/storagetypes/storagetypes.go rename to pkg/compute/consts/storages.go index 51d29d9103..b3a158be4a 100644 --- a/pkg/cloudcommon/consts/storagetypes/storagetypes.go +++ b/pkg/compute/consts/storages.go @@ -1,4 +1,4 @@ -package storagetypes +package consts // TODO: move models/storages.go storage types to this file const ( diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 06a3e254e7..0acf8626d5 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -18,7 +18,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -698,7 +698,7 @@ func (s *SKVMGuestInstance) delTmpDisks(ctx context.Context, migrated bool) erro if disk.Contains("path") { diskPath, _ := disk.GetString("path") d := storageman.GetManager().GetDiskByPath(diskPath) - if d != nil && d.GetType() == storagetypes.STORAGE_LOCAL && migrated { + if d != nil && d.GetType() == consts.STORAGE_LOCAL && migrated { if err := d.DeleteAllSnapshot(); err != nil { log.Errorln(err) return err @@ -1241,7 +1241,7 @@ func (s *SKVMGuestInstance) PrepareMigrate(liveMigrage bool) (*jsonutils.JSONDic if disk.Contains("path") { diskPath, _ := disk.GetString("path") d := storageman.GetManager().GetDiskByPath(diskPath) - if d.GetType() == storagetypes.STORAGE_LOCAL { + if d.GetType() == consts.STORAGE_LOCAL { back, err := d.PrepareMigrate(liveMigrage) if err != nil { return nil, err diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 523459ff9d..c9e536cd1f 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -16,9 +16,9 @@ import ( "yunion.io/x/pkg/utils" bare2 "yunion.io/x/onecloud/pkg/baremetal" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" bare1 "yunion.io/x/onecloud/pkg/compute/baremetal" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -320,9 +320,9 @@ func (h *SHostInfo) checkSystemServices() error { } func (h *SHostInfo) detectiveStorageSystem() { - var stype = storagetypes.DISK_TYPE_ROTATE + var stype = consts.DISK_TYPE_ROTATE if options.HostOptions.DiskIsSsd { - stype = storagetypes.DISK_TYPE_SSD + stype = consts.DISK_TYPE_SSD } h.sysinfo.StorageType = stype } @@ -1128,7 +1128,7 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { log.Infof("Storage %s(%s) mountpoint %s", storageName, storagetype, mountPoint) - if !utils.IsInStringArray(storagetype, storagetypes.Local) { + if !utils.IsInStringArray(storagetype, consts.Local) { storage := storageManager.NewSharedStorageInstance(mountPoint, storagetype) if storage != nil { storage.SetStoragecacheId(storagecacheId) @@ -1145,7 +1145,7 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { storage.SetStorageInfo(storageId, storageName, storageConf) } else { // XXX hack: storage type baremetal is a converted host，reserve storage - if storagetype != storagetypes.STORAGE_BAREMETAL { + if storagetype != consts.STORAGE_BAREMETAL { detachStorages = append(detachStorages, hs) } } diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 316fe55c1b..87e4954d7d 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" @@ -73,9 +73,9 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) { } func (s *SStorageManager) Remove(storage IStorage) { - if storage.StorageType() == storagetypes.STORAGE_NFS { + if storage.StorageType() == consts.STORAGE_NFS { delete(s.NfsStorageImagecacheManagers, storage.GetStoragecacheId()) - } else if storage.StorageType() == storagetypes.STORAGE_RBD { + } else if storage.StorageType() == consts.STORAGE_RBD { delete(s.RbdStorageImagecacheManagers, storage.GetStoragecacheId()) } for index, iS := range s.Storages { @@ -253,9 +253,9 @@ func (s *SStorageManager) NewSharedStorageInstance(mountPoint, storageType strin } func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheId, imagecachePath string, storage IStorage) { - if storageType == storagetypes.STORAGE_NFS { + if storageType == consts.STORAGE_NFS { s.InitNfsStorageImagecache(storagecacheId, imagecachePath) - } else if storageType == storagetypes.STORAGE_RBD { + } else if storageType == consts.STORAGE_RBD { if rbdStorage := s.GetStoragecacheById(storagecacheId); rbdStorage == nil { // Done s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId) @@ -280,7 +280,7 @@ func (s *SStorageManager) AddRbdStorageImagecache(imagecachePath string, storage s.RbdStorageImagecacheManagers = map[string]IImageCacheManger{} } if _, ok := s.RbdStorageImagecacheManagers[storagecacheId]; !ok { - if imagecache := NewImageCacheManager(s, imagecachePath, storage, storagecacheId, storagetypes.STORAGE_RBD); imagecache != nil { + if imagecache := NewImageCacheManager(s, imagecachePath, storage, storagecacheId, consts.STORAGE_RBD); imagecache != nil { s.RbdStorageImagecacheManagers[storagecacheId] = imagecache return } diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index 512ad3654d..bc9d004922 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" @@ -36,7 +36,7 @@ func NewLocalDisk(storage IStorage, id string) *SLocalDisk { } func (d *SBaseDisk) GetType() string { - return storagetypes.STORAGE_LOCAL + return consts.STORAGE_LOCAL } func (d *SLocalDisk) getPath() string { diff --git a/pkg/hostman/storageman/disk_nfs.go b/pkg/hostman/storageman/disk_nfs.go index df349d0928..9c7450f629 100644 --- a/pkg/hostman/storageman/disk_nfs.go +++ b/pkg/hostman/storageman/disk_nfs.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" ) type SNFSDisk struct { @@ -20,7 +20,7 @@ func NewNFSDisk(storage IStorage, id string) *SNFSDisk { } func (d *SNFSDisk) GetType() string { - return storagetypes.STORAGE_NFS + return consts.STORAGE_NFS } func (d *SNFSDisk) CreateFromTemplate(ctx context.Context, imageId, format string, size int64) (jsonutils.JSONObject, error) { diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index 18bd1a6150..f96bcd0b8e 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -13,7 +13,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) @@ -28,7 +28,7 @@ func NewRBDDisk(storage IStorage, id string) *SRBDDisk { } func (d *SRBDDisk) GetType() string { - return storagetypes.STORAGE_RBD + return consts.STORAGE_RBD } func (d *SRBDDisk) Probe() error { diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go index 14544b533a..fc7eb73c40 100644 --- a/pkg/hostman/storageman/imagecachemanager_rbd.go +++ b/pkg/hostman/storageman/imagecachemanager_rbd.go @@ -10,7 +10,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) @@ -49,7 +49,7 @@ func (factory *SRbdImageCacheManagerFactory) NewImageCacheManager(manager *SStor } func (factory *SRbdImageCacheManagerFactory) StorageType() string { - return storagetypes.STORAGE_RBD + return consts.STORAGE_RBD } func init() { diff --git a/pkg/hostman/storageman/storage_local.go b/pkg/hostman/storageman/storage_local.go index c4e2de339c..59b0867608 100644 --- a/pkg/hostman/storageman/storage_local.go +++ b/pkg/hostman/storageman/storage_local.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -56,7 +56,7 @@ func (s *SLocalStorage) GetFuseMountPath() string { } func (s *SLocalStorage) StorageType() string { - return storagetypes.STORAGE_LOCAL + return consts.STORAGE_LOCAL } func (s *SLocalStorage) GetSnapshotDir() string { diff --git a/pkg/hostman/storageman/storage_nfs.go b/pkg/hostman/storageman/storage_nfs.go index 9e9dc91d0e..6bc6511c60 100644 --- a/pkg/hostman/storageman/storage_nfs.go +++ b/pkg/hostman/storageman/storage_nfs.go @@ -8,7 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/fileutils2" @@ -27,7 +27,7 @@ func (factory *SNFSStorageFactory) NewStorage(manager *SStorageManager, mountPoi } func (factory *SNFSStorageFactory) StorageType() string { - return storagetypes.STORAGE_NFS + return consts.STORAGE_NFS } type SNFSStorage struct { @@ -44,7 +44,7 @@ func NewNFSStorage(manager *SStorageManager, path string) *SNFSStorage { } func (s *SNFSStorage) StorageType() string { - return storagetypes.STORAGE_NFS + return consts.STORAGE_NFS } func (s *SNFSStorage) CreateDisk(diskId string) IDisk { @@ -84,7 +84,7 @@ func (s *SNFSStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content := jsonutils.NewDict() content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb()))) content.Set("storage_type", jsonutils.NewString(s.StorageType())) - content.Set("status", jsonutils.NewString(storagetypes.STORAGE_ONLINE)) + content.Set("status", jsonutils.NewString(consts.STORAGE_ONLINE)) content.Set("zone", jsonutils.NewString(s.GetZone())) log.Infof("Sync storage info %s", s.StorageId) res, err := modules.Storages.Put( diff --git a/pkg/hostman/storageman/storage_rbd.go b/pkg/hostman/storageman/storage_rbd.go index 95c61691df..c01dc85c1f 100644 --- a/pkg/hostman/storageman/storage_rbd.go +++ b/pkg/hostman/storageman/storage_rbd.go @@ -16,7 +16,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -54,7 +54,7 @@ func (factory *SRbdStorageFactory) NewStorage(manager *SStorageManager, mountPoi } func (factory *SRbdStorageFactory) StorageType() string { - return storagetypes.STORAGE_RBD + return consts.STORAGE_RBD } func init() { @@ -62,7 +62,7 @@ func init() { } func (s *SRbdStorage) StorageType() string { - return storagetypes.STORAGE_RBD + return consts.STORAGE_RBD } func (s *SRbdStorage) GetSnapshotPathByIds(diskId, snapshotId string) string { @@ -355,7 +355,7 @@ func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content = map[string]interface{}{ "name": s.StorageName, "capacity": capacity, - "status": storagetypes.STORAGE_ONLINE, + "status": consts.STORAGE_ONLINE, "zone": s.GetZone(), } return modules.Storages.Put(hostutils.GetComputeSession(context.Background()), s.StorageId, jsonutils.Marshal(content)) diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 831d8f1a89..46d8eb49b2 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/cloudcommon/consts/storagetypes" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemutils" ) @@ -82,7 +82,7 @@ func (img *SQemuImage) parse() error { } else if strings.HasPrefix(img.Path, "sheepdog") { // sheepdog -> sheepdog[+tcp|+unix]://[host:port]/vdiname[?socket=path][#snapid|#tag] return cloudprovider.ErrNotImplemented - } else if strings.HasPrefix(img.Path, storagetypes.STORAGE_RBD) { + } else if strings.HasPrefix(img.Path, consts.STORAGE_RBD) { img.ActualSizeBytes = 0 } else { fileInfo, err := os.Stat(img.Path) From b49623a7e590a204d436726320c2cf6ec8811b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 18 Mar 2019 11:07:49 +0800 Subject: [PATCH 14/26] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=85=AC=E6=9C=89?= =?UTF-8?q?=E4=BA=91key=E5=8A=A0=E4=B8=8A=E5=89=8D=E7=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/hostdrivers/managedvirtual.go | 2 +- pkg/compute/models/syncutils.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index 2061231381..a7f501d75e 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -242,7 +242,7 @@ func (self *SManagedVirtualizationHostDriver) RequestAllocateDiskOnStorage(ctx c log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) } else { for key, value := range meta { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + if err := disk.SetMetadata(ctx, "ext:"+key, value, task.GetUserCred()); err != nil { log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) } } diff --git a/pkg/compute/models/syncutils.go b/pkg/compute/models/syncutils.go index e2a2300e95..155fed2e73 100644 --- a/pkg/compute/models/syncutils.go +++ b/pkg/compute/models/syncutils.go @@ -20,6 +20,7 @@ func getSyncOwnerProjectId(manager db.IModelManager, userCred mcclient.TokenCred type IMetadataSetter interface { SetAllMetadata(ctx context.Context, meta map[string]interface{}, userCred mcclient.TokenCredential) error + SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error } func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.ICloudResource) error { @@ -31,8 +32,9 @@ func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model log.Errorf("Get VM Metadata error: %v", err) return err } - - return model.SetAllMetadata(ctx, meta, userCred) + for key, value := range meta { + model.SetMetadata(ctx, "ext:"+key, value, userCred) + } } return nil } From 4b6ddfc3cf36230fac78019fd4d79d3aaa175a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 18 Mar 2019 20:03:11 +0800 Subject: [PATCH 15/26] =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E7=9A=84project=5Fsrc=E4=B8=BAlocal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/virtualresource.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index e41e6bdfbb..84a3bf9a4d 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -133,6 +133,7 @@ func (model *SVirtualResourceBase) CustomizeCreate(ctx context.Context, userCred } else { model.IsSystem = false } + model.ProjectSrc = string(PROJECT_SOURCE_LOCAL) return model.SStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) } From ca39f480580030b8f25f686a08de8a0c23d1e90d Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 18 Mar 2019 22:16:20 +0800 Subject: [PATCH 16/26] fix: more accurate sync_status update time --- pkg/compute/models/cloudaccounts.go | 32 ++++++++-- pkg/compute/models/cloudproviderregions.go | 43 +++++++++----- pkg/compute/models/cloudproviders.go | 58 ++++++++++++++----- pkg/compute/models/cloudsync.go | 2 +- pkg/compute/tasks/cloud_account_sync_task.go | 21 +++---- .../tasks/cloud_provider_sync_info_task.go | 11 ++-- 6 files changed, 116 insertions(+), 51 deletions(-) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 5f34a695f1..d3da154f1a 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -406,7 +406,24 @@ func (self *SCloudaccount) MarkSyncing(userCred mcclient.TokenCredential) error return nil } -func (self *SCloudaccount) MarkEndSync(userCred mcclient.TokenCredential) error { +func (self *SCloudaccount) MarkEndSyncWithLock(ctx context.Context, userCred mcclient.TokenCredential) error { + lockman.LockObject(ctx, self) + defer lockman.ReleaseObject(ctx, self) + + if self.SyncStatus == CLOUD_PROVIDER_SYNC_STATUS_IDLE { + return nil + } + + providers := self.GetCloudproviders() + for i := range providers { + if providers[i].SyncStatus != CLOUD_PROVIDER_SYNC_STATUS_IDLE { + return nil + } + } + return self.markEndSync(userCred) +} + +func (self *SCloudaccount) markEndSync(userCred mcclient.TokenCredential) error { _, err := db.Update(self, func() error { self.SyncStatus = CLOUD_PROVIDER_SYNC_STATUS_IDLE self.LastSyncEndAt = timeutils.UtcNow() @@ -1102,8 +1119,8 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc account.MarkSyncing(userCred) subaccounts, err := account.probeAccountStatus(ctx, userCred) if err != nil { - account.markAccountDiscconected(ctx, userCred) account.markAllProvidersDicconnected(ctx, userCred) + account.markAccountDiscconected(ctx, userCred) return err } account.markAccountConnected(ctx, userCred) @@ -1123,16 +1140,23 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre log.Debugf("syncAccountStatus %s %s", account.Id, account.Name) err := account.syncAccountStatus(ctx, userCred) if waitChan != nil { + if err != nil { + account.markEndSync(userCred) + } waitChan <- err } else { + syncCnt := 0 if err == nil && autoSync && account.Enabled && account.EnableAutoSync { syncRange := SSyncRange{FullSync: true} providers := account.GetEnabledCloudproviders() for i := range providers { - providers[i].syncCloudproviderRegions(userCred, &syncRange, nil) + providers[i].syncCloudproviderRegions(ctx, userCred, &syncRange, nil, autoSync) + syncCnt += 1 } } - account.MarkEndSync(userCred) + if syncCnt == 0 { + account.markEndSync(userCred) + } } }) } diff --git a/pkg/compute/models/cloudproviderregions.go b/pkg/compute/models/cloudproviderregions.go index bc3c1821c3..9bb3fbf55e 100644 --- a/pkg/compute/models/cloudproviderregions.go +++ b/pkg/compute/models/cloudproviderregions.go @@ -199,7 +199,19 @@ func (self *SCloudproviderregion) markSyncing(userCred mcclient.TokenCredential) return nil } -func (self *SCloudproviderregion) markEndSync(userCred mcclient.TokenCredential, syncResults SSyncResultSet) error { +func (self *SCloudproviderregion) markEndSync(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet) error { + err := self.markEndSyncInternal(userCred, syncResults) + if err != nil { + return err + } + err = self.GetProvider().markEndSyncWithLock(ctx, userCred) + if err != nil { + return err + } + return nil +} + +func (self *SCloudproviderregion) markEndSyncInternal(userCred mcclient.TokenCredential, syncResults SSyncResultSet) error { _, err := db.Update(self, func() error { self.SyncStatus = CLOUD_PROVIDER_SYNC_STATUS_IDLE self.LastSyncEndAt = timeutils.UtcNow() @@ -230,11 +242,10 @@ func (set SSyncResultSet) Add(manager db.IModelManager, result compare.SyncResul } func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange) error { - err := self.markSyncing(userCred) - if err != nil { - log.Errorf("start sync sql fail?? %s", err) - return err - } + syncResults := SSyncResultSet{} + + self.markSyncing(userCred) + defer self.markEndSync(ctx, userCred, syncResults) localRegion := self.GetRegion() provider := self.GetProvider() @@ -244,8 +255,6 @@ func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient. return err } - syncResults := SSyncResultSet{} - if localRegion.isManaged() { remoteRegion, err := driver.GetIRegionById(localRegion.ExternalId) if err == nil { @@ -255,14 +264,13 @@ func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient. err = syncOnPremiseCloudProviderInfo(ctx, userCred, syncResults, provider, driver, syncRange) } + if err != nil { + log.Errorf("dosync fail %s", err) + } + log.Debugf("%s", jsonutils.Marshal(syncResults)) - err = self.markEndSync(userCred, syncResults) - if err != nil { - log.Errorf("mark end sync failed...") - return err - } - return nil + return err } func (self *SCloudproviderregion) getSyncTaskKey() string { @@ -284,7 +292,7 @@ func (self *SCloudproviderregion) submitSyncTask(userCred mcclient.TokenCredenti }) } -func (cpr *SCloudproviderregion) needSync() bool { +func (cpr *SCloudproviderregion) needAutoSync() bool { if cpr.LastSyncEndAt.IsZero() { return true } @@ -297,7 +305,10 @@ func (cpr *SCloudproviderregion) needSync() bool { isEmpty = cpr.isEmptyPublicCloud() } if isEmpty { - intval = intval * 16 // no need to check empty region + intval = intval * 16 // no need to check empty region + if intval > 24*3600 { // at least once everyday + intval = 24 * 3600 + } region := cpr.GetRegion() log.Debugf("empty region %s! no need to check so frequently", region.GetName()) } diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 406651598c..02a02245b1 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -15,6 +15,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -525,7 +526,7 @@ func (self *SCloudprovider) markStartSync(userCred mcclient.TokenCredential) err return nil } -func (self *SCloudprovider) MarkSyncing(userCred mcclient.TokenCredential) error { +func (self *SCloudprovider) markSyncing(userCred mcclient.TokenCredential) error { _, err := db.Update(self, func() error { self.SyncStatus = CLOUD_PROVIDER_SYNC_STATUS_SYNCING self.LastSync = timeutils.UtcNow() @@ -539,7 +540,34 @@ func (self *SCloudprovider) MarkSyncing(userCred mcclient.TokenCredential) error return nil } -func (self *SCloudprovider) MarkEndSync(userCred mcclient.TokenCredential) error { +func (self *SCloudprovider) markEndSyncWithLock(ctx context.Context, userCred mcclient.TokenCredential) error { + err := func() error { + lockman.LockObject(ctx, self) + defer lockman.ReleaseObject(ctx, self) + + cprs := self.GetCloudproviderRegions() + for i := range cprs { + if cprs[i].SyncStatus != CLOUD_PROVIDER_SYNC_STATUS_IDLE { + return nil + } + } + + err := self.markEndSync(userCred) + if err != nil { + return err + } + return nil + }() + + if err != nil { + return err + } + + account := self.GetCloudaccount() + return account.MarkEndSyncWithLock(ctx, userCred) +} + +func (self *SCloudprovider) markEndSync(userCred mcclient.TokenCredential) error { _, err := db.Update(self, func() error { self.SyncStatus = CLOUD_PROVIDER_SYNC_STATUS_IDLE self.LastSyncEndAt = timeutils.UtcNow() @@ -857,22 +885,22 @@ func (provider *SCloudprovider) prepareCloudproviderRegions(ctx context.Context, return cprs, nil } -func (provider *SCloudprovider) GetEnabledCloudproviderRegions() []SCloudproviderregion { +func (provider *SCloudprovider) GetCloudproviderRegions() []SCloudproviderregion { q := CloudproviderRegionManager.Query() - q = q.IsTrue("enabled") q = q.Equals("cloudprovider_id", provider.Id) - q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE) + // q = q.IsTrue("enabled") + // q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE) return CloudproviderRegionManager.fetchRecordsByQuery(q) } -func (provider *SCloudprovider) syncCloudproviderRegions(userCred mcclient.TokenCredential, syncRange *SSyncRange, wg *sync.WaitGroup) { - if wg == nil { - provider.MarkSyncing(userCred) - } - cprs := provider.GetEnabledCloudproviderRegions() +func (provider *SCloudprovider) syncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, wg *sync.WaitGroup, autoSync bool) { + provider.markSyncing(userCred) + cprs := provider.GetCloudproviderRegions() + syncCnt := 0 for i := range cprs { - if cprs[i].needSync() { + if cprs[i].Enabled && cprs[i].CanSync() && (!autoSync || cprs[i].needAutoSync()) { + syncCnt += 1 var waitChan chan bool = nil if wg != nil { wg.Add(1) @@ -885,14 +913,14 @@ func (provider *SCloudprovider) syncCloudproviderRegions(userCred mcclient.Token } } } - if wg == nil { - provider.MarkEndSync(userCred) + if syncCnt == 0 { + provider.markEndSyncWithLock(ctx, userCred) } } -func (provider *SCloudprovider) SyncCallSyncCloudproviderRegions(userCred mcclient.TokenCredential, syncRange *SSyncRange) { +func (provider *SCloudprovider) SyncCallSyncCloudproviderRegions(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange) { var wg sync.WaitGroup - provider.syncCloudproviderRegions(userCred, syncRange, &wg) + provider.syncCloudproviderRegions(ctx, userCred, syncRange, &wg, false) wg.Wait() } diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index 1f21b419a9..52328310d5 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -24,7 +24,7 @@ type SSyncableBaseResource struct { func (self *SSyncableBaseResource) CanSync() bool { if self.SyncStatus == CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == CLOUD_PROVIDER_SYNC_STATUS_SYNCING { - if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 900*time.Second { + if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second { return true } else { return false diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index 14ad09adf9..3d936e35db 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -29,7 +29,7 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda err := cloudaccount.SyncCallSyncAccountTask(ctx, self.UserCred) if err != nil { - cloudaccount.MarkEndSync(self.UserCred) + cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred) db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred) self.SetStageFailed(ctx, err.Error()) logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false) @@ -45,24 +45,25 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda } if !syncRange.NeedSyncInfo() { - cloudaccount.MarkEndSync(self.UserCred) - db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_COMPLETE, "", self.UserCred) - self.SetStageComplete(ctx, nil) - logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, "", self.UserCred, true) + self.OnCloudaccountSyncComplete(ctx, obj, nil) return } - self.SetStage("on_cloudaccount_sync_complete", nil) - cloudproviders := cloudaccount.GetEnabledCloudproviders() - for i := range cloudproviders { - cloudproviders[i].StartSyncCloudProviderInfoTask(ctx, self.UserCred, &syncRange, self.GetId()) + + if len(cloudproviders) > 0 { + self.SetStage("on_cloudaccount_sync_complete", nil) + for i := range cloudproviders { + cloudproviders[i].StartSyncCloudProviderInfoTask(ctx, self.UserCred, &syncRange, self.GetId()) + } + } else { + self.OnCloudaccountSyncComplete(ctx, obj, nil) } } func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { cloudaccount := obj.(*models.SCloudaccount) - cloudaccount.MarkEndSync(self.UserCred) + cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred) db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_COMPLETE, "", self.UserCred) self.SetStageComplete(ctx, nil) logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, "", self.UserCred, true) diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index e3220a41d7..ac79c53b15 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -50,8 +50,6 @@ func (self *CloudProviderSyncInfoTask) OnInit(ctx context.Context, obj db.IStand db.OpsLog.LogEvent(provider, db.ACT_SYNCING_HOST, "", self.UserCred) - provider.MarkSyncing(self.UserCred) - syncRange := models.SSyncRange{} syncRangeJson, _ := self.Params.Get("sync_range") if syncRangeJson != nil { @@ -59,14 +57,17 @@ func (self *CloudProviderSyncInfoTask) OnInit(ctx context.Context, obj db.IStand } syncRange.Normalize() - provider.SyncCallSyncCloudproviderRegions(self.UserCred, &syncRange) + self.SetStage("OnSyncCloudProviderInfoComplete", nil) - self.OnSyncCloudProviderInfoComplete(ctx, provider, nil) + taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) { + provider.SyncCallSyncCloudproviderRegions(ctx, self.UserCred, &syncRange) + return nil, nil + }) } func (self *CloudProviderSyncInfoTask) OnSyncCloudProviderInfoComplete(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { provider := obj.(*models.SCloudprovider) - provider.MarkEndSync(self.UserCred) + // provider.MarkEndSync(self.UserCred) provider.CleanSchedCache() self.SetStageComplete(ctx, nil) db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, "", self.UserCred) From 43daa1ceb420d8b6176e92ee7c023d51c9c8e962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 18 Mar 2019 14:27:23 +0800 Subject: [PATCH 17/26] =?UTF-8?q?=E7=94=A8=E6=88=B7=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E6=A0=87=E7=AD=BE=E5=8A=A0=E5=89=8D=E7=BC=80user:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/servers.go | 27 +++-- pkg/cloudcommon/db/metadata.go | 125 ++++++++++++++++++---- pkg/cloudcommon/db/standalone.go | 59 ++++++++-- pkg/compute/hostdrivers/managedvirtual.go | 13 +-- pkg/compute/models/syncutils.go | 4 + pkg/mcclient/options/metadata.go | 6 +- pkg/mcclient/options/servers.go | 23 ++++ 7 files changed, 200 insertions(+), 57 deletions(-) diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 7acdcbf424..9ad2743bb5 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -85,16 +85,25 @@ func init() { return nil }) - type ServerSetMetadataOptions struct { - ID string `help:"ID or name of server" json:"-"` - KEY string `help:"Metadata key"` - VALUE string `help:"Metadata Value"` - } + R(&options.ServerMetadataOptions{}, "server-add-tag", "Set tag of a server", func(s *mcclient.ClientSession, opts *options.ServerMetadataOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + result, err := modules.Servers.PerformAction(s, opts.ID, "user-metadata", params) + if err != nil { + return err + } + printObject(result) + return nil + }) - R(&ServerSetMetadataOptions{}, "server-set-metadata", "Set metadata of a server", func(s *mcclient.ClientSession, opts *ServerSetMetadataOptions) error { - params := jsonutils.NewDict() - params.Add(jsonutils.NewString(opts.VALUE), opts.KEY) - result, err := modules.Servers.PerformAction(s, opts.ID, "metadata", params) + R(&options.ServerMetadataOptions{}, "server-set-tag", "Set tag of a server", func(s *mcclient.ClientSession, opts *options.ServerMetadataOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + result, err := modules.Servers.PerformAction(s, opts.ID, "set-user-metadata", params) if err != nil { return err } diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 16f29e801f..bd9589d30e 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -8,6 +8,7 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/sqlchemy" @@ -19,6 +20,12 @@ import ( const ( SYSTEM_ADMIN_PREFIX = "_" CLOUD_TAG_PREFIX = "ext:" + USER_TAG_PREFIX = "user:" + + TAG_DELETE_RANGE_USER = "user" + TAG_DELETE_RANGE_CLOUD = "cloud" + TAG_DELETE_RANGE_SYS = "sys" + TAG_DELETE_RANGE_ALL = "all" ) type SMetadataManager struct { @@ -32,6 +39,7 @@ type SMetadata struct { Key string `width:"64" charset:"utf8" primary:"true" list:"user" get:"user"` // = Column(VARCHAR(64, charset='ascii'), primary_key=True) Value string `charset:"utf8" list:"user" get:"user"` // = Column(TEXT(charset='utf8'), nullable=True) UpdatedAt time.Time `nullable:"false" updated_at:"true"` // = Column(DateTime, default=get_utcnow, nullable=False, onupdate=get_utcnow) + Deleted bool `nullable:"false" default:"false" index:"true"` } var Metadata *SMetadataManager @@ -63,6 +71,23 @@ func GetObjectIdstr(model IModel) string { return fmt.Sprintf("%s::%s", model.GetModelManager().Keyword(), model.GetId()) } +func (manager *SMetadataManager) Query(fields ...string) *sqlchemy.SQuery { + return manager.SModelBaseManager.Query(fields...).IsFalse("deleted") +} + +func (manager *SMetadataManager) RawQuery(fields ...string) *sqlchemy.SQuery { + return manager.SModelBaseManager.Query(fields...) +} + +func (m *SMetadata) MarkDelete() error { + m.Deleted = true + return nil +} + +func (m *SMetadata) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return DeleteModel(ctx, userCred, m) +} + func (manager *SMetadataManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return true } @@ -96,11 +121,10 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy if len(conditions) > 0 { q = q.Filter(sqlchemy.OR(conditions...)) } - if !jsonutils.QueryBoolean(query, "with_sys", false) { - q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), SYSTEM_ADMIN_PREFIX))) - } - if !jsonutils.QueryBoolean(query, "with_cloud", false) { - q = q.Filter(sqlchemy.NOT(sqlchemy.Startswith(q.Field("key"), CLOUD_TAG_PREFIX))) + for args, prefix := range map[string]string{"sys_meta": SYSTEM_ADMIN_PREFIX, "cloud_meta": CLOUD_TAG_PREFIX, "user_meta": USER_TAG_PREFIX} { + if jsonutils.QueryBoolean(query, args, false) { + q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix)) + } } return q, nil } @@ -158,35 +182,42 @@ func (manager *SMetadataManager) RemoveAll(ctx context.Context, model IModel, us lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) + changes := []sMetadataChange{} records := make([]SMetadata, 0) q := manager.Query().Equals("id", idStr) err := FetchModelObjects(manager, q, &records) if err != nil { return fmt.Errorf("find metadata for %s fail: %s", idStr, err) } - changes := make([]sMetadataChange, 0) for _, rec := range records { - if len(rec.Value) > 0 { - _, err := Update(&rec, func() error { - rec.Value = "" - return nil - }) - if err == nil { - changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value}) - } + if err = rec.Delete(ctx, userCred); err != nil { + log.Errorf("remove metadata %v error: %v", rec, err) + continue } + changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value}) } if len(changes) > 0 { - OpsLog.LogEvent(model, ACT_DEL_METADATA, jsonutils.Marshal(changes), userCred) + OpsLog.LogEvent(model, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred) } return nil } func (manager *SMetadataManager) SetValue(ctx context.Context, obj IModel, key string, value interface{}, userCred mcclient.TokenCredential) error { - return manager.SetAll(ctx, obj, map[string]interface{}{key: value}, userCred) + return manager.SetValuesWithLog(ctx, obj, map[string]interface{}{key: value}, userCred) } -func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error { +func (manager *SMetadataManager) SetValuesWithLog(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) error { + changes, err := manager.SetValues(ctx, obj, store, userCred) + if err != nil { + return err + } + if len(changes) > 0 { + OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred) + } + return nil +} + +func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential) ([]sMetadataChange, error) { idStr := GetObjectIdstr(obj) lockman.LockObject(ctx, obj) @@ -195,7 +226,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m changes := make([]sMetadataChange, 0) for key, value := range store { if strings.HasPrefix(key, SYSTEM_ADMIN_PREFIX) && (userCred == nil || !IsAdminAllowGetSpec(userCred, obj, "metadata")) { - return httperrors.NewForbiddenError("Ordinary users can't set the tags that begin with an underscore") + return nil, httperrors.NewForbiddenError("Ordinary users can't set the tags that begin with an underscore") } valStr := stringutils.Interface2String(value) @@ -204,7 +235,7 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m valStr = "" } record := SMetadata{} - err := manager.Query().Equals("id", idStr).Equals("key", key).First(&record) + err := manager.RawQuery().Equals("id", idStr).Equals("key", key).First(&record) //避免之前设置的tag被删除后再次设置时出现Duplicate entry error if err != nil { if err == sql.ErrNoRows { changes = append(changes, sMetadataChange{Key: key, NValue: valStr}) @@ -213,22 +244,70 @@ func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store m record.Value = valStr err = manager.TableSpec().Insert(&record) if err != nil { - return err + return nil, err } } else { - return err + return nil, err } } else { + deleted := record.Deleted _, err := Update(&record, func() error { + record.Deleted = false record.Value = valStr return nil }) if err != nil { - return err + return nil, err + } + if deleted { + changes = append(changes, sMetadataChange{Key: key, NValue: valStr}) + } else { + if record.Value != valStr { + changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr}) + } } - changes = append(changes, sMetadataChange{Key: key, OValue: record.Value, NValue: valStr}) } } + return changes, nil +} + +func (manager *SMetadataManager) SetAll(ctx context.Context, obj IModel, store map[string]interface{}, userCred mcclient.TokenCredential, delRange string) error { + changes, err := manager.SetValues(ctx, obj, store, userCred) + if err != nil { + return err + } + + idStr := GetObjectIdstr(obj) + + lockman.LockObject(ctx, obj) + defer lockman.ReleaseObject(ctx, obj) + + keys := []string{} + for key := range store { + keys = append(keys, key) + } + + records := []SMetadata{} + q := manager.Query().Equals("id", idStr) + switch delRange { + case TAG_DELETE_RANGE_USER: + q = q.Like("key", USER_TAG_PREFIX+"%") + case TAG_DELETE_RANGE_CLOUD: + q = q.Like("key", CLOUD_TAG_PREFIX+"%") + case TAG_DELETE_RANGE_SYS: + q = q.Like("key", SYSTEM_ADMIN_PREFIX+"%") + } + q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys))) + if err := FetchModelObjects(manager, q, &records); err != nil { + log.Errorf("failed to fetch metadata error: %v", err) + } + for _, rec := range records { + if err := rec.Delete(ctx, userCred); err != nil { + log.Errorf("failed to delete metadata error: %v", err) + continue + } + changes = append(changes, sMetadataChange{Key: rec.Key, OValue: rec.Value}) + } if len(changes) > 0 { OpsLog.LogEvent(obj, ACT_SET_METADATA, jsonutils.Marshal(changes), userCred) } diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index 88702015c3..bca40d119d 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "strings" - "unicode" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -227,7 +226,15 @@ func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictst return httperrors.NewNotSufficientPrivilegeError("not allow to set system key %s", k) } } - return Metadata.SetAll(ctx, model, dictstore, userCred) + return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred) +} + +func (model *SStandaloneResourceBase) SetUserMetadataValues(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { + return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred) +} + +func (model *SStandaloneResourceBase) SetUserMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { + return Metadata.SetAll(ctx, model, dictstore, userCred, "user") } func (model *SStandaloneResourceBase) RemoveMetadata(ctx context.Context, key string, userCred mcclient.TokenCredential) error { @@ -270,22 +277,52 @@ func (model *SStandaloneResourceBase) PerformMetadata(ctx context.Context, userC } dictStore := make(map[string]interface{}) for k, v := range dictMap { - if err := model.ValidateMetadataKey(k); err != nil { - return nil, err - } dictStore[k], _ = v.GetString() } err = model.SetAllMetadata(ctx, dictStore, userCred) return nil, err } -func (model *SStandaloneResourceBase) ValidateMetadataKey(key string) error { - for _, k := range []rune(key) { - if k != rune('_') && !unicode.IsLetter(k) && !unicode.IsDigit(k) { - return httperrors.NewInputParameterError(`Not support tag key with %s`, string(k)) - } +func (model *SStandaloneResourceBase) AllowPerformUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return IsAdminAllowPerform(userCred, model, "user-metadata") +} + +func (model *SStandaloneResourceBase) PerformUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + dict, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil, httperrors.NewInputParameterError("input data not key value dict") } - return nil + dictMap, err := dict.GetMap() + if err != nil { + return nil, err + } + dictStore := make(map[string]interface{}) + for k, v := range dictMap { + dictStore["user:"+k], _ = v.GetString() + } + err = model.SetUserMetadataValues(ctx, dictStore, userCred) + return nil, err +} + +func (model *SStandaloneResourceBase) AllowPerformSetUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return IsAdminAllowPerform(userCred, model, "set-user-metadata") +} + +func (model *SStandaloneResourceBase) PerformSetUserMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + dict, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil, httperrors.NewInputParameterError("input data not key value dict") + } + dictMap, err := dict.GetMap() + if err != nil { + return nil, err + } + dictStore := make(map[string]interface{}) + for k, v := range dictMap { + dictStore["user:"+k], _ = v.GetString() + } + err = model.SetUserMetadataAll(ctx, dictStore, userCred) + return nil, err } func (model *SStandaloneResourceBase) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index a7f501d75e..6cad743d4f 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -236,18 +236,7 @@ func (self *SManagedVirtualizationHostDriver) RequestAllocateDiskOnStorage(ctx c return nil, err } - if metaData := iDisk.GetMetadata(); metaData != nil { - meta := make(map[string]string) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) - } else { - for key, value := range meta { - if err := disk.SetMetadata(ctx, "ext:"+key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - } + models.SyncMetadata(ctx, task.GetUserCred(), disk, iDisk) data := jsonutils.NewDict() data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") diff --git a/pkg/compute/models/syncutils.go b/pkg/compute/models/syncutils.go index 155fed2e73..04fba8db97 100644 --- a/pkg/compute/models/syncutils.go +++ b/pkg/compute/models/syncutils.go @@ -38,3 +38,7 @@ func syncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model } return nil } + +func SyncMetadata(ctx context.Context, userCred mcclient.TokenCredential, model IMetadataSetter, remote cloudprovider.ICloudResource) error { + return syncMetadata(ctx, userCred, model, remote) +} diff --git a/pkg/mcclient/options/metadata.go b/pkg/mcclient/options/metadata.go index 24d0f70cf6..1116dbb0f3 100644 --- a/pkg/mcclient/options/metadata.go +++ b/pkg/mcclient/options/metadata.go @@ -2,6 +2,8 @@ package options type MetadataListOptions struct { Resources []string `help:"list of resource e.g server、disk、eip、snapshot, empty will show all metadata"` - WithSys *bool `help:"With sys metadata"` - WithCloud *bool `help:"With cloud metadata"` + SysMeta *bool `help:"Show sys metadata only"` + CloudMeta *bool `help:"Show cloud metadata olny"` + UserMeta *bool `help:"Show user metadata olny"` + Admin *bool `help:"Show all metadata"` } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 362fcf9990..afebec0138 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -350,3 +350,26 @@ type ServerLiveMigrateOptions struct { ID string `help:"ID of server" json:"-"` PreferHost string `help:"Server migration prefer host id or name" json:"prefer_host"` } + +type ServerMetadataOptions struct { + ID string `help:"ID or name of server" json:"-"` + TAGS []string `help:"Tags info, eg: hypervisor=aliyun、os_type=Linux、os_version"` +} + +func (opts *ServerMetadataOptions) Params() (*jsonutils.JSONDict, error) { + params := jsonutils.NewDict() + for _, tag := range opts.TAGS { + info := strings.Split(tag, "=") + if len(info) == 2 { + if len(info[0]) == 0 { + return nil, fmt.Errorf("invalidate tag info %s", tag) + } + params.Add(jsonutils.NewString(info[1]), info[0]) + } else if len(info) == 1 { + params.Add(jsonutils.NewString(info[0]), info[0]) + } else { + return nil, fmt.Errorf("invalidate tag info %s", tag) + } + } + return params, nil +} From a92f781a2e79e751ed6b6810f3611bac762dd5b4 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 19 Mar 2019 19:44:08 +0800 Subject: [PATCH 18/26] fix: baremetal tftp use local IP send response --- pkg/baremetal/pxe/pxe.go | 2 -- pkg/baremetal/pxe/tftp.go | 13 +++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/baremetal/pxe/pxe.go b/pkg/baremetal/pxe/pxe.go index cf126af2b7..8fb1a26529 100644 --- a/pkg/baremetal/pxe/pxe.go +++ b/pkg/baremetal/pxe/pxe.go @@ -113,8 +113,6 @@ func (s *Server) Serve() error { if err != nil { return err } - //tftpSrv := tftp.NewServer(tftpHandler.ReadHandler, nil) - //tftpSrv.SetTimeout(5 * time.Second) log.Infof("DHCPServer Bind %s %d", s.Address, s.DHCPPort) dhcpSrv, _, err := dhcp.NewDHCPServer2(s.Address, s.DHCPPort, false) diff --git a/pkg/baremetal/pxe/tftp.go b/pkg/baremetal/pxe/tftp.go index 894ef682ba..02b637e016 100644 --- a/pkg/baremetal/pxe/tftp.go +++ b/pkg/baremetal/pxe/tftp.go @@ -106,10 +106,23 @@ func (h *TFTPHandler) transferLog(clientAddr net.Addr, path string, err error) { } func (s *Server) serveTFTP(l net.PacketConn, handler *TFTPHandler) error { + bindDial := func(network, addr string) (net.Conn, error) { + localIp := l.LocalAddr().(*net.UDPAddr).IP + remoteAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return nil, err + } + return net.DialUDP("udp", &net.UDPAddr{ + IP: localIp, + Port: 0, // random free port + }, remoteAddr) + } + ts := tftp.Server{ Handler: handler.Handle, InfoLog: func(msg string) { log.Debugf("TFTP msg: %s", msg) }, TransferLog: handler.transferLog, + Dial: bindDial, } err := ts.Serve(l) if err != nil { From 29245c931c3a7d2bf2796b82d83115576a4787c2 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 19 Mar 2019 18:58:25 +0800 Subject: [PATCH 19/26] fix: turn delete prepaid vm to postpaid, so as to allow deleting the vm --- pkg/compute/models/guests.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0f2f09831c..e6fe7ac901 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1826,6 +1826,18 @@ func (self *SGuest) syncRemoveCloudVM(ctx context.Context, userCred mcclient.Tok lockman.LockObject(ctx, self) defer lockman.ReleaseObject(ctx, self) + if self.BillingType == BILLING_TYPE_PREPAID { + diff, err := db.Update(self, func() error { + self.BillingType = BILLING_TYPE_POSTPAID + self.ExpiredAt = time.Time{} + return nil + }) + if err != nil { + return err + } + db.OpsLog.LogSyncUpdate(self, diff, userCred) + } + return self.SetStatus(userCred, VM_UNKNOWN, "Sync lost") } From a81ee45f4ac9e4211069c31f950c881d7aa98c8b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 19 Mar 2019 21:03:06 +0800 Subject: [PATCH 20/26] region: support cloudregion capabilities --- cmd/climc/shell/cloudregions.go | 9 +++ cmd/climc/shell/isolatedevices.go | 4 + pkg/compute/capabilities/handler.go | 2 +- pkg/compute/models/capabilities.go | 103 +++++++++++++++++-------- pkg/compute/models/cloudregions.go | 36 +++++++++ pkg/compute/models/isolated_devices.go | 14 ++++ pkg/compute/models/zones.go | 4 +- 7 files changed, 136 insertions(+), 36 deletions(-) diff --git a/cmd/climc/shell/cloudregions.go b/cmd/climc/shell/cloudregions.go index 034ba9d8c0..a788a8c20e 100644 --- a/cmd/climc/shell/cloudregions.go +++ b/cmd/climc/shell/cloudregions.go @@ -158,4 +158,13 @@ func init() { printObject(result) return nil }) + + R(&CloudregionShowOptions{}, "cloud-region-capability", "Show region's capacibilities", func(s *mcclient.ClientSession, args *CloudregionShowOptions) error { + result, err := modules.Cloudregions.GetSpecific(s, args.ID, "capability", nil) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/cmd/climc/shell/isolatedevices.go b/cmd/climc/shell/isolatedevices.go index 38951b3e7f..59979a4c46 100644 --- a/cmd/climc/shell/isolatedevices.go +++ b/cmd/climc/shell/isolatedevices.go @@ -14,6 +14,7 @@ func init() { Unused bool `help:"Only show unused devices"` Gpu bool `help:"Only show gpu devices"` Host string `help:"Host ID or Name"` + Region string `help:"Cloudregion ID or Name"` Zone string `help:"Zone ID or Name"` } R(&DeviceListOptions{}, "isolated-device-list", "List isolated devices like GPU", func(s *mcclient.ClientSession, args *DeviceListOptions) error { @@ -35,6 +36,9 @@ func init() { if args.Gpu { params.Add(jsonutils.JSONTrue, "gpu") } + if len(args.Region) > 0 { + params.Add(jsonutils.NewString(args.Region), "region") + } if args.Zone != "" { params.Add(jsonutils.NewString(args.Zone), "zone") } diff --git a/pkg/compute/capabilities/handler.go b/pkg/compute/capabilities/handler.go index 8df5482467..0772fc5a50 100644 --- a/pkg/compute/capabilities/handler.go +++ b/pkg/compute/capabilities/handler.go @@ -25,7 +25,7 @@ func capaHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { httperrors.GeneralServerError(w, err) return } - capa, err := models.GetCapabilities(ctx, userCred, query, nil) + capa, err := models.GetCapabilities(ctx, userCred, query, nil, nil) if err != nil { httperrors.GeneralServerError(w, err) return diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index e95a471649..441dbeed47 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -25,23 +25,26 @@ type SCapabilities struct { Specs jsonutils.JSONObject } -func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, zone *SZone) (SCapabilities, error) { +func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, region *SCloudregion, zone *SZone) (SCapabilities, error) { capa := SCapabilities{} - capa.Hypervisors = getHypervisors(zone) - capa.ResourceTypes = getResourceTypes(zone) - capa.StorageTypes = getStorageTypes(zone, true) - capa.DataStorageTypes = getStorageTypes(zone, false) - capa.GPUModels = getGPUs(zone) - capa.SchedPolicySupport = isSchedPolicySupported(zone) - capa.MinNicCount = getMinNicCount(zone) - capa.MaxNicCount = getMaxNicCount(zone) - capa.MinDataDiskCount = getMinDataDiskCount(zone) - capa.MaxDataDiskCount = getMaxDataDiskCount(zone) - capa.Usable = isUsable(zone) + capa.Hypervisors = getHypervisors(region, zone) + capa.ResourceTypes = getResourceTypes(region, zone) + capa.StorageTypes = getStorageTypes(region, zone, true) + capa.DataStorageTypes = getStorageTypes(region, zone, false) + capa.GPUModels = getGPUs(region, zone) + capa.SchedPolicySupport = isSchedPolicySupported(region, zone) + capa.MinNicCount = getMinNicCount(region, zone) + capa.MaxNicCount = getMaxNicCount(region, zone) + capa.MinDataDiskCount = getMinDataDiskCount(region, zone) + capa.MaxDataDiskCount = getMaxDataDiskCount(region, zone) + capa.Usable = isUsable(region, zone) if query == nil { query = jsonutils.NewDict() } var err error + if region != nil { + query.(*jsonutils.JSONDict).Add(jsonutils.NewString(region.GetId()), "region") + } if zone != nil { query.(*jsonutils.JSONDict).Add(jsonutils.NewString(zone.GetId()), "zone") } @@ -50,8 +53,16 @@ func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, que return capa, err } -func getHypervisors(zone *SZone) []string { +func getRegionZoneSubq(region *SCloudregion) *sqlchemy.SSubQuery { + return ZoneManager.Query("id").Equals("cloudregion_id", region.GetId()).SubQuery() +} + +func getHypervisors(region *SCloudregion, zone *SZone) []string { q := HostManager.Query("host_type") + if region != nil { + subq := getRegionZoneSubq(region) + q = q.Filter(sqlchemy.In(q.Field("zone_id"), subq)) + } if zone != nil { q = q.Equals("zone_id", zone.Id) } @@ -75,8 +86,12 @@ func getHypervisors(zone *SZone) []string { return hypervisors } -func getResourceTypes(zone *SZone) []string { +func getResourceTypes(region *SCloudregion, zone *SZone) []string { q := HostManager.Query("resource_type") + if region != nil { + subq := getRegionZoneSubq(region) + q = q.Filter(sqlchemy.In(q.Field("zone_id"), subq)) + } if zone != nil { q = q.Equals("zone_id", zone.Id) } @@ -99,7 +114,7 @@ func getResourceTypes(zone *SZone) []string { return resourceTypes } -func getStorageTypes(zone *SZone, isSysDisk bool) []string { +func getStorageTypes(region *SCloudregion, zone *SZone, isSysDisk bool) []string { storages := StorageManager.Query().SubQuery() hostStorages := HoststorageManager.Query().SubQuery() hosts := HostManager.Query().SubQuery() @@ -113,6 +128,10 @@ func getStorageTypes(zone *SZone, isSysDisk bool) []string { hosts.Field("id"), hostStorages.Field("host_id"), )) + if region != nil { + subq := getRegionZoneSubq(region) + q = q.Filter(sqlchemy.In(storages.Field("zone_id"), subq)) + } if zone != nil { q = q.Filter(sqlchemy.Equals(storages.Field("zone_id"), zone.Id)) } @@ -144,11 +163,16 @@ func getStorageTypes(zone *SZone, isSysDisk bool) []string { return storageTypes } -func getGPUs(zone *SZone) []string { +func getGPUs(region *SCloudregion, zone *SZone) []string { devices := IsolatedDeviceManager.Query().SubQuery() hosts := HostManager.Query().SubQuery() q := devices.Query(devices.Field("model")) + if region != nil { + subq := getRegionZoneSubq(region) + q = q.Join(hosts, sqlchemy.Equals(devices.Field("host_id"), hosts.Field("id"))) + q = q.Filter(sqlchemy.In(hosts.Field("zone_id"), subq)) + } if zone != nil { q = q.Join(hosts, sqlchemy.Equals(devices.Field("host_id"), hosts.Field("id"))) q = q.Filter(sqlchemy.Equals(hosts.Field("zone_id"), zone.Id)) @@ -171,12 +195,17 @@ func getGPUs(zone *SZone) []string { return gpus } -func getNetworkCount(zone *SZone) int { +func getNetworkCount(region *SCloudregion, zone *SZone) int { + wires := WireManager.Query().SubQuery() networks := NetworkManager.Query().SubQuery() q := networks.Query() + if region != nil { + subq := getRegionZoneSubq(region) + q = q.Join(wires, sqlchemy.Equals(networks.Field("wire_id"), wires.Field("id"))) + q = q.Filter(sqlchemy.In(wires.Field("zone_id"), subq)) + } if zone != nil { - wires := WireManager.Query().SubQuery() q = q.Join(wires, sqlchemy.Equals(networks.Field("wire_id"), wires.Field("id"))) q = q.Filter(sqlchemy.Equals(wires.Field("zone_id"), zone.Id)) } @@ -185,44 +214,52 @@ func getNetworkCount(zone *SZone) int { return q.Count() } -func isSchedPolicySupported(zone *SZone) bool { +func isSchedPolicySupported(region *SCloudregion, zone *SZone) bool { return true } -func getMinNicCount(zone *SZone) int { +func getMinNicCount(region *SCloudregion, zone *SZone) int { + if region != nil { + return region.getMinNicCount() + } if zone != nil { return zone.getMinNicCount() - } else { - return 0 } + return 0 } -func getMaxNicCount(zone *SZone) int { +func getMaxNicCount(region *SCloudregion, zone *SZone) int { + if region != nil { + return region.getMaxNicCount() + } if zone != nil { return zone.getMaxNicCount() - } else { - return 0 } + return 0 } -func getMinDataDiskCount(zone *SZone) int { +func getMinDataDiskCount(region *SCloudregion, zone *SZone) int { + if region != nil { + return region.getMinDataDiskCount() + } if zone != nil { return zone.getMinDataDiskCount() - } else { - return 0 } + return 0 } -func getMaxDataDiskCount(zone *SZone) int { +func getMaxDataDiskCount(region *SCloudregion, zone *SZone) int { + if region != nil { + return region.getMaxDataDiskCount() + } if zone != nil { return zone.getMaxDataDiskCount() - } else { - return 0 } + return 0 } -func isUsable(zone *SZone) bool { - if getNetworkCount(zone) > 0 { +func isUsable(region *SCloudregion, zone *SZone) bool { + if getNetworkCount(region, zone) > 0 { return true } else { return false diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index b8479df8fe..c30981b700 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -15,6 +15,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -511,3 +512,38 @@ func (self *SCloudregion) ValidateUpdateCondition(ctx context.Context) error { } return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateCondition(ctx) } + +func (self *SCloudregion) AllowGetDetailsCapability(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return true +} + +func (self *SCloudregion) GetDetailsCapability(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + capa, err := GetCapabilities(ctx, userCred, query, self, nil) + if err != nil { + return nil, err + } + return jsonutils.Marshal(&capa), nil +} + +func (self *SCloudregion) getNetworkCount() int { + return getNetworkCount(self, nil) +} + +func (self *SCloudregion) getMinNicCount() int { + return options.Options.MinNicCount +} + +func (self *SCloudregion) getMaxNicCount() int { + if self.isManaged() { + return options.Options.MaxManagedNicCount + } + return options.Options.MaxNormalNicCount +} + +func (self *SCloudregion) getMinDataDiskCount() int { + return options.Options.MinDataDiskCount +} + +func (self *SCloudregion) getMaxDataDiskCount() int { + return options.Options.MaxDataDiskCount +} diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 5120871abb..13221ef09d 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -2,6 +2,7 @@ package models import ( "context" + "database/sql" "fmt" "strings" "time" @@ -139,6 +140,19 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(ctx context.Context, q *sq if jsonutils.QueryBoolean(query, "unused", false) { q = q.IsEmpty("guest_id") } + regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id"}) + if len(regionStr) > 0 { + region, err := CloudregionManager.FetchByIdOrName(nil, regionStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr) + } + return nil, httperrors.NewGeneralError(err) + } + hosts := HostManager.Query().SubQuery() + subq := ZoneManager.Query("id").Equals("cloudregion_id", region.GetId()).SubQuery() + q.Join(hosts, sqlchemy.Equals(q.Field("host_id"), hosts.Field("id"))).Filter(sqlchemy.In(hosts.Field("zone_id"), subq)) + } zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr) diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 746e6a70bd..2bd2772242 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -205,7 +205,7 @@ func (zone *SZone) getStorageCount() int { } func (zone *SZone) getNetworkCount() int { - return getNetworkCount(zone) + return getNetworkCount(nil, zone) } /*def host_count(self, status=None, host_status=None, enabled=None, host_type=None, is_baremetal=None): @@ -727,7 +727,7 @@ func (self *SZone) AllowGetDetailsCapability(ctx context.Context, userCred mccli } func (self *SZone) GetDetailsCapability(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - capa, err := GetCapabilities(ctx, userCred, query, self) + capa, err := GetCapabilities(ctx, userCred, query, nil, self) if err != nil { return nil, err } From ce04b4d4a92b1aba57172f2c7d89d7c649f6f785 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Tue, 19 Mar 2019 21:40:29 +0800 Subject: [PATCH 21/26] servers filter by vpc --- pkg/compute/models/guests.go | 28 ++++++++++++++++++++++++++++ pkg/mcclient/options/servers.go | 1 + 2 files changed, 29 insertions(+) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0f2f09831c..b80e2d875b 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -395,6 +395,23 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ q = q.In("host_id", sq) } + vpcFilter, err := queryDict.GetString("vpc") + if err == nil { + IVpc, err := VpcManager.FetchByIdOrName(userCred, vpcFilter) + if err != nil { + return nil, httperrors.NewResourceNotFoundError("Vpc %s not found", vpcFilter) + } + vpc := IVpc.(*SVpc) + guestnetwork := GuestnetworkManager.Query().SubQuery() + network := NetworkManager.Query().SubQuery() + wire := WireManager.Query().SubQuery() + sq := guestnetwork.Query(guestnetwork.Field("guest_id")).Join(network, + sqlchemy.Equals(guestnetwork.Field("network_id"), network.Field("id"))). + Join(wire, sqlchemy.Equals(network.Field("wire_id"), wire.Field("id"))). + Filter(sqlchemy.Equals(wire.Field("vpc_id"), vpc.Id)).SubQuery() + q = q.In("id", sq) + } + diskFilter, _ := queryDict.GetString("disk") if len(diskFilter) > 0 { diskI, _ := DiskManager.FetchByIdOrName(userCred, diskFilter) @@ -1366,6 +1383,17 @@ func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.JSONTrue, "can_recycle") } + guestnetworks, _ := self.GetNetworks("") + if len(guestnetworks) > 0 { + guestnetwork := guestnetworks[0] + network := guestnetwork.GetNetwork() + if network != nil { + vpc := network.GetVpc() + extra.Set("vpc_id", jsonutils.NewString(vpc.Id)) + extra.Set("vpc", jsonutils.NewString(vpc.Name)) + } + } + return extra } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 179304db20..97ac175e31 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -24,6 +24,7 @@ type ServerListOptions struct { WithoutEip *bool `help:"Show Servers without EIP"` OsType string `help:"OS Type" choices:"linux|windows|vmware"` OrderByDisk string `help:"Order by disk" choices:"asc|desc"` + Vpc string `help:"Vpc id or name"` ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` From 99a10db949d80155dba0b830498f2bcef2774e37 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 19 Mar 2019 22:14:01 +0800 Subject: [PATCH 22/26] fix: separate private cloud & on-premise --- pkg/cloudprovider/cloudprovider.go | 2 +- pkg/compute/models/cloudaccounts.go | 2 +- pkg/compute/models/cloudproviders.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 63bbd2f482..5f3503b4bd 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -140,7 +140,7 @@ func GetPublicProviders() []string { func GetPrivateProviders() []string { providers := make([]string, 0) for p, d := range providerTable { - if !d.IsPublicCloud() { + if !d.IsPublicCloud() && !d.IsOnPremise() { providers = append(providers, p) } } diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index d3da154f1a..f54e29b23d 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -889,7 +889,7 @@ func (manager *SCloudaccountManager) ListItemFilter(ctx context.Context, q *sqlc } if jsonutils.QueryBoolean(query, "private_cloud", false) { - q = q.IsFalse("is_public_cloud") + q = q.IsFalse("is_public_cloud").IsFalse("is_on_premise") } if jsonutils.QueryBoolean(query, "is_on_premise", false) { diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 02a02245b1..c7653e1578 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -840,6 +840,7 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql cloudaccounts := CloudaccountManager.Query().SubQuery() q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id"))) q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_public_cloud"))) + q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_on_premise"))) } if jsonutils.QueryBoolean(query, "is_on_premise", false) { From 11776d2781b8ea1500e1db4aec9f32f8a2df9e90 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 20 Mar 2019 11:59:20 +0800 Subject: [PATCH 23/26] filter by not in secgroup --- pkg/compute/models/guests.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index b80e2d875b..579453d8e2 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -344,16 +344,28 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ secgrpFilter, _ := queryDict.GetString("secgroup") if len(secgrpFilter) > 0 { + var notIn = false + // HACK FOR NOT IN SECGROUP + if strings.HasPrefix(secgrpFilter, "!") { + secgrpFilter = secgrpFilter[1:] + notIn = true + } secgrp, _ := SecurityGroupManager.FetchByIdOrName(nil, secgrpFilter) if secgrp == nil { return nil, httperrors.NewResourceNotFoundError("secgroup %s not found", secgrpFilter) } - q = q.Filter( - sqlchemy.OR( - sqlchemy.In(q.Field("id"), GuestsecgroupManager.Query("guest_id").Equals("secgroup_id", secgrp.GetId()).SubQuery()), - sqlchemy.Equals(q.Field("secgrp_id"), secgrp.GetId()), - ), - ) + + if notIn { + filter1 := sqlchemy.NotIn(q.Field("id"), + GuestsecgroupManager.Query("guest_id").Equals("secgroup_id", secgrp.GetId()).SubQuery()) + filter2 := sqlchemy.NotEquals(q.Field("secgrp_id"), secgrp.GetId()) + q = q.Filter(sqlchemy.AND(filter1, filter2)) + } else { + filter1 := sqlchemy.In(q.Field("id"), + GuestsecgroupManager.Query("guest_id").Equals("secgroup_id", secgrp.GetId()).SubQuery()) + filter2 := sqlchemy.Equals(q.Field("secgrp_id"), secgrp.GetId()) + q = q.Filter(sqlchemy.OR(filter1, filter2)) + } } zoneFilter, _ := queryDict.GetString("zone") From 19c341c297e71010f3f09c216525cb14917848af Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 9 Mar 2019 18:35:18 +0800 Subject: [PATCH 24/26] scheduler: use compute models --- cmd/climc/shell/schedtaghosts.go | 123 +++--- cmd/climc/shell/schedtags.go | 12 +- cmd/climc/shell/schedulers.go | 4 + cmd/scheduler/app/doc.go | 1 - cmd/scheduler/app/server.go | 107 ----- cmd/scheduler/main.go | 4 +- cmd/scheduler/options/doc.go | 1 - pkg/compute/models/disks.go | 2 + pkg/compute/models/dynamicschedtags.go | 8 + pkg/compute/models/hosts.go | 18 +- pkg/compute/models/hostschedtags.go | 56 +-- pkg/compute/models/initdb.go | 1 + pkg/compute/models/networks.go | 10 +- pkg/compute/models/schedtagjoint.go | 143 +++++++ pkg/compute/models/schedtags.go | 135 ++++++- pkg/compute/models/storages.go | 4 + pkg/compute/models/storageschedtags.go | 66 +++ pkg/compute/service/handlers.go | 1 + pkg/mcclient/modules/mod_schedtaghosts.go | 22 +- pkg/mcclient/modules/mod_schedtags.go | 2 +- pkg/mcclient/modules/modules.go | 1 + pkg/mcclient/modules/resource.go | 4 + pkg/mcclient/options/schedtags.go | 23 ++ pkg/mcclient/options/schedulers.go | 7 +- .../predicates/aggregate_predicate.go | 245 ++--------- .../algorithm/predicates/baremetal/base.go | 2 +- .../predicates/baremetal/network_predicate.go | 18 +- .../predicates/disk_schedtag_predicate.go | 168 ++++++++ .../predicates/guest/hypervisor_predicate.go | 2 +- .../predicates/guest/network_predicate.go | 47 +-- .../predicates/guest/storage_predicate.go | 11 +- .../predicates/k8s/network_predicate.go | 28 +- .../algorithm/predicates/schedtag_helper.go | 263 ++++++++++++ .../priorities/guest/avoid_same_cluster.go | 55 --- pkg/scheduler/algorithmprovider/defaults.go | 1 + pkg/scheduler/api/candidate.go | 61 +-- pkg/scheduler/api/sched.go | 61 +-- pkg/scheduler/api/types.go | 7 +- pkg/scheduler/cache/candidate/baremetals.go | 25 +- pkg/scheduler/cache/candidate/base.go | 246 ++++++++++++ pkg/scheduler/cache/candidate/common.go | 50 +++ pkg/scheduler/cache/candidate/default.go | 9 +- pkg/scheduler/cache/candidate/desc.go | 260 ------------ pkg/scheduler/cache/candidate/hosts.go | 379 ++++++++---------- pkg/scheduler/cache/candidate/interface.go | 2 +- pkg/scheduler/cache/db/default.go | 2 +- pkg/scheduler/cache/sync/default.go | 2 +- pkg/scheduler/core/context.go | 12 + pkg/scheduler/core/generic_scheduler.go | 19 +- pkg/scheduler/core/types.go | 32 +- .../data_manager/candidate_manager.go | 42 +- pkg/scheduler/db/models/host.go | 4 +- pkg/scheduler/db/models/storage.go | 2 +- pkg/scheduler/handler/handler.go | 20 +- pkg/scheduler/handler/helper.go | 27 ++ pkg/scheduler/handler/ping.go | 2 +- pkg/scheduler/manager/completed_queue.go | 2 +- pkg/scheduler/manager/expire_queue.go | 2 +- pkg/scheduler/manager/manager.go | 20 +- pkg/scheduler/manager/scheduler.go | 1 - pkg/scheduler/manager/task_history.go | 2 +- pkg/scheduler/manager/task_queue.go | 2 +- {cmd => pkg}/scheduler/options/options.go | 111 +---- pkg/scheduler/service/service.go | 95 +++++ vendor/yunion.io/x/pkg/utils/misc.go | 26 +- 65 files changed, 1849 insertions(+), 1271 deletions(-) delete mode 100644 cmd/scheduler/app/doc.go delete mode 100644 cmd/scheduler/app/server.go delete mode 100644 cmd/scheduler/options/doc.go create mode 100644 pkg/compute/models/schedtagjoint.go create mode 100644 pkg/compute/models/storageschedtags.go create mode 100644 pkg/mcclient/options/schedtags.go create mode 100644 pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go create mode 100644 pkg/scheduler/algorithm/predicates/schedtag_helper.go delete mode 100644 pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go create mode 100644 pkg/scheduler/cache/candidate/base.go create mode 100644 pkg/scheduler/cache/candidate/common.go delete mode 100644 pkg/scheduler/cache/candidate/desc.go create mode 100644 pkg/scheduler/handler/helper.go rename {cmd => pkg}/scheduler/options/options.go (59%) create mode 100644 pkg/scheduler/service/service.go diff --git a/cmd/climc/shell/schedtaghosts.go b/cmd/climc/shell/schedtaghosts.go index 8973621b84..b68b5144e6 100644 --- a/cmd/climc/shell/schedtaghosts.go +++ b/cmd/climc/shell/schedtaghosts.go @@ -1,6 +1,8 @@ package shell import ( + "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" @@ -8,57 +10,84 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/options" ) -func init() { - type SchedtagHostListOptions struct { - options.BaseListOptions - Schedtag string `help:"ID or Name of schedtag"` +type schedtagModelHelper struct { + managers []modules.JointResourceManager +} + +func newSchedtagModelHelper(mans ...modules.JointResourceManager) *schedtagModelHelper { + return &schedtagModelHelper{managers: mans} +} + +func (h *schedtagModelHelper) register() { + for _, man := range h.managers { + h.list(man.Slave, man.Slave.GetKeyword()) + h.add(man, man.Slave.GetKeyword()) + h.remove(man, man.Slave.GetKeyword()) } - R(&SchedtagHostListOptions{}, "schedtag-host-list", "List all scheduler tag and host pairs", func(s *mcclient.ClientSession, args *SchedtagHostListOptions) error { - mod, err := modules.GetJointModule2(s, &modules.Schedtags, &modules.Hosts) - if err != nil { - return err - } - var params *jsonutils.JSONDict - { - var err error - params, err = args.BaseListOptions.Params() +} + +func (h *schedtagModelHelper) list(slave modules.Manager, kw string) { + R( + &options.SchedtagModelListOptions{}, + fmt.Sprintf("schedtag-%s-list", kw), + fmt.Sprintf("List all scheduler tag and %s pairs", kw), + func(s *mcclient.ClientSession, args *options.SchedtagModelListOptions) error { + mod, err := modules.GetJointModule2(s, &modules.Schedtags, slave) if err != nil { return err - } - } - var result *modules.ListResult - if len(args.Schedtag) > 0 { - result, err = mod.ListDescendent(s, args.Schedtag, params) - } else { - result, err = mod.List(s, params) - } - if err != nil { - return err - } - printList(result, mod.GetColumns(s)) - return nil - }) + params, err := args.Params() + if err != nil { + return err + } + var result *modules.ListResult + if len(args.Schedtag) > 0 { + result, err = mod.ListDescendent(s, args.Schedtag, params) + } else { + result, err = mod.List(s, params) + } + if err != nil { + return err + } + printList(result, mod.GetColumns(s)) + return nil + }, + ) +} - type SchedtagHostPair struct { - SCHEDTAG string `help:"Scheduler tag"` - HOST string `help:"Host"` - } - R(&SchedtagHostPair{}, "schedtag-host-add", "Add a schedtag to a host", func(s *mcclient.ClientSession, args *SchedtagHostPair) error { - schedtag, err := modules.Schedtaghosts.Attach(s, args.SCHEDTAG, args.HOST, nil) - if err != nil { - return err - } - printObject(schedtag) - return nil - }) +func (h *schedtagModelHelper) add(man modules.JointResourceManager, kw string) { + R( + &options.SchedtagModelPairOptions{}, + fmt.Sprintf("schedtag-%s-add", kw), + fmt.Sprintf("Add a schedtag to a %s", kw), + func(s *mcclient.ClientSession, args *options.SchedtagModelPairOptions) error { + schedtag, err := man.Attach(s, args.SCHEDTAG, args.OBJECT, nil) + if err != nil { + return err + } + printObject(schedtag) + return nil + }) +} - R(&SchedtagHostPair{}, "schedtag-host-remove", "Remove a schedtag from a host", func(s *mcclient.ClientSession, args *SchedtagHostPair) error { - schedtag, err := modules.Schedtaghosts.Detach(s, args.SCHEDTAG, args.HOST, nil) - if err != nil { - return err - } - printObject(schedtag) - return nil - }) +func (h *schedtagModelHelper) remove(man modules.JointResourceManager, kw string) { + R( + &options.SchedtagModelPairOptions{}, + fmt.Sprintf("schedtag-%s-remove", kw), + fmt.Sprintf("Remove a schedtag to a %s", kw), + func(s *mcclient.ClientSession, args *options.SchedtagModelPairOptions) error { + schedtag, err := man.Detach(s, args.SCHEDTAG, args.OBJECT, nil) + if err != nil { + return err + } + printObject(schedtag) + return nil + }) +} + +func init() { + newSchedtagModelHelper( + modules.Schedtaghosts, + modules.Schedtagstorages, + ).register() } diff --git a/cmd/climc/shell/schedtags.go b/cmd/climc/shell/schedtags.go index f4fa28a9e2..ee688dca95 100644 --- a/cmd/climc/shell/schedtags.go +++ b/cmd/climc/shell/schedtags.go @@ -13,17 +13,21 @@ import ( func init() { type SchedtagListOptions struct { options.BaseListOptions + Type string `help:"Filter by resource type"` } - R(&SchedtagListOptions{}, "schedtag-list", "List schedule tags", func(s *mcclient.ClientSession, suboptions *SchedtagListOptions) error { + R(&SchedtagListOptions{}, "schedtag-list", "List schedule tags", func(s *mcclient.ClientSession, args *SchedtagListOptions) error { var params *jsonutils.JSONDict { var err error - params, err = suboptions.BaseListOptions.Params() + params, err = args.BaseListOptions.Params() if err != nil { return err } } + if len(args.Type) > 0 { + params.Add(jsonutils.NewString(args.Type), "resource_type") + } result, err := modules.Schedtags.List(s, params) if err != nil { return err @@ -57,6 +61,7 @@ func init() { NAME string `help:"Name of new schedtag"` Strategy string `help:"Policy" choices:"require|exclude|prefer|avoid"` Desc string `help:"Description"` + Type string `help:"Resource type" choices:"hosts|storages"` } R(&SchedtagCreateOptions{}, "schedtag-create", "Create a schedule tag", func(s *mcclient.ClientSession, args *SchedtagCreateOptions) error { params := jsonutils.NewDict() @@ -67,6 +72,9 @@ func init() { if len(args.Desc) > 0 { params.Add(jsonutils.NewString(args.Desc), "description") } + if len(args.Type) > 0 { + params.Add(jsonutils.NewString(args.Type), "resource_type") + } schedtag, err := modules.Schedtags.Create(s, params) if err != nil { return err diff --git a/cmd/climc/shell/schedulers.go b/cmd/climc/shell/schedulers.go index d9433bc757..8c3c47b808 100644 --- a/cmd/climc/shell/schedulers.go +++ b/cmd/climc/shell/schedulers.go @@ -45,6 +45,7 @@ func init() { type SchedulerCandidateListOptions struct { Type string `help:"Sched type filter" choices:"baremetal|host"` + Region string `help:"Cloud region ID"` Zone string `help:"Zone ID"` Limit int `default:"50" help:"Page limit"` Offset int `default:"0" help:"Page offset"` @@ -58,6 +59,9 @@ func init() { if args.Offset > 0 { params.Add(jsonutils.NewInt(int64(args.Offset)), "offset") } + if len(args.Region) > 0 { + params.Add(jsonutils.NewString(args.Region), "region") + } if len(args.Zone) > 0 { params.Add(jsonutils.NewString(args.Zone), "zone") } diff --git a/cmd/scheduler/app/doc.go b/cmd/scheduler/app/doc.go deleted file mode 100644 index 0a42c091c2..0000000000 --- a/cmd/scheduler/app/doc.go +++ /dev/null @@ -1 +0,0 @@ -package app // import "yunion.io/x/onecloud/cmd/scheduler/app" diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go deleted file mode 100644 index 3eaf29de64..0000000000 --- a/cmd/scheduler/app/server.go +++ /dev/null @@ -1,107 +0,0 @@ -// Package app implements a Server object for running the scheduler. -package app - -import ( - "fmt" - "io/ioutil" - "net" - "net/http" - "strconv" - - gin "gopkg.in/gin-gonic/gin.v1" - - "yunion.io/x/log" - "yunion.io/x/pkg/util/prometheus" - "yunion.io/x/pkg/utils" - - o "yunion.io/x/onecloud/cmd/scheduler/options" - "yunion.io/x/onecloud/pkg/mcclient/auth" - _ "yunion.io/x/onecloud/pkg/scheduler/algorithmprovider" - "yunion.io/x/onecloud/pkg/scheduler/db/models" - schedhandler "yunion.io/x/onecloud/pkg/scheduler/handler" - schedman "yunion.io/x/onecloud/pkg/scheduler/manager" - "yunion.io/x/onecloud/pkg/util/gin/middleware" -) - -type SchedulerServer struct { - Address string - Port int32 - SQLConn string - DBDialect string - AuthInfo *auth.AuthInfo -} - -func NewServerFromConfig() *SchedulerServer { - authURL := o.GetOptions().AuthURL - adminUser := o.GetOptions().AdminUser - adminPasswd := o.GetOptions().AdminPasswd - adminTenant := o.GetOptions().AdminTenant - a := auth.NewV2AuthInfo(authURL, adminUser, adminPasswd, adminTenant) - - s := &SchedulerServer{} - s.Address = o.GetOptions().Address - s.Port = int32(o.GetOptions().Port) - sqlDialect, sqlConn, err := utils.TransSQLAchemyURL(o.GetOptions().SqlConnection) - if err != nil { - log.Fatalf("Backup python sql_connection config err: %v", err) - } - s.SQLConn = sqlConn - s.DBDialect = sqlDialect - s.AuthInfo = a - - return s -} - -func Run(s *SchedulerServer) error { - startSched := func() { - err := models.Init(s.DBDialect, s.SQLConn) - if err != nil { - log.Fatalf("DB init error: %v, dialect: %s, url: %s", err, s.DBDialect, s.SQLConn) - } - - stopEverything := make(chan struct{}) - schedman.InitAndStart(stopEverything) - } - - debug := o.GetOptions().LogLevel == "debug" - - auth.AsyncInit(s.AuthInfo, debug, true, "", "", startSched) - - return startHTTP(s) -} - -func startHTTP(s *SchedulerServer) error { - gin.DefaultWriter = ioutil.Discard - - router := gin.Default() - router.Use(middleware.Logger()) - router.Use(middleware.ErrorHandler) - router.Use(middleware.KeystoneTokenVerifyMiddleware()) - - prometheus.InstallHandler(router) - schedhandler.InstallHandler(router) - - server := &http.Server{ - Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), - Handler: router, - } - - log.Infof("Start server on: %s:%d", s.Address, s.Port) - - if o.GetOptions().EnableSsl { - return server.ListenAndServeTLS(o.GetOptions().SslCertfile, - o.GetOptions().SslKeyfile) - } else { - return server.ListenAndServe() - } -} - -func Execute() error { - o.Parse() - - err := Run(NewServerFromConfig()) - if err != nil { - err = fmt.Errorf("scheduler app failed to run: %v", err) - } - return err -} diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index 383a6d328b..cf2e23ffc1 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -5,11 +5,11 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/cmd/scheduler/app" + "yunion.io/x/onecloud/pkg/scheduler/service" ) func main() { - if err := app.Execute(); err != nil { + if err := service.StartService(); err != nil { log.Errorln(err) os.Exit(-1) } diff --git a/cmd/scheduler/options/doc.go b/cmd/scheduler/options/doc.go deleted file mode 100644 index 75aded7ae6..0000000000 --- a/cmd/scheduler/options/doc.go +++ /dev/null @@ -1 +0,0 @@ -package options // import "yunion.io/x/onecloud/cmd/scheduler/options" diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 452835442d..2b757d7f6f 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -1159,6 +1159,8 @@ type SDiskConfig struct { Medium string `json:"medium"` ImageProperties map[string]string `json:"image_properties"` + Schedtags map[string]string `json:"schedtags"` + DiskId string `json:"-"` // import only } diff --git a/pkg/compute/models/dynamicschedtags.go b/pkg/compute/models/dynamicschedtags.go index 92922ff48b..a2c0059c3f 100644 --- a/pkg/compute/models/dynamicschedtags.go +++ b/pkg/compute/models/dynamicschedtags.go @@ -109,6 +109,10 @@ func (self *SDynamicschedtag) ValidateUpdateData(ctx context.Context, userCred m return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } +func (self *SDynamicschedtag) GetSchedtag() *SSchedtag { + return self.getSchedtag() +} + func (self *SDynamicschedtag) getSchedtag() *SSchedtag { obj, err := SchedtagManager.FetchById(self.SchedtagId) if err != nil { @@ -139,6 +143,10 @@ func (self *SDynamicschedtag) GetExtraDetails(ctx context.Context, userCred mccl return self.getMoreColumns(extra), nil } +func (manager *SDynamicschedtagManager) GetAllEnabledDynamicSchedtags() []SDynamicschedtag { + return manager.getAllEnabledDynamicSchedtags() +} + func (manager *SDynamicschedtagManager) getAllEnabledDynamicSchedtags() []SDynamicschedtag { rules := make([]SDynamicschedtag, 0) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index d547875b48..10bd0a64e5 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -2078,20 +2078,8 @@ func (self *SHost) GetBaremetalServer() *SGuest { return &guest } -func (self *SHost) getSchedtags() []SSchedtag { - tags := make([]SSchedtag, 0) - schedtags := SchedtagManager.Query().SubQuery() - hostschedtags := HostschedtagManager.Query().SubQuery() - q := schedtags.Query() - q = q.Join(hostschedtags, sqlchemy.AND(sqlchemy.Equals(hostschedtags.Field("schedtag_id"), schedtags.Field("id")), - sqlchemy.IsFalse(hostschedtags.Field("deleted")))) - q = q.Filter(sqlchemy.Equals(hostschedtags.Field("host_id"), self.Id)) - err := db.FetchModelObjects(SchedtagManager, q, &tags) - if err != nil { - log.Errorf("%s", err) - return nil - } - return tags +func (self *SHost) GetSchedtags() []SSchedtag { + return GetSchedtags(HostschedtagManager, self.Id) } type SHostGuestResourceUsage struct { @@ -2157,7 +2145,7 @@ func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict extra.Add(jsonutils.NewInt(int64(len(nicInfos))), "nic_count") extra.Add(jsonutils.NewArray(nicInfos...), "nic_info") } - schedtags := self.getSchedtags() + schedtags := self.GetSchedtags() if schedtags != nil && len(schedtags) > 0 { info := make([]jsonutils.JSONObject, len(schedtags)) for i := 0; i < len(schedtags); i += 1 { diff --git a/pkg/compute/models/hostschedtags.go b/pkg/compute/models/hostschedtags.go index 71c80e69fc..40fc16e999 100644 --- a/pkg/compute/models/hostschedtags.go +++ b/pkg/compute/models/hostschedtags.go @@ -4,14 +4,13 @@ import ( "context" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/mcclient" ) type SHostschedtagManager struct { - SHostJointsManager + *SSchedtagJointsManager } var HostschedtagManager *SHostschedtagManager @@ -19,11 +18,12 @@ var HostschedtagManager *SHostschedtagManager func init() { db.InitManager(func() { HostschedtagManager = &SHostschedtagManager{ - SHostJointsManager: NewHostJointsManager( + SSchedtagJointsManager: NewSchedtagJointsManager( SHostschedtag{}, "aggregate_hosts_tbl", "schedtaghost", "schedtaghosts", + HostManager, SchedtagManager, ), } @@ -31,55 +31,37 @@ func init() { } type SHostschedtag struct { - SHostJointsBase + SSchedtagJointsBase - HostId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False) - SchedtagId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // =Column(VARCHAR(36, charset='ascii'), nullable=False) + HostId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False) } -func (joint *SHostschedtag) Master() db.IStandaloneModel { - return db.JointMaster(joint) +func (self *SHostschedtag) GetHost() *SHost { + return self.Master().(*SHost) } -func (joint *SHostschedtag) Slave() db.IStandaloneModel { - return db.JointSlave(joint) +func (self *SHostschedtag) GetHosts() ([]SHost, error) { + hosts := []SHost{} + err := self.GetSchedtag().GetObjects(&hosts) + return hosts, err +} + +func (self *SHostschedtag) Master() db.IStandaloneModel { + return self.SSchedtagJointsBase.master(self) } func (self *SHostschedtag) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { - extra := self.SHostJointsBase.GetCustomizeColumns(ctx, userCred, query) - return db.JointModelExtra(self, extra) + return self.SSchedtagJointsBase.getCustomizeColumns(self, ctx, userCred, query) } func (self *SHostschedtag) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) { - extra, err := self.SHostJointsBase.GetExtraDetails(ctx, userCred, query) - if err != nil { - return nil, err - } - return db.JointModelExtra(self, extra), nil -} - -func (self *SHostschedtag) getHost() *SHost { - obj, err := HostManager.FetchById(self.HostId) - if err != nil { - log.Errorf("%s", err) - return nil - } - return obj.(*SHost) -} - -func (self *SHostschedtag) getSchedtag() *SSchedtag { - obj, err := SchedtagManager.FetchById(self.SchedtagId) - if err != nil { - log.Errorf("%s", err) - return nil - } - return obj.(*SSchedtag) + return self.SSchedtagJointsBase.getExtraDetails(self, ctx, userCred, query) } func (self *SHostschedtag) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { - return db.DeleteModel(ctx, userCred, self) + return self.SSchedtagJointsBase.delete(self, ctx, userCred) } func (self *SHostschedtag) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { - return db.DetachJoint(ctx, userCred, self) + return self.SSchedtagJointsBase.detach(self, ctx, userCred) } diff --git a/pkg/compute/models/initdb.go b/pkg/compute/models/initdb.go index 498d9336df..774489d9cd 100644 --- a/pkg/compute/models/initdb.go +++ b/pkg/compute/models/initdb.go @@ -27,6 +27,7 @@ func InitDB() error { LoadbalancerListenerRuleManager, LoadbalancerBackendGroupManager, LoadbalancerBackendManager, + SchedtagManager, } { err := manager.InitializeData() if err != nil { diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 9e559fc0b5..01c79bd2ed 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -218,6 +218,10 @@ func (self *SNetwork) GetUsedAddresses() map[string]bool { return used } +func (self *SNetwork) GetIPRange() netutils.IPV4AddrRange { + return self.getIPRange() +} + func (self *SNetwork) getIPRange() netutils.IPV4AddrRange { start, _ := netutils.NewIPV4Addr(self.GuestIpStart) end, _ := netutils.NewIPV4Addr(self.GuestIpEnd) @@ -895,6 +899,10 @@ func (self *SNetwork) getRegion() *SCloudregion { return nil } +func (self *SNetwork) GetPorts() int { + return self.getIPRange().AddressCount() +} + func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { wire := self.GetWire() extra.Add(jsonutils.NewString(wire.Name), "wire") @@ -903,7 +911,7 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND } else { extra.Add(jsonutils.JSONFalse, "exit") } - extra.Add(jsonutils.NewInt(int64(self.getIPRange().AddressCount())), "ports") + extra.Add(jsonutils.NewInt(int64(self.GetPorts())), "ports") extra.Add(jsonutils.NewInt(int64(self.GetTotalNicCount())), "ports_used") extra.Add(jsonutils.NewInt(int64(self.GetGuestnicsCount())), "vnics") extra.Add(jsonutils.NewInt(int64(self.GetBaremetalNicsCount())), "bm_vnics") diff --git a/pkg/compute/models/schedtagjoint.go b/pkg/compute/models/schedtagjoint.go new file mode 100644 index 0000000000..09087ad88d --- /dev/null +++ b/pkg/compute/models/schedtagjoint.go @@ -0,0 +1,143 @@ +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SSchedtagJointsManager struct { + db.SJointResourceBaseManager +} + +func NewSchedtagJointsManager( + dt interface{}, + tableName string, + keyword string, + keywordPlural string, + master db.IStandaloneModelManager, + slave db.IStandaloneModelManager, +) *SSchedtagJointsManager { + return &SSchedtagJointsManager{ + SJointResourceBaseManager: db.NewJointResourceBaseManager( + dt, + tableName, + keyword, + keywordPlural, + master, + slave, + ), + } +} + +type SSchedtagJointsBase struct { + db.SJointResourceBase + + SchedtagId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // =Column(VARCHAR(36, charset='ascii'), nullable=False) +} + +func (man *SSchedtagJointsManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowList(userCred, man) +} + +func (man *SSchedtagJointsManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return db.IsAdminAllowCreate(userCred, man) +} + +func (man *SSchedtagJointsManager) FetchSchedtagById(id string) *SSchedtag { + schedtagObj, _ := SchedtagManager.FetchById(id) + if schedtagObj == nil { + return nil + } + return schedtagObj.(*SSchedtag) +} + +func (man *SSchedtagJointsManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + schedtagId, err := data.GetString("schedtag_id") + if err != nil || schedtagId == "" { + return nil, httperrors.NewInputParameterError("schedtag_id not provide") + } + resourceType := man.GetMasterManager().KeywordPlural() + if !utils.IsInStringArray(resourceType, SchedtagManager.GetResourceTypes()) { + return nil, httperrors.NewInputParameterError("Not support resource_type %s", resourceType) + } + schedtag := man.FetchSchedtagById(schedtagId) + if schedtag == nil { + return nil, httperrors.NewNotFoundError("Schedtag %s", schedtagId) + } + if resourceType != schedtag.ResourceType { + return nil, httperrors.NewInputParameterError("Schedtag %s resource_type mismatch: %s != %s", schedtag.GetName(), schedtag.ResourceType, resourceType) + } + return man.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (man *SSchedtagJointsManager) AllowListDescendent(ctx context.Context, userCred mcclient.TokenCredential, model db.IStandaloneModel, query jsonutils.JSONObject) bool { + return db.IsAdminAllowList(userCred, man) +} + +func (man *SSchedtagJointsManager) GetMasterIdKey(m db.IJointModelManager) string { + return fmt.Sprintf("%s_id", m.GetMasterManager().Keyword()) +} + +func (man *SSchedtagJointsManager) AllowAttach(ctx context.Context, userCred mcclient.TokenCredential, master db.IStandaloneModel, slave db.IStandaloneModel) bool { + return db.IsAdminAllowCreate(userCred, man) +} + +func (self *SSchedtagJointsBase) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowGet(userCred, self) +} + +func (self *SSchedtagJointsBase) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool { + return db.IsAdminAllowUpdate(userCred, self) +} + +func (self *SSchedtagJointsBase) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return db.IsAdminAllowDelete(userCred, self) +} + +func (joint *SSchedtagJointsBase) master(obj db.IJointModel) db.IStandaloneModel { + return db.JointMaster(obj) +} + +func (joint *SSchedtagJointsBase) GetSchedtag() *SSchedtag { + return joint.Slave().(*SSchedtag) +} + +func (joint *SSchedtagJointsBase) Slave() db.IStandaloneModel { + return db.JointSlave(joint) +} + +func (joint *SSchedtagJointsBase) getCustomizeColumns(obj db.IJointModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := joint.SJointResourceBase.GetCustomizeColumns(ctx, userCred, query) + return db.JointModelExtra(obj, extra) +} + +func (joint *SSchedtagJointsBase) getExtraDetails(obj db.IJointModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) { + extra, err := joint.SJointResourceBase.GetExtraDetails(ctx, userCred, query) + if err != nil { + return nil, err + } + return db.JointModelExtra(obj, extra), nil +} + +func (joint *SSchedtagJointsBase) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return fmt.Errorf("Delete must be override") +} + +func (joint *SSchedtagJointsBase) delete(obj db.IJointModel, ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, joint) +} + +func (joint *SSchedtagJointsBase) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { + return fmt.Errorf("Detach must be override") +} + +func (joint *SSchedtagJointsBase) detach(obj db.IJointModel, ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DetachJoint(ctx, userCred, joint) +} diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index 8d56e24e4b..a5a3c68024 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -3,6 +3,7 @@ package models import ( "context" "database/sql" + "fmt" "strings" "yunion.io/x/jsonutils" @@ -29,8 +30,15 @@ const ( var STRATEGY_LIST = []string{STRATEGY_REQUIRE, STRATEGY_EXCLUDE, STRATEGY_PREFER, STRATEGY_AVOID} +type ISchedtagJointManager interface { + db.IJointModelManager + GetMasterIdKey(db.IJointModelManager) string +} + type SSchedtagManager struct { db.SStandaloneResourceBaseManager + + jointsManager map[string]ISchedtagJointManager } var SchedtagManager *SSchedtagManager @@ -43,19 +51,64 @@ func init() { "schedtag", "schedtags", ), + jointsManager: make(map[string]ISchedtagJointManager), } } +func (manager *SSchedtagManager) InitializeData() error { + // set old schedtags resource_type to hosts + schedtags := []SSchedtag{} + q := manager.Query().IsNullOrEmpty("resource_type") + err := db.FetchModelObjects(manager, q, &schedtags) + if err != nil { + return err + } + for _, tag := range schedtags { + tmp := &tag + db.Update(tmp, func() error { + tmp.ResourceType = HostManager.KeywordPlural() + return nil + }) + } + manager.BindJointManagers( + HostschedtagManager, + StorageschedtagManager, + ) + return nil +} + +func (manager *SSchedtagManager) BindJointManagers(ms ...ISchedtagJointManager) { + for _, m := range ms { + manager.jointsManager[m.GetMasterManager().KeywordPlural()] = m + } +} + +func (manager *SSchedtagManager) GetResourceTypes() []string { + ret := []string{} + for key := range manager.jointsManager { + ret = append(ret, key) + } + return ret +} + type SSchedtag struct { db.SStandaloneResourceBase DefaultStrategy string `width:"16" charset:"ascii" nullable:"true" default:"" list:"user" update:"admin" create:"admin_optional"` // Column(VARCHAR(16, charset='ascii'), nullable=True, default='') + ResourceType string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"required"` // Column(VARCHAR(16, charset='ascii'), nullable=True, default='') } func (manager *SSchedtagManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return true } +func (manager *SSchedtagManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + if resType := jsonutils.GetAnyString(query, []string{"type", "resource_type"}); resType != "" { + q = q.Equals("resource_type", resType) + } + return manager.SResourceBaseManager.ListItemFilter(ctx, q, userCred, query) +} + func (self *SSchedtag) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return true } @@ -111,9 +164,30 @@ func (manager *SSchedtagManager) ValidateCreateData(ctx context.Context, userCre return nil, err } } + // set resourceType to hosts if not provided by client + resourceType, _ := data.GetString("resource_type") + if resourceType == "" { + resourceType = HostManager.KeywordPlural() + data.Set("resource_type", jsonutils.NewString(resourceType)) + } + if !utils.IsInStringArray(resourceType, manager.GetResourceTypes()) { + return nil, httperrors.NewInputParameterError("Not support resource_type %s", resourceType) + } return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) } +func (manager *SSchedtagManager) GetResourceSchedtags(resType string) ([]SSchedtag, error) { + jointMan := manager.jointsManager[resType] + if jointMan == nil { + return nil, fmt.Errorf("Not found joint manager by resource type: %s", resType) + } + tags := make([]SSchedtag, 0) + if err := manager.Query().Equals("resource_type", resType).All(&tags); err != nil { + return nil, err + } + return tags, nil +} + func (self *SSchedtag) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { defStrategy, _ := data.GetString("default_strategy") if len(defStrategy) > 0 { @@ -126,8 +200,8 @@ func (self *SSchedtag) ValidateUpdateData(ctx context.Context, userCred mcclient } func (self *SSchedtag) ValidateDeleteCondition(ctx context.Context) error { - if self.GetHostCount() > 0 { - return httperrors.NewNotEmptyError("Tag is associated with hosts") + if self.GetObjectCount() > 0 { + return httperrors.NewNotEmptyError("Tag is associated with %s", self.ResourceType) } if self.getDynamicSchedtagCount() > 0 { return httperrors.NewNotEmptyError("tag has dynamic rules") @@ -147,30 +221,35 @@ func (self *SSchedtag) AllowDeleteItem(ctx context.Context, userCred mcclient.To return userCred.IsSystemAdmin() }*/ -func (self *SSchedtag) GetHosts() []SHost { - q := self.GetHostQuery() - hosts := make([]SHost, 0) - err := db.FetchModelObjects(HostManager, q, &hosts) +func (self *SSchedtag) GetObjects(objs interface{}) error { + q := self.GetObjectQuery() + masterMan := self.GetJointManager().GetMasterManager() + err := db.FetchModelObjects(masterMan, q, objs) if err != nil { - log.Errorf("GetHosts query fail %s", err) - return nil + return err } - return hosts + return nil } -func (self *SSchedtag) GetHostQuery() *sqlchemy.SQuery { - hosts := HostManager.Query().SubQuery() - hostschedtags := HostschedtagManager.Query().SubQuery() - q := hosts.Query() - q = q.Join(hostschedtags, sqlchemy.AND(sqlchemy.Equals(hostschedtags.Field("host_id"), hosts.Field("id")), - sqlchemy.IsFalse(hostschedtags.Field("deleted")))) - q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) - q = q.Filter(sqlchemy.Equals(hostschedtags.Field("schedtag_id"), self.Id)) +func (self *SSchedtag) GetObjectQuery() *sqlchemy.SQuery { + jointMan := self.GetJointManager() + masterMan := jointMan.GetMasterManager() + objs := masterMan.Query().SubQuery() + objschedtags := jointMan.Query().SubQuery() + q := objs.Query() + q = q.Join(objschedtags, sqlchemy.AND(sqlchemy.Equals(objschedtags.Field(jointMan.GetMasterIdKey(jointMan)), objs.Field("id")), + sqlchemy.IsFalse(objschedtags.Field("deleted")))) + q = q.Filter(sqlchemy.IsTrue(objs.Field("enabled"))) + q = q.Filter(sqlchemy.Equals(objschedtags.Field("schedtag_id"), self.Id)) return q } -func (self *SSchedtag) GetHostCount() int { - return HostschedtagManager.Query().Equals("schedtag_id", self.Id).Count() +func (self *SSchedtag) GetJointManager() ISchedtagJointManager { + return SchedtagManager.jointsManager[self.ResourceType] +} + +func (self *SSchedtag) GetObjectCount() int { + return self.GetJointManager().Query().Equals("schedtag_id", self.Id).Count() } func (self *SSchedtag) getSchedPoliciesCount() int { @@ -182,7 +261,7 @@ func (self *SSchedtag) getDynamicSchedtagCount() int { } func (self *SSchedtag) getMoreColumns(extra *jsonutils.JSONDict) *jsonutils.JSONDict { - extra.Add(jsonutils.NewInt(int64(self.GetHostCount())), "host_count") + extra.Add(jsonutils.NewInt(int64(self.GetObjectCount())), fmt.Sprintf("%s_count", self.GetJointManager().GetMasterManager().Keyword())) extra.Add(jsonutils.NewInt(int64(self.getDynamicSchedtagCount())), "dynamic_schedtag_count") extra.Add(jsonutils.NewInt(int64(self.getSchedPoliciesCount())), "schedpolicy_count") return extra @@ -210,3 +289,19 @@ func (self *SSchedtag) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { desc.Add(jsonutils.NewString(self.DefaultStrategy), "default") return desc } + +func GetSchedtags(jointMan ISchedtagJointManager, masterId string) []SSchedtag { + tags := make([]SSchedtag, 0) + schedtags := SchedtagManager.Query().SubQuery() + objschedtags := jointMan.Query().SubQuery() + q := schedtags.Query() + q = q.Join(objschedtags, sqlchemy.AND(sqlchemy.Equals(objschedtags.Field("schedtag_id"), schedtags.Field("id")), + sqlchemy.IsFalse(objschedtags.Field("deleted")))) + q = q.Filter(sqlchemy.Equals(objschedtags.Field(jointMan.GetMasterIdKey(jointMan)), masterId)) + err := db.FetchModelObjects(SchedtagManager, q, &tags) + if err != nil { + log.Errorf("GetSchedtags error: %s", err) + return nil + } + return tags +} diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 25ca4ba068..6357910f10 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -1158,3 +1158,7 @@ func (self *SStorage) IsPrepaidRecycleResource() bool { } return hosts[0].IsPrepaidRecycleResource() } + +func (self *SStorage) GetSchedtags() []SSchedtag { + return GetSchedtags(StorageschedtagManager, self.Id) +} diff --git a/pkg/compute/models/storageschedtags.go b/pkg/compute/models/storageschedtags.go new file mode 100644 index 0000000000..e7ccc47aee --- /dev/null +++ b/pkg/compute/models/storageschedtags.go @@ -0,0 +1,66 @@ +package models + +import ( + "context" + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SStorageschedtagManager struct { + *SSchedtagJointsManager +} + +var StorageschedtagManager *SStorageschedtagManager + +func init() { + db.InitManager(func() { + StorageschedtagManager = &SStorageschedtagManager{ + SSchedtagJointsManager: NewSchedtagJointsManager( + SStorageschedtag{}, + "schedtag_storages_tbl", + "schedtagstorage", + "schedtagstorages", + StorageManager, + SchedtagManager, + ), + } + }) +} + +type SStorageschedtag struct { + SSchedtagJointsBase + + StorageId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(36, charset='ascii'), nullable=False) +} + +func (s *SStorageschedtag) GetStorage() *SStorage { + return s.Master().(*SStorage) +} + +func (s *SStorageschedtag) GetStorages() ([]SStorage, error) { + storages := []SStorage{} + err := s.GetSchedtag().GetObjects(&storages) + return storages, err +} + +func (joint *SStorageschedtag) Master() db.IStandaloneModel { + return joint.SSchedtagJointsBase.master(joint) +} + +func (joint *SStorageschedtag) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + return joint.SSchedtagJointsBase.getCustomizeColumns(joint, ctx, userCred, query) +} + +func (joint *SStorageschedtag) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) { + return joint.SSchedtagJointsBase.getExtraDetails(joint, ctx, userCred, query) +} + +func (joint *SStorageschedtag) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return joint.SSchedtagJointsBase.delete(joint, ctx, userCred) +} + +func (joint *SStorageschedtag) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { + return joint.SSchedtagJointsBase.detach(joint, ctx, userCred) +} diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 5f4aee8bb9..52d9c5537c 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -96,6 +96,7 @@ func InitHandlers(app *appsrv.Application) { models.HostnetworkManager, models.HoststorageManager, models.HostschedtagManager, + models.StorageschedtagManager, models.GuestnetworkManager, models.GuestsecgroupManager, models.LoadbalancernetworkManager, diff --git a/pkg/mcclient/modules/mod_schedtaghosts.go b/pkg/mcclient/modules/mod_schedtaghosts.go index 6b745b5654..f22dd756b9 100644 --- a/pkg/mcclient/modules/mod_schedtaghosts.go +++ b/pkg/mcclient/modules/mod_schedtaghosts.go @@ -1,14 +1,28 @@ package modules var ( - Schedtaghosts JointResourceManager + Schedtaghosts JointResourceManager + Schedtagstorages JointResourceManager ) +func newSchedtagJointManager(keyword, keywordPlural string, columns, adminColumns []string, slave Manager) JointResourceManager { + columns = append(columns, "Schedtag_ID", "Schedtag") + return NewJointComputeManager(keyword, keywordPlural, + columns, adminColumns, &Schedtags, slave) +} + func init() { - Schedtaghosts = NewJointComputeManager("schedtaghost", "schedtaghosts", - []string{"Host_ID", "Host", "Schedtag_ID", "Schedtag"}, + Schedtaghosts = newSchedtagJointManager("schedtaghost", "schedtaghosts", + []string{"Host_ID", "Host"}, []string{}, - &Schedtags, &Hosts) + + Schedtagstorages = newSchedtagJointManager("schedtagstorage", "schedtagstorages", + []string{"Storage_ID", "Storage"}, + []string{}, + &Storages) + registerCompute(&Schedtaghosts) + + registerCompute(&Schedtagstorages) } diff --git a/pkg/mcclient/modules/mod_schedtags.go b/pkg/mcclient/modules/mod_schedtags.go index 88f7230069..849c5f4a73 100644 --- a/pkg/mcclient/modules/mod_schedtags.go +++ b/pkg/mcclient/modules/mod_schedtags.go @@ -55,7 +55,7 @@ func (this *SchedtagManager) DoBatchSchedtagHostAddRemove(s *mcclient.ClientSess func init() { Schedtags = SchedtagManager{NewComputeManager("schedtag", "schedtags", - []string{"ID", "Name", "Default_strategy"}, + []string{"ID", "Name", "Default_strategy", "Resource_type"}, []string{})} registerCompute(&Schedtags) diff --git a/pkg/mcclient/modules/modules.go b/pkg/mcclient/modules/modules.go index 7bbb90ec95..a0b252cee3 100644 --- a/pkg/mcclient/modules/modules.go +++ b/pkg/mcclient/modules/modules.go @@ -11,6 +11,7 @@ import ( type BaseManagerInterface interface { Version() string + GetKeyword() string KeyString() string ServiceType() string EndpointType() string diff --git a/pkg/mcclient/modules/resource.go b/pkg/mcclient/modules/resource.go index b092dfc49a..730b2681ef 100644 --- a/pkg/mcclient/modules/resource.go +++ b/pkg/mcclient/modules/resource.go @@ -33,6 +33,10 @@ type ResourceManager struct { idFieldName string } +func (this *ResourceManager) GetKeyword() string { + return this.Keyword +} + func (this *ResourceManager) KeyString() string { return this.KeywordPlural } diff --git a/pkg/mcclient/options/schedtags.go b/pkg/mcclient/options/schedtags.go new file mode 100644 index 0000000000..43ca8e0161 --- /dev/null +++ b/pkg/mcclient/options/schedtags.go @@ -0,0 +1,23 @@ +package options + +import ( + "yunion.io/x/jsonutils" +) + +type SchedtagModelListOptions struct { + BaseListOptions + Schedtag string `help:"ID or Name of schedtag"` +} + +func (o SchedtagModelListOptions) Params() (*jsonutils.JSONDict, error) { + params, err := o.BaseListOptions.Params() + if err != nil { + return nil, err + } + return params, nil +} + +type SchedtagModelPairOptions struct { + SCHEDTAG string `help:"Scheduler tag"` + OBJECT string `help:"Object id"` +} diff --git a/pkg/mcclient/options/schedulers.go b/pkg/mcclient/options/schedulers.go index bbf92b0ea4..85832ef394 100644 --- a/pkg/mcclient/options/schedulers.go +++ b/pkg/mcclient/options/schedulers.go @@ -13,10 +13,11 @@ type SchedulerTestBaseOptions struct { Mem int64 `help:"Memory size (MB), default 512" metavar:"MEMORY" default:"512"` Ncpu int64 `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` Disk []string `help:"Disk descriptions" nargs:"+"` + DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` BaremetalDiskConfig []string `help:"Baremetal disk layout configuration"` Net []string `help:"Network descriptions" metavar:"NETWORK"` IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` - SchedTag []string `help:"Schedule policy, key = SchedTag name, value = require|exclude|prefer|avoid" metavar:""` + Schedtag []string `help:"Schedule policy, key = SchedTag name, value = require|exclude|prefer|avoid" metavar:""` Zone string `help:"Preferred zone where virtual server should be created"` Host string `help:"Preferred host where virtual server should be created"` Project string `help:"Owner project ID or Name"` @@ -48,8 +49,8 @@ func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*jsonutils.JS if len(o.Zone) > 0 { data.Add(jsonutils.NewString(o.Zone), "prefer_zone") } - if len(o.SchedTag) > 0 { - for i, aggr := range o.SchedTag { + if len(o.Schedtag) > 0 { + for i, aggr := range o.Schedtag { data.Add(jsonutils.NewString(aggr), fmt.Sprintf("aggregate.%d", i)) } } diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go index 6563aaf7b0..bcd477b839 100644 --- a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -1,17 +1,13 @@ package predicates import ( - "fmt" - "yunion.io/x/jsonutils" - "yunion.io/x/log" + computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/core/score" - "yunion.io/x/onecloud/pkg/scheduler/db/models" - "yunion.io/x/onecloud/pkg/util/conditionparser" ) // NOTE: Aggregate Description @@ -26,101 +22,16 @@ import ( type AggregatePredicate struct { BasePredicate plugin.BasePlugin - AggregateHosts hostsAggregatesMap - RequireAggregates []api.Aggregate - ExcludeAggregates []api.Aggregate - AvoidAggregates []api.Aggregate - PreferAggregates []api.Aggregate - AggregateMap map[string]api.Aggregate + + SchedtagPredicate *SchedtagPredicate } -type hostAggregates []*models.Aggregate - -type hostsAggregatesMap map[string]hostAggregates - func (p *AggregatePredicate) Name() string { return "host_aggregate" } func (p *AggregatePredicate) Clone() core.FitPredicate { - return &AggregatePredicate{ - AggregateMap: make(map[string]api.Aggregate, 0), - } -} - -func getHostAndServerSchedDesc(u *core.Unit, c core.Candidater) *jsonutils.JSONDict { - ret := jsonutils.NewDict() - hostSchedDesc := c.GetSchedDesc() - srvSchedDesc := jsonutils.Marshal(u.SchedData()) - ret.Add(hostSchedDesc, "host") - ret.Add(srvSchedDesc, "server") - return ret -} - -func getHostDynamicSchedtags(u *core.Unit, c core.Candidater) ([]*models.Aggregate, error) { - schedDesc := getHostAndServerSchedDesc(u, c) - - dynamicTags, err := models.FetchEnabledDynamicschedtags() - if err != nil { - return nil, err - } - aggs := []*models.Aggregate{} - for _, tag := range dynamicTags { - matched, err := conditionparser.Eval(tag.Condition, schedDesc) - if err != nil { - log.Errorf("Condition parse eval: condition: %q, desc: %s, error: %v", tag.Condition, schedDesc, err) - continue - } - if !matched { - continue - } - aggregate, err := tag.FetchSchedTag() - if err != nil { - log.Errorf("Get dynamic schedtag %q error: %v", tag.SchedtagId, err) - continue - } - aggs = append(aggs, aggregate) - } - return aggs, nil -} - -func mergeHostSchedtags(c core.Candidater, staticTags, dynamicTags []*models.Aggregate) []*models.Aggregate { - isIn := func(tags []*models.Aggregate, dt *models.Aggregate) bool { - for _, t := range tags { - if t.ID == dt.ID { - return true - } - } - return false - } - ret := []*models.Aggregate{} - ret = append(ret, staticTags...) - for _, dt := range dynamicTags { - if !isIn(staticTags, dt) { - ret = append(ret, dt) - log.Debugf("Append dynamic schedtag %s to host %q", dt, c.IndexKey()) - } - } - return ret -} - -func hostsAggregatesInfo(u *core.Unit, cs []core.Candidater) (hostsAggregatesMap, []*models.Aggregate) { - ret := make(map[string]hostAggregates, 0) - allAggs := make([]*models.Aggregate, 0) - for _, c := range cs { - hostAggs := c.GetHostAggregates() - dynamicMatchedAggs, err := getHostDynamicSchedtags(u, c) - if err != nil { - log.Errorf("Get host %q dynamic schedtag error: %v", c.IndexKey(), err) - } else { - hostAggs = mergeHostSchedtags(c, hostAggs, dynamicMatchedAggs) - } - ret[c.IndexKey()] = hostAggs - } - if len(cs) > 0 { - allAggs = cs[0].GetAggregates() - } - return ret, allAggs + return &AggregatePredicate{} } func (p *AggregatePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { @@ -130,72 +41,16 @@ func (p *AggregatePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (boo return false, nil } - hsMap, allAggs := hostsAggregatesInfo(u, cs) - p.AggregateHosts = hsMap - appendedAggIds := make(map[string]int, len(data.Aggregates)) - - for _, aggregate := range data.Aggregates { - switch aggregate.Strategy { - case api.AggregateStrategyRequire: - p.RequireAggregates = append(p.RequireAggregates, aggregate) - case api.AggregateStrategyExclude: - p.ExcludeAggregates = append(p.ExcludeAggregates, aggregate) - - case api.AggregateStrategyPrefer: - p.PreferAggregates = append(p.PreferAggregates, aggregate) - - case api.AggregateStrategyAvoid: - p.AvoidAggregates = append(p.AvoidAggregates, aggregate) - } - - p.AggregateMap[aggregate.Idx] = aggregate - appendedAggIds[aggregate.Idx] = 1 - } - - for _, aggregate := range allAggs { - _, nameOk := appendedAggIds[aggregate.Name] - _, idOk := appendedAggIds[aggregate.ID] - if !(nameOk || idOk) { - agg := api.Aggregate{Idx: aggregate.ID, Strategy: aggregate.DefaultStrategy} - switch agg.Strategy { - case api.AggregateStrategyRequire: - p.RequireAggregates = append(p.RequireAggregates, agg) - case api.AggregateStrategyExclude: - p.ExcludeAggregates = append(p.ExcludeAggregates, agg) - - case api.AggregateStrategyPrefer: - p.PreferAggregates = append(p.PreferAggregates, agg) - - case api.AggregateStrategyAvoid: - p.AvoidAggregates = append(p.AvoidAggregates, agg) - } - } + allAggs, err := GetAllSchedtags(computemodels.HostManager.KeywordPlural()) + if err != nil { + return false, err } + p.SchedtagPredicate = NewSchedtagPredicate(data.Schedtags, allAggs) u.AppendSelectPlugin(p) return true, nil } -func getHostAggregateCount(inAggs []api.Aggregate, hAggs []*models.Aggregate, strategy string) (countMap map[string]int) { - countMap = make(map[string]int) - - in := func(hAgg *models.Aggregate, inAggs []api.Aggregate) bool { - for _, agg := range inAggs { - if agg.Idx == hAgg.ID || agg.Idx == hAgg.Name { - return true - } - } - return false - } - - for _, hAgg := range hAggs { - if in(hAgg, inAggs) { - countMap[fmt.Sprintf("%s:%s:%s", hAgg.ID, hAgg.Name, strategy)]++ - } - } - return -} - func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(p, u, c) @@ -206,66 +61,46 @@ func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []c return h.GetResult() } +type schedtagCandidateW struct { + core.Candidater + schedData *api.SchedData +} + +func (w schedtagCandidateW) GetDynamicSchedDesc() *jsonutils.JSONDict { + ret := jsonutils.NewDict() + hostSchedDesc := w.GetSchedDesc() + srvSchedDesc := jsonutils.Marshal(w.schedData) + ret.Add(hostSchedDesc, "host") + ret.Add(srvSchedDesc, "server") + return ret +} + +func (w schedtagCandidateW) GetSchedtags() []computemodels.SSchedtag { + return w.Getter().HostSchedtags() +} + +func (w schedtagCandidateW) ResourceType() string { + return computemodels.HostManager.KeywordPlural() +} + func (p *AggregatePredicate) exec(h *PredicateHelper) string { - ahs := p.AggregateHosts - candidateID := h.Candidate.IndexKey() - - log.V(10).Debugf(">>>> ExcludeAggregates: %#v, RequireAggregates: %#v, AvoidAggregates: %#v, PreferAggregates: %#v, candidateID: %v", p.ExcludeAggregates, p.RequireAggregates, p.AvoidAggregates, p.PreferAggregates, candidateID) - - if len(p.ExcludeAggregates) > 0 { - inExclude := func(a *models.Aggregate) bool { - for _, agg := range p.ExcludeAggregates { - if agg.Idx == a.ID || agg.Idx == a.Name { - return true - } - } - - return false - } - - if ah, ok := ahs[candidateID]; ok { - for _, a := range ah { - if inExclude(a) { - return fmt.Sprintf("exclude by aggregate: '%s:%s'", a.Name, a.ID) - } - } - } - } - - if len(p.RequireAggregates) > 0 { - var as []*models.Aggregate = nil - if ah, ok := ahs[candidateID]; ok { - as = ah - } - - inRequire := func(agg api.Aggregate) bool { - for _, a := range as { - if a.ID == agg.Idx || a.Name == agg.Idx { - return true - } - } - - return false - } - - for _, agg := range p.RequireAggregates { - if !inRequire(agg) { - return fmt.Sprintf("need aggregate: '%s'", agg.Idx) - } - } + if err := p.SchedtagPredicate.Check( + schedtagCandidateW{ + Candidater: h.Candidate, + schedData: h.Unit.SchedData(), + }, + ); err != nil { + return err.Error() } return "" } func (p *AggregatePredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { - hostAggs, ok := p.AggregateHosts[c.IndexKey()] - if !ok { - return - } + hostAggs := c.Getter().HostSchedtags() - avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid) - preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer) + avoidCountMap := GetSchedtagCount(p.SchedtagPredicate.GetAvoidTags(), hostAggs, api.AggregateStrategyAvoid) + preferCountMap := GetSchedtagCount(p.SchedtagPredicate.GetPreferTags(), hostAggs, api.AggregateStrategyPrefer) setScore := func(aggCountMap map[string]int, postiveScore bool) { stepScore := core.PriorityStep diff --git a/pkg/scheduler/algorithm/predicates/baremetal/base.go b/pkg/scheduler/algorithm/predicates/baremetal/base.go index 2e1e55ebb7..b5aef10654 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/base.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/base.go @@ -1,9 +1,9 @@ package baremetal import ( - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" "yunion.io/x/onecloud/pkg/scheduler/core" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type BasePredicate struct { diff --git a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go index 973244cc0b..62191c39ae 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go @@ -61,29 +61,29 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor var errMsgs []string for _, network := range candidate.Networks { appendError := func(errMsg string) { - errMsgs = append(errMsgs, fmt.Sprintf("%s: %s", network.ID, errMsg)) + errMsgs = append(errMsgs, fmt.Sprintf("%s: %s", network.Id, errMsg)) } - if !((network.Ports > 0 || isMigrate()) && network.IsExit == exit) { + if !((network.GetPorts() > 0 || isMigrate()) && network.IsExitNetwork() == exit) { appendError(predicates.ErrNoPorts) } - if wire != "" && !utils.HasPrefix(wire, network.Wire) && !utils.HasPrefix(wire, network.WireID) { // re + if wire != "" && !utils.HasPrefix(wire, network.WireId) && !utils.HasPrefix(wire, network.GetWire().GetName()) { // re appendError(predicates.ErrWireIsNotMatch) } - if (!private && network.IsPublic) || (private && !network.IsPublic && network.TenantID == schedData.OwnerTenantID) { + if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.OwnerTenantID) { // TODO: support reservedNetworks reservedNetworks := 0 - restPort := int64(network.Ports - reservedNetworks) + restPort := int64(network.GetPorts() - reservedNetworks) if restPort == 0 { appendError("not enough network port") continue } - counter := u.CounterManager.GetOrCreate("net:"+network.ID, func() core.Counter { + counter := u.CounterManager.GetOrCreate("net:"+network.Id, func() core.Counter { return core.NewNormalCounter(restPort) }) - u.SharedResourceManager.Add(network.ID, counter) + u.SharedResourceManager.Add(network.Id, counter) counters.Add(counter) - p.SelectedNetworks.Store(network.ID, counter.GetCount()) + p.SelectedNetworks.Store(network.Id, counter.GetCount()) return "" } else { appendError(predicates.ErrNotOwner) @@ -105,7 +105,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire) } for _, net := range candidate.Networks { - if (network.Idx == net.ID || network.Idx == net.Name) && (net.IsPublic || net.TenantID == schedData.OwnerTenantID) && (net.Ports > 0 || isMigrate()) { + if (network.Idx == net.Id || network.Idx == net.Name) && (net.IsPublic || net.ProjectId == schedData.OwnerTenantID) && (net.GetPorts() > 0 || isMigrate()) { h.SetCapacity(1) return "" } diff --git a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go new file mode 100644 index 0000000000..e6ba918a9e --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go @@ -0,0 +1,168 @@ +package predicates + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/errors" + + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" + "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +type DiskStoragesMap map[int][]*api.CandidateStorage + +type CandidateDiskStoragesMap map[string]DiskStoragesMap + +type DiskSchedtagPredicate struct { + BasePredicate + plugin.BasePlugin + + SchedtagPredicate *SchedtagPredicate + + CandidateDiskStoragesMap CandidateDiskStoragesMap +} + +func (p *DiskSchedtagPredicate) Name() string { + return "disk_schedtag" +} + +func (p *DiskSchedtagPredicate) Clone() core.FitPredicate { + return &DiskSchedtagPredicate{ + CandidateDiskStoragesMap: make(map[string]DiskStoragesMap), + } +} + +func (p *DiskSchedtagPredicate) getSchedtagDisks(disks []*api.Disk) ([]*api.Disk, []*api.Disk) { + noTagDisk := make([]*api.Disk, 0) + tagDisk := make([]*api.Disk, 0) + for _, d := range disks { + if len(d.Schedtags) != 0 { + tagDisk = append(tagDisk, d) + } else { + noTagDisk = append(noTagDisk, d) + } + } + return noTagDisk, tagDisk +} + +func (p *DiskSchedtagPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + disks := u.SchedData().Disks + if len(disks) == 0 { + return false, nil + } + + // always select each storages to disks + u.AppendSelectPlugin(p) + + return true, nil +} + +type schedtagStorageW struct { + candidater *api.CandidateStorage + disk *api.Disk +} + +func (w schedtagStorageW) IndexKey() string { + return fmt.Sprintf("%d:%s", w.disk.Size, w.disk.Backend) +} + +func (w schedtagStorageW) GetDynamicSchedDesc() *jsonutils.JSONDict { + return nil +} + +func (w schedtagStorageW) GetSchedtags() []models.SSchedtag { + return w.candidater.Schedtags +} + +func (w schedtagStorageW) ResourceType() string { + return models.StorageManager.KeywordPlural() +} + +func (p *DiskSchedtagPredicate) check(d *api.Disk, s *api.CandidateStorage) (bool, error) { + allTags, err := GetAllSchedtags(models.StorageManager.KeywordPlural()) + if err != nil { + return false, err + } + tagPredicate := NewSchedtagPredicate(d.Schedtags, allTags) + if err := tagPredicate.Check( + schedtagStorageW{ + candidater: s, + disk: d, + }, + ); err != nil { + return false, err + } + return true, nil +} + +func (p *DiskSchedtagPredicate) checkStorages(d *api.Disk, storages []*api.CandidateStorage) ([]*api.CandidateStorage, error) { + errs := make([]error, 0) + ret := make([]*api.CandidateStorage, 0) + for _, s := range storages { + _, err := p.check(d, s) + if err != nil { + // append err, storage not suit disk + errs = append(errs, err) + continue + } + ret = append(ret, s) + } + if len(ret) == 0 { + return nil, errors.NewAggregate(errs) + } + return ret, nil +} + +func (p *DiskSchedtagPredicate) GetDiskStoragesMap(candidateId string) DiskStoragesMap { + ret, ok := p.CandidateDiskStoragesMap[candidateId] + if !ok { + ret = make(map[int][]*api.CandidateStorage) + p.CandidateDiskStoragesMap[candidateId] = ret + } + return ret +} + +func (p *DiskSchedtagPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := NewPredicateHelper(p, u, c) + + //noTagDisks, tagDisks := p.getSchedtagDisks(u.SchedData().Disks) + storages := c.Getter().Storages() + ds := p.GetDiskStoragesMap(c.IndexKey()) + disks := u.SchedData().Disks + for _, d := range disks { + matchedStorages, err := p.checkStorages(d, storages) + if err != nil { + h.Exclude(err.Error()) + } + ds[d.Index] = matchedStorages + } + + return h.GetResult() +} + +func (p *DiskSchedtagPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { + res := u.GetAllocatedResource(c.IndexKey()) + diskStorages := p.GetDiskStoragesMap(c.IndexKey()) + res.Disks = make([]*core.DiskAllocatedResource, len(diskStorages)) + disks := u.SchedData().Disks + for idx, ds := range diskStorages { + res.Disks[idx] = p.allocatedDiskResource(c, disks[idx], ds) + } + log.Errorf("============OnSelectEnd %s called: %#v", c.Getter().Name(), jsonutils.Marshal(res.Disks).String()) +} + +func (p *DiskSchedtagPredicate) allocatedDiskResource(c core.Candidater, disk *api.Disk, storages []*api.CandidateStorage) *core.DiskAllocatedResource { + storage := p.selectStorage(disk, storages) + return &core.DiskAllocatedResource{ + Index: disk.Index, + StorageId: storage.Id, + } +} + +func (p *DiskSchedtagPredicate) selectStorage(d *api.Disk, storages []*api.CandidateStorage) *api.CandidateStorage { + return storages[0] +} diff --git a/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go b/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go index 788d0f0ca2..eb0cfe9c22 100644 --- a/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go @@ -27,7 +27,7 @@ func (f *HypervisorPredicate) Clone() core.FitPredicate { } func hostHasContainerTag(c core.Candidater) bool { - aggs := c.GetHostAggregates() + aggs := c.Getter().HostSchedtags() for _, agg := range aggs { if agg.Name == CONTAINER_ALLOWED_TAG { return true diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index c90f700f4e..cd42c648f1 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -8,11 +8,12 @@ import ( "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" - networks "yunion.io/x/onecloud/pkg/scheduler/db/models" ) // NetworkPredicate will filter the current network information with @@ -56,16 +57,16 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor } // ServerType's value is 'guest', 'container' or ''(support all type) will return true. - isMatchServerType := func(network *networks.NetworkSchedResult) bool { + isMatchServerType := func(network *models.SNetwork) bool { return sets.NewString("guest", "", "container").Has(network.ServerType) } - counterOfNetwork := func(u *core.Unit, n *networks.NetworkSchedResult, r int) core.Counter { - counter := u.CounterManager.GetOrCreate("net:"+n.ID, func() core.Counter { - return core.NewNormalCounter(int64(n.Ports - r)) + counterOfNetwork := func(u *core.Unit, n *models.SNetwork, r int) core.Counter { + counter := u.CounterManager.GetOrCreate("net:"+n.Id, func() core.Counter { + return core.NewNormalCounter(int64(n.GetPorts() - r)) }) - u.SharedResourceManager.Add(n.ID, counter) + u.SharedResourceManager.Add(n.GetId(), counter) return counter } @@ -81,36 +82,36 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor errMsgs = append(errMsgs, errMsg) } - if !isMatchServerType(n) { + if !isMatchServerType(&n) { appendError(predicates.ErrServerTypeIsNotMatch) } - if n.IsExit != exit { + if n.IsExitNetwork() != exit { appendError(predicates.ErrExitIsNotMatch) } - if !(n.Ports > 0 || isMigrate()) { + if !(n.GetPorts() > 0 || isMigrate()) { appendError(predicates.ErrNoPorts) } - if wire != "" && !utils.HasPrefix(wire, n.Wire) && !utils.HasPrefix(wire, n.WireID) { // re + if wire != "" && !utils.HasPrefix(wire, n.WireId) && !utils.HasPrefix(wire, n.GetWire().GetName()) { // re appendError(predicates.ErrWireIsNotMatch) } - if !((!private && n.IsPublic) || (private && !n.IsPublic && n.TenantID == d.OwnerTenantID)) { + if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.OwnerTenantID)) { appendError(predicates.ErrNotOwner) } if len(errMsgs) == 0 { // add resource reservedNetworks := 0 - counter := counterOfNetwork(u, n, reservedNetworks) - p.SelectedNetworks.Store(n.ID, counter.GetCount()) + counter := counterOfNetwork(u, &n, reservedNetworks) + p.SelectedNetworks.Store(n.GetId(), counter.GetCount()) counters.Add(counter) found = true } else { fullErrMsgs = append(fullErrMsgs, - fmt.Sprintf("%s: %s", n.ID, strings.Join(errMsgs, ",")), + fmt.Sprintf("%s: %s", n.Id, strings.Join(errMsgs, ",")), ) } } @@ -131,7 +132,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor } isNetworkAvaliable := func(n *api.Network, counters *core.MinCounters, - networks []*networks.NetworkSchedResult) string { + networks []models.SNetwork) string { if n.Idx == "" { counters0 := core.NewCounters() ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0) @@ -149,21 +150,21 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID)) continue }*/ - if !(n.Idx == net.ID || n.Idx == net.Name) { - errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.ID)) - } else if !(net.IsPublic || net.TenantID == d.OwnerTenantID) { - errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.ID, net.TenantID, d.OwnerTenantID)) - } else if !(net.Ports > 0 || isMigrate()) { - errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.ID)) + if !(n.Idx == net.GetId() || n.Idx == net.GetName()) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id)) + } else if !(net.IsPublic || net.ProjectId == d.OwnerTenantID) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.OwnerTenantID)) + } else if !(net.GetPorts() > 0 || isMigrate()) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id)) } else { // add resource reservedNetworks := 0 - counter := counterOfNetwork(u, net, reservedNetworks) + counter := counterOfNetwork(u, &net, reservedNetworks) if counter.GetCount() < d.Count { errMsgs = append(errMsgs, fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count)) continue } - p.SelectedNetworks.Store(net.ID, counter.GetCount()) + p.SelectedNetworks.Store(net.Id, counter.GetCount()) counters.Add(counter) return "" } diff --git a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go index 0a57a74285..30a84ea995 100644 --- a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" @@ -52,7 +53,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor isStorageAccessible := func(storage string) bool { for _, s := range hc.Storages { - if storage == s.ID || storage == s.Name { + if storage == s.Id || storage == s.Name { return true } } @@ -82,9 +83,11 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor ss := []string{} for _, s := range hc.Storages { if s.StorageType == backend { - total := int64(float64(s.Capacity) * s.Cmtbound) - free := total - s.UsedCapacity - s.WasteCapacity - ss = append(ss, fmt.Sprintf("(%v-%v-%v=%v)", total, s.UsedCapacity, s.WasteCapacity, free)) + total := int64(float32(s.Capacity) * s.Cmtbound) + used := s.GetUsedCapacity(tristate.True) + waste := s.GetUsedCapacity(tristate.False) + free := total - int64(used) - int64(waste) + ss = append(ss, fmt.Sprintf("(%v-%v-%v=%v)", total, used, waste, free)) } } return strings.Join(ss, " + ") diff --git a/pkg/scheduler/algorithm/predicates/k8s/network_predicate.go b/pkg/scheduler/algorithm/predicates/k8s/network_predicate.go index 835ff0a74a..3eb5980417 100644 --- a/pkg/scheduler/algorithm/predicates/k8s/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/k8s/network_predicate.go @@ -7,9 +7,11 @@ import ( "k8s.io/client-go/kubernetes" "yunion.io/x/pkg/util/errors" + "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" - "yunion.io/x/onecloud/pkg/scheduler/db/models" ) const ( @@ -56,13 +58,13 @@ func (p *NetworkPredicate) Execute(cli *kubernetes.Clientset, pod *v1.Pod, node return true, nil } -func (p NetworkPredicate) checkByNetworks(nets []*models.NetworkSchedResult) error { +func (p NetworkPredicate) checkByNetworks(nets []models.SNetwork) error { if len(nets) == 0 { return fmt.Errorf("Network is empty") } errs := make([]error, 0) for _, net := range nets { - err := p.checkByNetwork(net) + err := p.checkByNetwork(&net) if err == nil { return nil } @@ -71,23 +73,23 @@ func (p NetworkPredicate) checkByNetworks(nets []*models.NetworkSchedResult) err return errors.NewAggregate(errs) } -func (p NetworkPredicate) checkByNetwork(net *models.NetworkSchedResult) error { - if net.Ports <= 0 { +func (p NetworkPredicate) checkByNetwork(net *models.SNetwork) error { + if net.GetPorts() <= 0 { return fmt.Errorf("Network %s no free IPs", net.Name) } - if !(p.network == net.Name || p.network == net.ID) { - return fmt.Errorf("Network %s:%s or id not match %s", net.Name, net.ID, p.network) + if !(p.network == net.Name || p.network == net.Id) { + return fmt.Errorf("Network %s:%s or id not match %s", net.Name, net.Id, p.network) } return nil } -func (p NetworkPredicate) checkNetworksIP(ip string, nets []*models.NetworkSchedResult) error { +func (p NetworkPredicate) checkNetworksIP(ip string, nets []models.SNetwork) error { if len(nets) == 0 { return fmt.Errorf("Network is empty") } errs := make([]error, 0) for _, net := range nets { - err := p.checkNetworkIP(ip, net) + err := p.checkNetworkIP(ip, &net) if err == nil { return nil } @@ -96,10 +98,12 @@ func (p NetworkPredicate) checkNetworksIP(ip string, nets []*models.NetworkSched return errors.NewAggregate(errs) } -func (p NetworkPredicate) checkNetworkIP(ip string, net *models.NetworkSchedResult) error { - if ok, err := net.ContainsIp(ip); err != nil { +func (p NetworkPredicate) checkNetworkIP(ip string, net *models.SNetwork) error { + ipAddr, err := netutils.NewIPV4Addr(ip) + if err != nil { return err - } else if !ok { + } + if ok := net.GetIPRange().Contains(ipAddr); !ok { return fmt.Errorf("Network %s not contains ip %s", net.Name, ip) } return nil diff --git a/pkg/scheduler/algorithm/predicates/schedtag_helper.go b/pkg/scheduler/algorithm/predicates/schedtag_helper.go new file mode 100644 index 0000000000..868677205d --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/schedtag_helper.go @@ -0,0 +1,263 @@ +package predicates + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/util/conditionparser" +) + +type ISchedtagPredicate interface { + GetExcludeTags() []api.Schedtag + GetRequireTags() []api.Schedtag + GetAvoidTags() []api.Schedtag + GetPreferTags() []api.Schedtag +} + +type ISchedtagCandidate interface { + IndexKey() string + ResourceType() string + // GetSchedtags return schedtags bind to this candidate + GetSchedtags() []models.SSchedtag + // GetDynamicSchedDesc return schedule description used by dynamic schedtags condition eval + GetDynamicSchedDesc() *jsonutils.JSONDict +} + +type SchedtagPredicate struct { + requireTags []api.Schedtag + execludeTags []api.Schedtag + preferTags []api.Schedtag + avoidTags []api.Schedtag + checker *SchedtagChecker +} + +func NewSchedtagPredicate(reqTags []api.Schedtag, allTags []models.SSchedtag) *SchedtagPredicate { + p := new(SchedtagPredicate) + requireTags, execludeTags, preferTags, avoidTags := GetRequestSchedtags(reqTags, allTags) + p.requireTags = requireTags + p.execludeTags = execludeTags + p.preferTags = preferTags + p.avoidTags = avoidTags + p.checker = new(SchedtagChecker) + return p +} + +func (p *SchedtagPredicate) GetExcludeTags() []api.Schedtag { + return p.execludeTags +} + +func (p *SchedtagPredicate) GetRequireTags() []api.Schedtag { + return p.requireTags +} + +func (p *SchedtagPredicate) GetAvoidTags() []api.Schedtag { + return p.avoidTags +} + +func (p *SchedtagPredicate) GetPreferTags() []api.Schedtag { + return p.preferTags +} + +func (p *SchedtagPredicate) Check(candidate ISchedtagCandidate) error { + return p.checker.Check(p, candidate) +} + +func GetSchedtagCount(inTags []api.Schedtag, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { + countMap = make(map[string]int) + + in := func(objTag models.SSchedtag, inTags []api.Schedtag) bool { + for _, tag := range inTags { + if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + return true + } + } + return false + } + + for _, objTag := range objTags { + if in(objTag, inTags) { + countMap[fmt.Sprintf("%s:%s:%s", objTag.Id, objTag.Name, strategy)]++ + } + } + return +} + +func GetAllSchedtags(resType string) ([]models.SSchedtag, error) { + tags, err := models.SchedtagManager.GetResourceSchedtags(resType) + if err != nil { + return nil, err + } + return tags, nil +} + +func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (requireTags, execludeTags, preferTags, avoidTags []api.Schedtag) { + requireTags = make([]api.Schedtag, 0) + execludeTags = make([]api.Schedtag, 0) + preferTags = make([]api.Schedtag, 0) + avoidTags = make([]api.Schedtag, 0) + + appendedTagIds := make(map[string]int) + + appendTagByStrategy := func(tag api.Schedtag) { + switch tag.Strategy { + case models.STRATEGY_REQUIRE: + requireTags = append(requireTags, tag) + case models.STRATEGY_EXCLUDE: + execludeTags = append(execludeTags, tag) + case models.STRATEGY_PREFER: + preferTags = append(preferTags, tag) + case models.STRATEGY_AVOID: + avoidTags = append(avoidTags, tag) + } + } + + for _, tag := range reqTags { + appendTagByStrategy(tag) + + appendedTagIds[tag.Idx] = 1 + } + + for _, tag := range allTags { + _, nameOk := appendedTagIds[tag.Name] + _, idOk := appendedTagIds[tag.Id] + + if !(nameOk || idOk) { + apiTag := api.Schedtag{Idx: tag.Id, Strategy: tag.DefaultStrategy} + appendTagByStrategy(apiTag) + } + } + + return +} + +type SchedtagChecker struct{} + +type apiTags []api.Schedtag + +func (t apiTags) contains(objTag models.SSchedtag) bool { + for _, tag := range t { + if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + return true + } + } + return false +} + +type objTags []models.SSchedtag + +func (t objTags) contains(atag api.Schedtag) bool { + for _, tag := range t { + if tag.Id == atag.Idx || tag.Name == atag.Idx { + return true + } + } + return false +} + +func (c *SchedtagChecker) contains(tags []api.Schedtag, objTag models.SSchedtag) bool { + for _, tag := range tags { + if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + return true + } + } + return false +} + +func (c *SchedtagChecker) HasIntersection(tags []api.Schedtag, objTags []models.SSchedtag) (bool, *models.SSchedtag) { + var atags apiTags = tags + for _, objTag := range objTags { + if atags.contains(objTag) { + return true, &objTag + } + } + return false, nil +} + +func (c *SchedtagChecker) Contains(objectTags []models.SSchedtag, tags []api.Schedtag) (bool, *api.Schedtag) { + var otags objTags = objectTags + for _, tag := range tags { + if !otags.contains(tag) { + return false, &tag + } + } + return true, nil +} + +func (p *SchedtagChecker) getDynamicSchedtags(schedDesc *jsonutils.JSONDict) ([]models.SSchedtag, error) { + if schedDesc == nil { + return []models.SSchedtag{}, nil + } + dynamicTags := models.DynamicschedtagManager.GetAllEnabledDynamicSchedtags() + + tags := []models.SSchedtag{} + for _, tag := range dynamicTags { + matched, err := conditionparser.Eval(tag.Condition, schedDesc) + if err != nil { + log.Errorf("Condition parse eval: condition: %q, desc: %s, error: %v", tag.Condition, schedDesc, err) + continue + } + if !matched { + continue + } + objTag := tag.GetSchedtag() + if objTag != nil { + tags = append(tags, *objTag) + } + } + return tags, nil +} + +func (c *SchedtagChecker) mergeSchedtags(candiate ISchedtagCandidate, staticTags, dynamicTags []models.SSchedtag) []models.SSchedtag { + isIn := func(tags []models.SSchedtag, dt models.SSchedtag) bool { + for _, t := range tags { + if t.Id == dt.Id { + return true + } + } + return false + } + ret := []models.SSchedtag{} + ret = append(ret, staticTags...) + for _, dt := range dynamicTags { + if !isIn(staticTags, dt) { + ret = append(ret, dt) + log.Debugf("Append dynamic schedtag %s to %s %q", dt, candiate.ResourceType(), candiate.IndexKey()) + } + } + return ret +} + +func (c *SchedtagChecker) GetCandidateSchedtags(candidate ISchedtagCandidate) ([]models.SSchedtag, error) { + staticTags := candidate.GetSchedtags() + dynamicTags, err := c.getDynamicSchedtags(candidate.GetDynamicSchedDesc()) + if err != nil { + return nil, err + } + return c.mergeSchedtags(candidate, staticTags, dynamicTags), nil +} + +func (c *SchedtagChecker) Check(p ISchedtagPredicate, candidate ISchedtagCandidate) error { + candidateTags, err := c.GetCandidateSchedtags(candidate) + if err != nil { + return err + } + execludeTags := p.GetExcludeTags() + if len(execludeTags) > 0 { + if ok, tag := c.HasIntersection(execludeTags, candidateTags); ok { + return fmt.Errorf("Execlude by schedtag: '%s:%s'", tag.Name, tag.Id) + } + } + + requireTags := p.GetRequireTags() + if len(requireTags) > 0 { + if ok, tag := c.Contains(candidateTags, requireTags); !ok { + return fmt.Errorf("Need schedtag: '%s'", tag.Idx) + } + } + + return nil +} diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go deleted file mode 100644 index dfa0d0e6ac..0000000000 --- a/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go +++ /dev/null @@ -1,55 +0,0 @@ -package guest - -import ( - "yunion.io/x/onecloud/pkg/scheduler/algorithm/priorities" - "yunion.io/x/onecloud/pkg/scheduler/core" -) - -type AvoidSameClusterPriority struct { - priorities.BasePriority - ClusterTbl map[string]int -} - -func (p *AvoidSameClusterPriority) Name() string { - return "guest_avoid_same_cluster" -} - -func (p *AvoidSameClusterPriority) Clone() core.Priority { - return &AvoidSameClusterPriority{ClusterTbl: make(map[string]int)} -} - -func (p *AvoidSameClusterPriority) PreExecute(u *core.Unit, cs []core.Candidater) (bool, []core.PredicateFailureReason, error) { - d := u.SchedData() - clusterTbl := make(map[string]int, 0) - ownerTenantID := d.OwnerTenantID - - for _, c := range cs { - hc, err := p.HostCandidate(c) - if err != nil { - return false, nil, err - } - - if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { - clusterId := hc.ClusterID - - if count0, ok := clusterTbl[clusterId]; ok { - clusterTbl[clusterId] = count0 + int(count) - } else { - clusterTbl[clusterId] = int(count) - } - } - } - - p.ClusterTbl = clusterTbl - return true, nil, nil -} - -func (p *AvoidSameClusterPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { - h := priorities.NewPriorityHelper(p, u, c) - - if count, ok := p.ClusterTbl[c.Get("ClusterID").(string)]; ok { - h.SetScore(-20 * count) - } - - return h.GetResult() -} diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go index 54f6670b63..7a1b8ec849 100644 --- a/pkg/scheduler/algorithmprovider/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -27,6 +27,7 @@ func defaultPredicates() sets.String { factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicateguest.NetworkPredicate{}), factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicateguest.IsolatedDevicePredicate{}), factory.RegisterFitPredicate("l-GuestResourceTypeFilter", &predicates.ResourceTypePredicate{}), + factory.RegisterFitPredicate("m-GuestDiskschedtagFilter", &predicates.DiskSchedtagPredicate{}), ) } diff --git a/pkg/scheduler/api/candidate.go b/pkg/scheduler/api/candidate.go index 07ba0aca02..467fd3f39c 100644 --- a/pkg/scheduler/api/candidate.go +++ b/pkg/scheduler/api/candidate.go @@ -1,22 +1,23 @@ package api import ( + "encoding/json" + "io" "strconv" simplejson "github.com/bitly/go-simplejson" - - "yunion.io/x/log" ) // CandidateListArgs is a struct just for parsing candidate // resource list parameters. type CandidateListArgs struct { - Type string - Zone string - Pool string - Limit int64 - Offset int64 - Avaliable bool + Type string `json:"type"` + Region string `json:"region"` + Zone string `json:"zone"` + //Pool string + Limit int64 `json:"limit"` + Offset int64 `json:"offset"` + Avaliable bool `json:"available"` } type ResultResource struct { @@ -75,44 +76,20 @@ const ( // NewCandidateListArgs provides a function that // will parse candidate's list args from a json data. -func NewCandidateListArgs(sjson *simplejson.Json) (*CandidateListArgs, error) { - args := &CandidateListArgs{ - Limit: DefaultCandidateListArgsLimit, +func NewCandidateListArgs(r io.Reader) (*CandidateListArgs, error) { + args := CandidateListArgs{} + err := json.NewDecoder(r).Decode(&args) + if err != nil { + return nil, err } - if argsType, ok := sjson.CheckGet("type"); ok { - args.Type = argsType.MustString() - } else { + if args.Limit == 0 { + args.Limit = DefaultCandidateListArgsLimit + } + if args.Type == "" { args.Type = "all" } - if zone, ok := sjson.CheckGet("zone"); ok { - args.Zone = zone.MustString() - } - - if pool, ok := sjson.CheckGet("pool"); ok { - args.Pool = pool.MustString() - } - - if limit, ok := sjson.CheckGet("limit"); ok { - limitv, err := limit.Int64() - if err != nil { - limitv, err = strconv.ParseInt(limit.MustString(), 10, 64) - if err != nil { - log.Errorln(err) - } - } - args.Limit = limitv - } - - if offset, ok := sjson.CheckGet("offset"); ok { - args.Offset = offset.MustInt64() - } - - if avaliable, ok := sjson.CheckGet("avaliable"); ok { - args.Avaliable = avaliable.MustBool() - } - - return args, nil + return &args, nil } // CandidateDetailArgs is a struct just for parsing candidate diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index 6cde908351..eca23da386 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -11,27 +11,29 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/compute/models" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type Meta map[string]string type Disk struct { - Backend string `json:"backend"` - ImageID string `json:"image_id"` - Fs *string `json:"fs"` - Os string `json:"os"` - OSDistribution string `json:"os_distribution"` - OsVersion string `json:"os_version"` - Format string `json:"format"` - MountPoint *string `json:"mountpoint"` - Driver *string `json:"driver"` - Cache *string `json:"cache"` - ImageDiskFormat string `json:"image_disk_format"` - Size int64 `json:"size"` - Storage *string `json:"storage"` + Backend string `json:"backend"` + ImageID string `json:"image_id"` + Fs *string `json:"fs"` + Os string `json:"os"` + OSDistribution string `json:"os_distribution"` + OsVersion string `json:"os_version"` + Format string `json:"format"` + MountPoint *string `json:"mountpoint"` + Driver *string `json:"driver"` + Cache *string `json:"cache"` + ImageDiskFormat string `json:"image_disk_format"` + Size int64 `json:"size"` + Storage *string `json:"storage"` + Schedtags []Schedtag `json:"schedtags"` + Index int `json:"index"` } type Network struct { @@ -62,7 +64,7 @@ type ForGuest struct { Name string `json:"name"` } -type Aggregate struct { +type Schedtag struct { Idx string `json:"idx"` Strategy string `json:"strategy"` } @@ -78,8 +80,8 @@ type SchedData struct { Type string `json:"type"` IsContainer bool `json:"is_container"` Count int64 `json:"count"` + RegionID string `json:"region_id"` ZoneID string `json:"zone_id"` - PoolID string `json:"pool_id"` HostID string `json:"host_id"` Candidates []string `json:"candidates"` OwnerTenantID string `json:"owner_tenant_id"` @@ -90,7 +92,7 @@ type SchedData struct { Name string `json:"name"` Networks []*Network `json:"networks"` IsolatedDevices []*IsolatedDevice `json:"isolated_devices"` - Aggregates []Aggregate `json:"aggregate_stategy"` + Schedtags []Schedtag `json:"aggregate_stategy"` Meta Meta `json:"__meta__"` ForGuests []*ForGuest `json:"for_guests"` GuestStatus string `json:"guest_status"` @@ -132,9 +134,9 @@ func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, } } - if poolID, ok := sjson.CheckGet("prefer_pool_id"); ok { - if str, err := poolID.String(); err == nil { - data.PoolID = str + if regionID, ok := sjson.CheckGet("prefer_region_id"); ok { + if str, err := regionID.String(); err == nil { + data.RegionID = str } } @@ -231,7 +233,7 @@ func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, data.fillForGuests(sjson) - if err := data.fillAggregates(sjson, byTest); err != nil { + if err := data.fillSchedtags(sjson, byTest); err != nil { return nil, err } @@ -652,11 +654,12 @@ func (d *SchedData) fillDisksInfo(sjson *simplejson.Json, byTest bool) error { if !ok { break } - index++ disk, err := newDiskFromSimpleJson(d, byTest) if err != nil { return err } + disk.Index = index + index++ disks = append(disks, disk) } if index == 0 && d.Hypervisor != SchedTypeContainer { @@ -703,7 +706,7 @@ func (d *SchedData) fillForGuests(sjson *simplejson.Json) { } } -func NewSchedTagFromCmdline(str string) (agg Aggregate, err error) { +func NewSchedTagFromCmdline(str string) (agg Schedtag, err error) { rs := strings.Split(str, ":") if len(rs) == 1 || rs[1] == "" { err = fmt.Errorf("SchedTag %q no strategy.", str) @@ -711,22 +714,22 @@ func NewSchedTagFromCmdline(str string) (agg Aggregate, err error) { } name, strategy := rs[0], rs[1] - err = AggregateStrategyCheck(strategy) + err = SchedtagStrategyCheck(strategy) if err != nil { return } - agg = Aggregate{name, strategy} + agg = Schedtag{name, strategy} return } -func (d *SchedData) fillAggregates(sjson *simplejson.Json, byTest bool) error { - d.Aggregates = []Aggregate{} +func (d *SchedData) fillSchedtags(sjson *simplejson.Json, byTest bool) error { + d.Schedtags = []Schedtag{} if !byTest { if aggNode, ok := sjson.CheckGet("aggregate_strategy"); ok { for name, strategy := range aggNode.MustMap() { - d.Aggregates = append(d.Aggregates, Aggregate{ + d.Schedtags = append(d.Schedtags, Schedtag{ Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), }) } @@ -743,7 +746,7 @@ func (d *SchedData) fillAggregates(sjson *simplejson.Json, byTest bool) error { if err != nil { return err } - d.Aggregates = append(d.Aggregates, agg) + d.Schedtags = append(d.Schedtags, agg) index++ } } diff --git a/pkg/scheduler/api/types.go b/pkg/scheduler/api/types.go index 2a2cca01ce..2862157484 100644 --- a/pkg/scheduler/api/types.go +++ b/pkg/scheduler/api/types.go @@ -74,9 +74,14 @@ func init() { } } -func AggregateStrategyCheck(strategy string) (err error) { +func SchedtagStrategyCheck(strategy string) (err error) { if !AggregateStrategySets.Has(strategy) { err = fmt.Errorf("Strategy %q must in set %v", strategy, AggregateStrategySets.List()) } return } + +type CandidateStorage struct { + *models.SStorage + Schedtags []models.SSchedtag `json:"schedtags"` +} diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go index 8109ee0589..51dfc2b831 100644 --- a/pkg/scheduler/cache/candidate/baremetals.go +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -14,10 +14,12 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/cache/db" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/db/models" + + computemodels "yunion.io/x/onecloud/pkg/compute/models" ) type BaremetalDesc struct { - *baseHostDesc + *BaseHostDesc Storages []*baremetal.BaremetalStorage `json:"storages"` StorageType string `json:"storage_type"` @@ -47,7 +49,7 @@ func (bd *BaremetalDesc) Type() int { func (bd *BaremetalDesc) Get(key string) interface{} { switch key { case "ID": - return bd.ID + return bd.Id case "Name": return bd.Name @@ -55,17 +57,14 @@ func (bd *BaremetalDesc) Get(key string) interface{} { case "Status": return bd.Status - case "PoolID": - return bd.PoolID - case "ZoneID": - return bd.ZoneID + return bd.ZoneId case "ServerID": return bd.ServerID case "CPUCount": - return int64(bd.CPUCount) + return int64(bd.CpuCount) case "FreeCPUCount": return bd.FreeCPUCount() @@ -117,19 +116,19 @@ func (bd *BaremetalDesc) XGet(key string, kind core.Kind) interface{} { } func (bd *BaremetalDesc) IndexKey() string { - return bd.ID + return bd.Id } func (bd *BaremetalDesc) FreeCPUCount() int64 { if bd.ServerID == "" { - return bd.CPUCount + return int64(bd.CpuCount) } return 0 } func (bd *BaremetalDesc) FreeMemSize() int64 { if bd.ServerID == "" { - return bd.MemSize + return int64(bd.MemSize) } return 0 } @@ -190,6 +189,7 @@ func (bb *BaremetalBuilder) Type() string { } func (bb *BaremetalBuilder) AllIDs() ([]string, error) { + log.Errorf("=======get baremetal host ids") return models.AllBaremetalIDs() } @@ -220,12 +220,13 @@ func (bb *BaremetalBuilder) build() ([]interface{}, error) { } func (bb *BaremetalBuilder) buildOne(bm *models.Host) (interface{}, error) { - baseDesc, err := newBaseHostDesc(bm) + hostObj := computemodels.HostManager.FetchHostById(bm.ID) + baseDesc, err := newBaseHostDesc(hostObj) if err != nil { return nil, err } desc := &BaremetalDesc{ - baseHostDesc: baseDesc, + BaseHostDesc: baseDesc, } desc.StorageDriver = bm.StorageDriver diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go new file mode 100644 index 0000000000..fc7fc7aab2 --- /dev/null +++ b/pkg/scheduler/cache/candidate/base.go @@ -0,0 +1,246 @@ +package candidate + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/scheduler/core" + "yunion.io/x/onecloud/pkg/scheduler/db/models" + + computedb "yunion.io/x/onecloud/pkg/cloudcommon/db" + computemodels "yunion.io/x/onecloud/pkg/compute/models" +) + +type BaseHostDesc struct { + *computemodels.SHost + Region *computemodels.SCloudregion `json:"region"` + Zone *computemodels.SZone `json:"zone"` + Cloudprovider *computemodels.SCloudprovider `json:"cloudprovider"` + Networks []computemodels.SNetwork `json:"networks"` + Storages []*api.CandidateStorage `json:"storages"` + + Tenants map[string]int64 `json:"tenants"` + HostSchedtags []computemodels.SSchedtag `json:"schedtags"` +} + +type baseHostGetter struct { + h *BaseHostDesc +} + +func newBaseHostGetter(h *BaseHostDesc) *baseHostGetter { + return &baseHostGetter{h} +} + +func (b *BaseHostDesc) Getter() core.CandidatePropertyGetter { + return newBaseHostGetter(b) +} + +func (b baseHostGetter) Id() string { + return b.h.GetId() +} + +func (b baseHostGetter) Name() string { + return b.h.GetName() +} + +func (b baseHostGetter) Zone() *computemodels.SZone { + return b.h.Zone +} + +func (b baseHostGetter) Region() *computemodels.SCloudregion { + return b.h.Region +} + +func (b baseHostGetter) HostType() string { + return b.h.HostType +} + +func (b baseHostGetter) HostSchedtags() []computemodels.SSchedtag { + return b.h.HostSchedtags +} + +func (b baseHostGetter) Storages() []*api.CandidateStorage { + return b.h.Storages +} + +func reviseResourceType(resType string) string { + if resType == "" { + return computemodels.HostResourceTypeDefault + } + return resType +} + +func newBaseHostDesc(host *computemodels.SHost) (*BaseHostDesc, error) { + host.ResourceType = reviseResourceType(host.ResourceType) + desc := &BaseHostDesc{ + SHost: host, + } + + if err := desc.fillCloudProvider(host); err != nil { + return nil, fmt.Errorf("Fill cloudprovider info error: %v", err) + } + + if err := desc.fillNetworks(desc.Id); err != nil { + return nil, fmt.Errorf("Fill networks error: %v", err) + } + + if err := desc.fillZone(host); err != nil { + return nil, fmt.Errorf("Fill zone error: %v", err) + } + + if err := desc.fillRegion(host); err != nil { + return nil, fmt.Errorf("Fill region error: %v", err) + } + + if err := desc.fillResidentTenants(host); err != nil { + return nil, fmt.Errorf("Fill resident tenants error: %v", err) + } + + return desc, nil +} + +func (b BaseHostDesc) GetSchedDesc() *jsonutils.JSONDict { + desc := jsonutils.Marshal(b.SHost).(*jsonutils.JSONDict) + + if b.Cloudprovider != nil { + p := b.Cloudprovider + cloudproviderDesc := jsonutils.NewDict() + cloudproviderDesc.Add(jsonutils.NewString(p.ProjectId), "tenant_id") + cloudproviderDesc.Add(jsonutils.NewString(p.Provider), "provider") + desc.Add(cloudproviderDesc, "cloudprovider") + } + + return desc +} + +func (b BaseHostDesc) GetResourceType() string { + return b.ResourceType +} + +func (b *BaseHostDesc) fillCloudProvider(host *computemodels.SHost) error { + b.Cloudprovider = host.GetCloudprovider() + return nil +} + +func (b *BaseHostDesc) fillRegion(host *computemodels.SHost) error { + b.Region = host.GetRegion() + return nil +} + +func (b *BaseHostDesc) fillZone(host *computemodels.SHost) error { + zone := host.GetZone() + b.Zone = zone + b.ZoneId = host.ZoneId + return nil +} + +func (b *BaseHostDesc) fillResidentTenants(host *computemodels.SHost) error { + rets, err := HostResidentTenantCount(host.Id) + if err != nil { + return err + } + + b.Tenants = rets + + return nil +} + +func (b *BaseHostDesc) fillSchedtags(allSchedtags []computemodels.SSchedtag) error { + //b.Schedtags = allSchedtags + b.HostSchedtags = b.SHost.GetSchedtags() + return nil +} + +func (b *BaseHostDesc) fillNetworks(hostID string) error { + hostwires := computemodels.HostwireManager.Query().SubQuery() + sq := hostwires.Query(sqlchemy.DISTINCT("wire_id", hostwires.Field("wire_id"))).Equals("host_id", hostID) + networks := computemodels.NetworkManager.Query().SubQuery() + q := networks.Query().In("wire_id", sq) + + nets := make([]computemodels.SNetwork, 0) + err := computedb.FetchModelObjects(computemodels.NetworkManager, q, &nets) + + if err != nil { + return err + } + b.Networks = nets + return nil +} + +func (b *BaseHostDesc) fillStorages(storages []*computemodels.SStorage) error { + ss := make([]*api.CandidateStorage, 0) + for _, s := range storages { + ss = append(ss, &api.CandidateStorage{ + SStorage: s, + Schedtags: s.GetSchedtags(), + }) + } + b.Storages = ss + return nil +} + +func (h *BaseHostDesc) GetEnableStatus() string { + if h.Enabled { + return "enable" + } + return "disable" +} + +func (h *BaseHostDesc) GetHostType() string { + if h.HostType == api.HostTypeBaremetal && h.IsBaremetal { + return api.HostTypeBaremetal + } + return h.HostType +} + +func HostsResidentTenantStats(hostIDs []string) (map[string]map[string]interface{}, error) { + residentTenantStats, err := models.ResidentTenantsInHosts(hostIDs) + if err != nil { + return nil, err + } + stat3 := make([]utils.StatItem3, len(residentTenantStats)) + for i, item := range residentTenantStats { + stat3[i] = item + } + return utils.ToStatDict3(stat3) +} + +func HostResidentTenantCount(id string) (map[string]int64, error) { + residentTenantDict, err := HostsResidentTenantStats([]string{id}) + if err != nil { + return nil, err + } + tenantMap, ok := residentTenantDict[id] + if !ok { + log.V(10).Infof("Not found host ID: %s when fill resident tenants, may be no guests on it.", id) + return nil, nil + } + rets := make(map[string]int64, len(tenantMap)) + for tenantID, countObj := range tenantMap { + rets[tenantID] = countObj.(int64) + } + return rets, nil +} + +type DescBuilder struct { + dbGroupCache DBGroupCacher + syncGroupCache SyncGroupCacher + actor BuildActor +} + +func NewDescBuilder(db DBGroupCacher, sync SyncGroupCacher, act BuildActor) *DescBuilder { + return &DescBuilder{ + dbGroupCache: db, + syncGroupCache: sync, + actor: act, + } +} + +func (d *DescBuilder) Build(ids []string) ([]interface{}, error) { + return d.actor.Do(ids, d.dbGroupCache, d.syncGroupCache) +} diff --git a/pkg/scheduler/cache/candidate/common.go b/pkg/scheduler/cache/candidate/common.go new file mode 100644 index 0000000000..1a9f8b54bc --- /dev/null +++ b/pkg/scheduler/cache/candidate/common.go @@ -0,0 +1,50 @@ +package candidate + +import ( + //"yunion.io/x/log" + + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/compute/models" +) + +var ( + VMRunningStatus = sets.NewString( + models.VM_START_START, + models.VM_STARTING, + models.VM_RUNNING, + models.VM_STOP_FAILED, + models.VM_BLOCK_STREAM, + models.VM_UNKNOWN, + models.VM_BACKUP_STARTING, + ) + + VMCreatingStatus = sets.NewString( + models.VM_CREATE_NETWORK, + models.VM_CREATE_DISK, + models.VM_START_DEPLOY, + models.VM_DEPLOYING, + models.VM_BACKUP_CREATING, + models.VM_DEPLOYING_BACKUP, + ) +) + +func FetchGuestByHostIDs(ids []string) ([]models.SGuest, error) { + gs := make([]models.SGuest, 0) + q := models.GuestManager.Query().In("host_id", ids) + err := db.FetchModelObjects(models.GuestManager, q, &gs) + return gs, err +} + +func IsGuestRunning(g models.SGuest) bool { + return VMRunningStatus.Has(g.Status) +} + +func IsGuestCreating(g models.SGuest) bool { + return VMCreatingStatus.Has(g.Status) +} + +func IsGuestPendingDelete(g models.SGuest) bool { + return g.PendingDeleted +} diff --git a/pkg/scheduler/cache/candidate/default.go b/pkg/scheduler/cache/candidate/default.go index 7c9dc7e967..5ad5f1191e 100644 --- a/pkg/scheduler/cache/candidate/default.go +++ b/pkg/scheduler/cache/candidate/default.go @@ -9,9 +9,9 @@ import ( "yunion.io/x/log" u "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/cache" "yunion.io/x/onecloud/pkg/scheduler/db/models" + "yunion.io/x/onecloud/pkg/scheduler/options" ) const ( @@ -32,7 +32,7 @@ func defaultCadidateItems(db DBGroupCacher, sync SyncGroupCacher) []cache.Cached } func uuidKey(obj interface{}) (string, error) { - return obj.(descer).UUID(), nil + return obj.(descer).GetId(), nil } func generalUpdateFunc(db DBGroupCacher, sync SyncGroupCacher, act BuildActor, mutex *gosync.Mutex) cache.UpdateFunc { @@ -92,7 +92,10 @@ func generalGetUpdateFunc(isBaremetal bool) cache.GetUpdateFunc { for _, item := range d { r := reflect.ValueOf(item) f := reflect.Indirect(r) - key := f.FieldByName("ID") + key := f.FieldByName("Id") + if !key.IsValid() { + key = f.FieldByName("ID") + } value := f.FieldByName("UpdatedAt") if key.IsValid() && value.IsValid() { allStatus[key.String()] = value.Interface().(time.Time) diff --git a/pkg/scheduler/cache/candidate/desc.go b/pkg/scheduler/cache/candidate/desc.go deleted file mode 100644 index 2bc41db960..0000000000 --- a/pkg/scheduler/cache/candidate/desc.go +++ /dev/null @@ -1,260 +0,0 @@ -package candidate - -import ( - "fmt" - "time" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/pkg/utils" - - cloudmodels "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/scheduler/api" - "yunion.io/x/onecloud/pkg/scheduler/db/models" -) - -type baseDesc struct { - ID string `json:"id"` - Name string `json:"name"` - UpdatedAt time.Time `json:"updated_at"` -} - -func (b *baseDesc) UUID() string { - return b.ID -} - -type baseHostDesc struct { - baseDesc - models.BillingResourceBase - - ManagerID *string `json:"manager_id"` - Status string `json:"status"` - CPUCount int64 `json:"cpu_count"` - MemSize int64 `json:"mem_size"` - Networks []*models.NetworkSchedResult `json:"networks"` - HostStatus string `json:"host_status"` - Enabled bool `json:"enabled"` - HostType string `json:"host_type"` - IsBaremetal bool `json:"is_baremetal"` - IsMaintenance bool `json:"is_maintenance"` - NodeCount int64 `json:"node_count"` - Tenants map[string]int64 `json:"tenants"` - ZoneID string `json:"zone_id"` - Zone *models.Zone `json:"zone"` - PoolID string `json:"pool_id"` - ClusterID string `json:"cluster_id"` - Aggregates []*models.Aggregate `json:"aggregates"` - HostAggregates []*models.Aggregate `json:"host_aggregates"` - Cloudprovider *models.Cloudprovider `json:"cloudprovider"` - ResourceType string `json:"resource_type"` - RealExternalId string `json:"real_external_id"` -} - -func reviseResourceType(resType string) string { - if resType == "" { - return cloudmodels.HostResourceTypeDefault - } - return resType -} - -func newBaseHostDesc(host *models.Host) (*baseHostDesc, error) { - desc := &baseHostDesc{ - baseDesc: baseDesc{ - ID: host.ID, - Name: host.Name, - UpdatedAt: host.UpdatedAt, - }, - BillingResourceBase: host.BillingResourceBase, - ManagerID: host.ManagerID, - Status: host.Status, - CPUCount: host.CPUCount, - MemSize: host.MemSize, - HostStatus: host.HostStatus, - Enabled: host.Enabled, - HostType: host.HostType, - IsBaremetal: host.IsBaremetal, - NodeCount: host.NodeCount, - IsMaintenance: host.IsMaintenance, - ResourceType: reviseResourceType(host.ResourceType), - RealExternalId: host.RealExternalId, - } - - if err := desc.fillCloudProvider(host); err != nil { - return nil, fmt.Errorf("Fill cloudprovider info error: %v", err) - } - - if err := desc.fillNetworks(desc.ID); err != nil { - return nil, fmt.Errorf("Fill networks error: %v", err) - } - - if err := desc.fillZone(host); err != nil { - return nil, fmt.Errorf("Fill zone error: %v", err) - } - - if err := desc.fillResidentTenants(host); err != nil { - return nil, fmt.Errorf("Fill resident tenants error: %v", err) - } - - if err := desc.fillAggregates(); err != nil { - return nil, fmt.Errorf("Fill schetags error: %v", err) - } - return desc, nil -} - -func (b baseHostDesc) GetSchedDesc() *jsonutils.JSONDict { - desc := jsonutils.NewDict() - - desc.Add(jsonutils.NewString(b.ID), "id") - desc.Add(jsonutils.NewString(b.Name), "name") - desc.Add(jsonutils.NewInt(b.CPUCount), "cpu_count") - desc.Add(jsonutils.NewInt(b.MemSize), "mem_size") - desc.Add(jsonutils.NewString(b.HostType), "host_type") - desc.Add(jsonutils.NewString(b.ZoneID), "zone_id") - desc.Add(jsonutils.NewString(b.Zone.Name), "zone") - desc.Add(jsonutils.NewString(b.ResourceType), "resource_type") - - if b.Cloudprovider != nil { - p := b.Cloudprovider - cloudproviderDesc := jsonutils.NewDict() - cloudproviderDesc.Add(jsonutils.NewString(p.ProjectId), "tenant_id") - cloudproviderDesc.Add(jsonutils.NewString(p.Provider), "provider") - desc.Add(cloudproviderDesc, "cloudprovider") - } - - return desc -} - -func (b baseHostDesc) GetResourceType() string { - return b.ResourceType -} - -func (b *baseHostDesc) fillCloudProvider(host *models.Host) error { - if host.ManagerID == nil { - log.Debugf("Host %q manager id is empty, no cloud provider", host.Name) - return nil - } - provider, err := models.FetchCloudproviderById(*(host.ManagerID)) - if err != nil { - return err - } - b.Cloudprovider = provider - return nil -} - -func (b *baseHostDesc) fillZone(host *models.Host) error { - zone, err := models.FetchZoneByID(host.ZoneID) - if err != nil { - return err - } - b.Zone = zone - b.ZoneID = host.ZoneID - return nil -} - -func (b *baseHostDesc) fillResidentTenants(host *models.Host) error { - rets, err := HostResidentTenantCount(host.ID) - if err != nil { - return err - } - - b.Tenants = rets - - return nil -} - -func (b *baseHostDesc) fillAggregates() error { - b.Aggregates = make([]*models.Aggregate, 0) - objs, err := models.All(models.Aggregates) - if err != nil { - return err - } - for _, obj := range objs { - agg := obj.(*models.Aggregate) - b.Aggregates = append(b.Aggregates, agg) - } - - aggs, err := models.HostAggregates(b.ID) - if err != nil { - return err - } - b.HostAggregates = aggs - return nil -} - -func (b *baseHostDesc) GetAggregates() []*models.Aggregate { - return b.Aggregates -} - -func (b *baseHostDesc) GetHostAggregates() []*models.Aggregate { - return b.HostAggregates -} - -func (b *baseHostDesc) fillNetworks(hostID string) error { - net, err := models.HostNetworkSchedResults(hostID) - if err != nil { - return err - } - b.Networks = net - return nil -} - -func (h *baseHostDesc) GetEnableStatus() string { - if h.Enabled { - return "enable" - } - return "disable" -} - -func (h *baseHostDesc) GetHostType() string { - if h.HostType == api.HostTypeBaremetal && h.IsBaremetal { - return api.HostTypeBaremetal - } - return h.HostType -} - -func HostsResidentTenantStats(hostIDs []string) (map[string]map[string]interface{}, error) { - residentTenantStats, err := models.ResidentTenantsInHosts(hostIDs) - if err != nil { - return nil, err - } - stat3 := make([]utils.StatItem3, len(residentTenantStats)) - for i, item := range residentTenantStats { - stat3[i] = item - } - return utils.ToStatDict3(stat3) -} - -func HostResidentTenantCount(id string) (map[string]int64, error) { - residentTenantDict, err := HostsResidentTenantStats([]string{id}) - if err != nil { - return nil, err - } - tenantMap, ok := residentTenantDict[id] - if !ok { - log.V(10).Infof("Not found host ID: %s when fill resident tenants, may be no guests on it.", id) - return nil, nil - } - rets := make(map[string]int64, len(tenantMap)) - for tenantID, countObj := range tenantMap { - rets[tenantID] = countObj.(int64) - } - return rets, nil -} - -type DescBuilder struct { - dbGroupCache DBGroupCacher - syncGroupCache SyncGroupCacher - actor BuildActor -} - -func NewDescBuilder(db DBGroupCacher, sync SyncGroupCacher, act BuildActor) *DescBuilder { - return &DescBuilder{ - dbGroupCache: db, - syncGroupCache: sync, - actor: act, - } -} - -func (d *DescBuilder) Build(ids []string) ([]interface{}, error) { - return d.actor.Do(ids, d.dbGroupCache, d.syncGroupCache) -} diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 7b3b331698..32b9399530 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -13,23 +13,20 @@ import ( "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/util/workqueue" "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" - o "yunion.io/x/onecloud/cmd/scheduler/options" - "yunion.io/x/onecloud/pkg/scheduler/cache" - "yunion.io/x/onecloud/pkg/scheduler/cache/db" + computedb "yunion.io/x/onecloud/pkg/cloudcommon/db" + computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/db/models" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type HostDesc struct { - *baseHostDesc + *BaseHostDesc // cpu - CPUMHZ int64 `json:"cpu_mhz"` - CPUCmtbound float64 `json:"cpu_cmtbound"` - CPUDesc string `json:"cpu_desc"` - CPUCache int64 `json:"cpu_cache"` - CPUReserved int64 `json:"cpu_reserved"` + CPUCmtbound float32 `json:"cpu_cmtbound"` CPUBoundCount int64 `json:"cpu_bound_count"` CPULoad *float64 `json:"cpu_load"` TotalCPUCount int64 `json:"total_cpu_count"` @@ -40,8 +37,7 @@ type HostDesc struct { FreeCPUCount int64 `json:"free_cpu_count"` // memory - MemCmtbound float64 `json:"mem_cmtbound"` - MemReserved int64 `json:"mem_reserved"` + MemCmtbound float32 `json:"mem_cmtbound"` TotalMemSize int64 `json:"total_mem_size"` FreeMemSize int64 `json:"free_mem_size"` RunningMemSize int64 `json:"running_mem_size"` @@ -50,8 +46,7 @@ type HostDesc struct { FakeDeletedMemSize int64 `json:"fake_deleted_mem_size"` // storage - Storages []*Storage `json:"storages"` - StorageTypes []string `json:"storage_types"` + StorageTypes []string `json:"storage_types"` // IO IOBoundCount int64 `json:"io_bound_count"` @@ -96,10 +91,10 @@ func StorageIsolatedDevReservedSize() int64 { return o.GetOptions().StorageReservedPerIsolatedDevice } -func NewGuestReservedResourceByBuilder(b *HostBuilder, host *models.Host) (ret *ReservedResource) { +func NewGuestReservedResourceByBuilder(b *HostBuilder, host *computemodels.SHost) (ret *ReservedResource) { ret = NewReservedResource(0, 0, 0) //isoDevs := b.getUnusedIsolatedDevices(host.ID) - isoDevs := b.getIsolatedDevices(host.ID) + isoDevs := b.getIsolatedDevices(host.Id) hostDevsCount := int64(len(isoDevs)) if hostDevsCount == 0 { return @@ -115,9 +110,9 @@ func NewGuestReservedResourceByBuilder(b *HostBuilder, host *models.Host) (ret * return } -func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *models.Host) (ret *ReservedResource, err error) { +func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *computemodels.SHost) (ret *ReservedResource, err error) { ret = NewReservedResource(0, 0, 0) - gst := b.getIsolatedDeviceGuests(host.ID) + gst := b.getIsolatedDeviceGuests(host.Id) if len(gst) == 0 { return } @@ -133,7 +128,7 @@ func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *models.Host) (r return } disk += dSize - if o.GetOptions().IgnoreNonRunningGuests && !g.IsRunning() { + if o.GetOptions().IgnoreNonrunningGuests && !g.IsRunning() { continue } cpu += g.VCPUCount @@ -145,54 +140,51 @@ func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *models.Host) (r return } -type Storage struct { - ID string `json:"id"` - Name string `json:"name"` - Capacity int64 `json:"capacity"` - StorageType string `json:"type"` - UsedCapacity int64 `json:"used"` - WasteCapacity int64 `json:"waste"` - FreeCapacity int64 `json:"free"` - VCapacity int64 `json:"vcapacity"` - Cmtbound float64 `json:"cmtbound"` - StorageDriver string `json:"driver"` - Adapter string `json:"adapter"` - Splits []string `json:"splits"` - Range string `json:"range"` - Conf string `json:"conf"` - MinStripSize int `json:"min_strip_size"` - MaxStripSize int `json:"max_strip_size"` - Size int `json:"size"` -} +//type Storage struct { +//ID string `json:"id"` +//Name string `json:"name"` +//Capacity int64 `json:"capacity"` +//StorageType string `json:"type"` +//UsedCapacity int64 `json:"used"` +//WasteCapacity int64 `json:"waste"` +//FreeCapacity int64 `json:"free"` +//VCapacity int64 `json:"vcapacity"` +//Cmtbound float64 `json:"cmtbound"` +//StorageDriver string `json:"driver"` +//Adapter string `json:"adapter"` +//Splits []string `json:"splits"` +//Range string `json:"range"` +//Conf string `json:"conf"` +//MinStripSize int `json:"min_strip_size"` +//MaxStripSize int `json:"max_strip_size"` +//Size int `json:"size"` +//} -func (storage *Storage) GetFreeSize() int64 { - return storage.GetTotalSize() - storage.UsedCapacity - storage.WasteCapacity -} +//func (storage *Storage) GetFreeSize() int64 { +//return storage.GetTotalSize() - storage.UsedCapacity - storage.WasteCapacity +//} -func (storage *Storage) GetTotalSize() int64 { - return int64(float64(storage.Capacity) * storage.Cmtbound) -} +//func (storage *Storage) GetTotalSize() int64 { +//return int64(float64(storage.Capacity) * storage.Cmtbound) +//} -func (storage *Storage) IsLocal() bool { - return utils.IsLocalStorage(storage.StorageType) -} +//func (storage *Storage) IsLocal() bool { +//return utils.IsLocalStorage(storage.StorageType) +//} type HostBuilder struct { - clusters cache.Cache - residentTenantDict map[string]map[string]interface{} - hosts []interface{} + hosts []computemodels.SHost hostDict map[string]interface{} - guests []interface{} + guests []computemodels.SGuest guestDict map[string]interface{} guestIDs []string - hostStorages []interface{} - hostStoragesDict map[string][]interface{} + hostStorages []computemodels.SHoststorage + hostStoragesDict map[string][]*computemodels.SStorage storages []interface{} - storageDict map[string]interface{} storageStatesSizeDict map[string]map[string]interface{} hostGuests map[string][]interface{} @@ -212,6 +204,8 @@ type HostBuilder struct { isolatedDevicesDict map[string][]interface{} cpuIOLoads map[string]map[string]float64 + + schedtags []computemodels.SSchedtag } func (h *HostDesc) String() string { @@ -231,25 +225,19 @@ func (h *HostDesc) GetGuestCount() int64 { func (h *HostDesc) Get(key string) interface{} { switch key { case "ID": - return h.ID + return h.Id case "Name": return h.Name case "CPUCount": - return h.CPUCount + return h.CpuCount case "MemSize": return h.MemSize - case "PoolID": - return h.PoolID - case "ZoneID": - return h.ZoneID - - case "ClusterID": - return h.ClusterID + return h.ZoneId case "TotalCPUCount": return h.GetTotalCPUCount(true) @@ -332,7 +320,7 @@ func (h *HostDesc) totalStorageSize(onlyLocal, useRsvd bool) int64 { total := int64(0) for _, storage := range h.Storages { if !onlyLocal || storage.IsLocal() { - total += storage.GetTotalSize() + total += int64(storage.GetCapacity()) } } @@ -346,7 +334,7 @@ func (h *HostDesc) freeStorageSize(onlyLocal, useRsvd bool) int64 { total := int64(0) for _, storage := range h.Storages { if !onlyLocal || storage.IsLocal() { - total += storage.GetFreeSize() + total += int64(storage.GetFreeCapacity()) } } @@ -370,7 +358,7 @@ func (h *HostDesc) freeStorageSizeOfType(storageType string, useRsvd bool) int64 total := int64(0) for _, storage := range h.Storages { if storage.StorageType == storageType { - total += storage.GetFreeSize() + total += int64(storage.GetFreeCapacity()) } } if utils.IsLocalStorage(storageType) { @@ -431,11 +419,11 @@ func (h *HostDesc) GuestReservedStorageSizeFree() int64 { } func (h *HostDesc) GetReservedMemSize() int64 { - return h.GuestReservedResource.MemorySize + h.MemReserved + return h.GuestReservedResource.MemorySize + int64(h.MemReserved) } func (h *HostDesc) GetReservedCPUCount() int64 { - return h.GuestReservedResource.CPUCount + h.CPUReserved + return h.GuestReservedResource.CPUCount + int64(h.CpuReserved) } func (h *HostDesc) GetReservedStorageSize() int64 { @@ -451,7 +439,7 @@ func (h *HostDesc) GetFreeCPUCount(useRsvd bool) int64 { } func (h *HostDesc) IndexKey() string { - return h.ID + return h.Id } func (h *HostDesc) UnusedIsolatedDevices() []*IsolatedDeviceDesc { @@ -544,8 +532,8 @@ func (b *HostBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGr errMessageChannel := make(chan error, 12) defer close(errMessageChannel) setFuncs := []func(){ - func() { b.setClusters(dbCache, errMessageChannel) }, func() { b.setHosts(ids, errMessageChannel) }, + func() { b.setSchedtags(ids, errMessageChannel) }, func() { b.setGuests(ids, errMessageChannel) b.setGroupInfo(errMessageChannel) @@ -582,88 +570,70 @@ func (b *HostBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGr return nil } -func (b *HostBuilder) setClusters(dbCache DBGroupCacher, errMessageChannel chan error) { - clusters, err := dbCache.Get(db.ClusterDBCache) - if err != nil { - errMessageChannel <- err - return - } - b.clusters = clusters - return -} - func (b *HostBuilder) setHosts(ids []string, errMessageChannel chan error) { - hosts, err := models.FetchHypervisorHostByIDs(ids) + hosts := computemodels.HostManager.Query() + q := hosts.In("id", ids).NotEquals("host_type", computemodels.HOST_TYPE_BAREMETAL) + hostObjs := make([]computemodels.SHost, 0) + err := computedb.FetchModelObjects(computemodels.HostManager, q, &hostObjs) if err != nil { errMessageChannel <- err return } - hostDict, err := utils.ToDict(hosts, func(obj interface{}) (string, error) { - host, ok := obj.(*models.Host) + + hostDict, err := utils.ToDict(hostObjs, func(obj interface{}) (string, error) { + host, ok := obj.(computemodels.SHost) if !ok { - return "", utils.ConvertError(obj, "*models.Host") + return "", utils.ConvertError(obj, "computemodels.Host") } - return host.ID, nil + return host.Id, nil }) if err != nil { errMessageChannel <- err return } - b.hosts = hosts + b.hosts = hostObjs b.hostDict = hostDict return } +func (b *HostBuilder) setSchedtags(ids []string, errMessageChannel chan error) { + tags := make([]computemodels.SSchedtag, 0) + if err := computemodels.SchedtagManager.Query().All(&tags); err != nil { + errMessageChannel <- err + return + } + b.schedtags = tags +} + func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { - hostStorages, err := models.FetchByHostIDs(models.HostStorages, ids) - if err != nil { - errMessageChannel <- err - return - } - storageIDs := make([]string, len(hostStorages)) - func() { - for i, s := range hostStorages { - storageIDs[i] = s.(*models.HostStorage).StorageID - } - }() - storages, err := models.FetchByIDs(models.Storages, storageIDs) + q := computemodels.HoststorageManager.Query().In("host_id", ids) + hostStorages := make([]computemodels.SHoststorage, 0) + err := computedb.FetchModelObjects(computemodels.HoststorageManager, q, &hostStorages) if err != nil { errMessageChannel <- err return } - hostStoragesDict, err := utils.GroupBy(hostStorages, func(obj interface{}) (string, error) { - storage, ok := obj.(*models.HostStorage) - if !ok { - return "", utils.ConvertError(obj, "*models.HostStorage") - } - return storage.HostID, nil - }) - if err != nil { - errMessageChannel <- err - return - } - storageDict, err := utils.ToDict(storages, func(obj interface{}) (string, error) { - storage, ok := obj.(*models.Storage) - if !ok { - return "", utils.ConvertError(obj, "*models.Storage") - } - return storage.ID, nil - }) + hostStoragesDict := make(map[string][]*computemodels.SStorage) - if err != nil { - errMessageChannel <- err - return + for _, s := range hostStorages { + if ss, ok := hostStoragesDict[s.HostId]; !ok { + storage := s.GetStorage() + ss = make([]*computemodels.SStorage, 0) + ss = append(ss, storage) + hostStoragesDict[s.HostId] = ss + } else { + ss = append(ss, s.GetStorage()) + } } + b.hostStorages = hostStorages b.hostStoragesDict = hostStoragesDict - b.storages = storages - b.storageDict = storageDict return } func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { - guests, err := models.FetchGuestByHostIDs(ids) + guests, err := FetchGuestByHostIDs(ids) if err != nil { errMessageChannel <- err return @@ -671,16 +641,16 @@ func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { guestIDs := make([]string, len(guests)) func() { for i, gst := range guests { - guestIDs[i] = gst.(*models.Guest).ID + guestIDs[i] = gst.GetId() } }() hostGuests, err := utils.GroupBy(guests, func(obj interface{}) (string, error) { - gst, ok := obj.(*models.Guest) + gst, ok := obj.(computemodels.SGuest) if !ok { - return "", utils.ConvertError(obj, "*models.Guest") + return "", utils.ConvertError(obj, "computemodels.SGuest") } - return gst.HostID, nil + return gst.HostId, nil }) if err != nil { errMessageChannel <- err @@ -688,11 +658,11 @@ func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { } hostBackupGuests, err := utils.GroupBy(guests, func(obj interface{}) (string, error) { - gst, ok := obj.(*models.Guest) + gst, ok := obj.(computemodels.SGuest) if !ok { - return "", utils.ConvertError(obj, "*models.Guest") + return "", utils.ConvertError(obj, "computemodels.SGuest") } - return gst.BackupHostID, nil + return gst.BackupHostId, nil }) if err != nil { errMessageChannel <- err @@ -700,11 +670,11 @@ func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { } guestDict, err := utils.ToDict(guests, func(obj interface{}) (string, error) { - gst, ok := obj.(*models.Guest) + gst, ok := obj.(computemodels.SGuest) if !ok { - return "", utils.ConvertError(obj, "*models.Guest") + return "", utils.ConvertError(obj, "computemodels.SGuest") } - return gst.ID, nil + return gst.GetId(), nil }) if err != nil { errMessageChannel <- err @@ -913,7 +883,23 @@ func (b *HostBuilder) Type() string { } func (b *HostBuilder) AllIDs() ([]string, error) { - return models.AllHostIDs() + q := computemodels.HostManager.Query("id") + q = q.Filter(sqlchemy.NotEquals(q.Field("host_type"), computemodels.HOST_TYPE_BAREMETAL)) + rs, err := q.Rows() + if err != nil { + return nil, err + } + ret := []string{} + defer rs.Close() + for rs.Next() { + var id string + if err := rs.Scan(&id); err != nil { + return nil, err + } + ret = append(ret, id) + } + log.Errorf("==============================\n ============================get hostsIDs: %v", ret) + return ret, nil } func (b *HostBuilder) Do(ids []string, dbCache DBGroupCacher, syncCache SyncGroupCacher) ([]interface{}, error) { @@ -940,7 +926,7 @@ func (b *HostBuilder) build() ([]interface{}, error) { return } host := b.hosts[i] - desc, err := b.buildOne(host.(*models.Host)) + desc, err := b.buildOne(&host) if err != nil { descResultLock.Lock() errs = append(errs, err) @@ -963,27 +949,19 @@ func (b *HostBuilder) build() ([]interface{}, error) { return schedDescs, nil } -func (b *HostBuilder) buildOne(host *models.Host) (interface{}, error) { +func (b *HostBuilder) buildOne(host *computemodels.SHost) (interface{}, error) { baseDesc, err := newBaseHostDesc(host) if err != nil { return nil, err } desc := &HostDesc{ - baseHostDesc: baseDesc, + BaseHostDesc: baseDesc, } desc.Metadata = make(map[string]string) - desc.ManagerID = host.ManagerID - desc.CPUCmtbound = host.CPUOverCommitBound() - desc.CPUDesc = host.CPUDesc - desc.CPUCache = host.CPUCache - desc.CPUReserved = host.CPUReserved - desc.NodeCount = host.NodeCount - desc.CPUMHZ = host.CPUMHZ - - desc.MemCmtbound = host.MemOverCommitBound() - desc.MemReserved = host.MemReserved + desc.CPUCmtbound = host.GetCPUOvercommitBound() + desc.MemCmtbound = host.GetMemoryOvercommitBound() desc.GuestReservedResource = NewGuestReservedResourceByBuilder(b, host) guestRsvdUsed, err := NewGuestReservedResourceUsedByBuilder(b, host) @@ -992,9 +970,10 @@ func (b *HostBuilder) buildOne(host *models.Host) (interface{}, error) { } desc.GuestReservedResourceUsed = guestRsvdUsed - fillFuncs := []func(*HostDesc, *models.Host) error{ + fillFuncs := []func(*HostDesc, *computemodels.SHost) error{ b.fillGuestsResourceInfo, b.fillStorages, + b.fillSchedtags, b.fillResidentGroups, b.fillMetadata, b.fillIsolatedDevices, @@ -1020,7 +999,7 @@ func _in(s string, ss []string) bool { return false } -func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *models.Host) error { +func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *computemodels.SHost) error { var ( guestCount int64 runningCount int64 @@ -1036,37 +1015,37 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *models.Host) creatingCPUCount int64 creatingGuestCount int64 ) - guestsOnHost, ok := b.hostGuests[host.ID] + guestsOnHost, ok := b.hostGuests[host.Id] if !ok { guestsOnHost = []interface{}{} } - backupGuestsOnHost, ok := b.hostBackupGuests[host.ID] + backupGuestsOnHost, ok := b.hostBackupGuests[host.Id] if ok { guestsOnHost = append(guestsOnHost, backupGuestsOnHost...) } for _, gst := range guestsOnHost { - guest := gst.(*models.Guest) - if guest.IsRunning() { + guest := gst.(computemodels.SGuest) + if IsGuestRunning(guest) { runningCount++ - memSize += guest.VMemSize - cpuCount += guest.VCPUCount - } else if guest.IsCreating() { + memSize += int64(guest.VmemSize) + cpuCount += int64(guest.VcpuCount) + } else if IsGuestCreating(guest) { creatingGuestCount++ - creatingMemSize += guest.VMemSize - creatingCPUCount += guest.VCPUCount - } else if guest.IsGuestFakeDeleted() && _in(guest.Status, []string{models.VmReady}) { - memFakeDeletedSize += guest.VMemSize - cpuFakeDeletedCount += guest.VCPUCount + creatingMemSize += int64(guest.VmemSize) + creatingCPUCount += int64(guest.VcpuCount) + } else if IsGuestPendingDelete(guest) { + memFakeDeletedSize += int64(guest.VmemSize) + cpuFakeDeletedCount += int64(guest.VcpuCount) } guestCount++ - cpuReqCount += guest.VCPUCount - memReqSize += guest.VMemSize + cpuReqCount += int64(guest.VcpuCount) + memReqSize += int64(guest.VmemSize) appTags := b.guestAppTags(guest) for _, tag := range appTags { if tag == "cpu_bound" { - cpuBoundCount += guest.VCPUCount + cpuBoundCount += int64(guest.VcpuCount) } else if tag == "io_bound" { ioBoundCount++ } @@ -1084,12 +1063,12 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *models.Host) desc.CreatingCPUCount = creatingCPUCount desc.FakeDeletedCPUCount = cpuFakeDeletedCount - desc.TotalMemSize = int64(float64(desc.MemSize) * desc.MemCmtbound) - desc.TotalCPUCount = int64(float64(desc.CPUCount) * desc.CPUCmtbound) + desc.TotalMemSize = int64(float32(desc.MemSize) * desc.MemCmtbound) + desc.TotalCPUCount = int64(float32(desc.CpuCount) * desc.CPUCmtbound) var memFreeSize int64 var cpuFreeCount int64 - if o.GetOptions().IgnoreNonRunningGuests { + if o.GetOptions().IgnoreNonrunningGuests { memFreeSize = desc.TotalMemSize - desc.RunningMemSize - desc.CreatingMemSize cpuFreeCount = desc.TotalCPUCount - desc.RunningCPUCount - desc.CreatingCPUCount } else { @@ -1125,8 +1104,8 @@ func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *models.Host) return nil } -func (b *HostBuilder) guestAppTags(guest *models.Guest) []string { - metadatas, ok := b.guestMetadatasDict[guest.ID] +func (b *HostBuilder) guestAppTags(guest computemodels.SGuest) []string { + metadatas, ok := b.guestMetadatasDict[guest.GetId()] if !ok { return []string{} } @@ -1146,50 +1125,12 @@ func (b *HostBuilder) guestAppTags(guest *models.Guest) []string { return []string{} } -func (b *HostBuilder) fillStorages(desc *HostDesc, host *models.Host) error { - objs, ok := b.hostStoragesDict[host.ID] - if !ok { - return nil - } +func (b *HostBuilder) fillStorages(desc *HostDesc, host *computemodels.SHost) error { + return desc.fillStorages(b.hostStoragesDict[host.GetId()]) +} - var ( - rets = make([]*Storage, 0) - storageTypeMap = make(map[string]int, 0) - ) - for _, obj := range objs { - hostStorage, ok := obj.(*models.HostStorage) - if !ok { - return utils.ConvertError(obj, "*models.HostStorage") - } - storageID := hostStorage.StorageID - storageObj, ok := b.storageDict[storageID] - if !ok { - log.Warningf("Storage ID: %q not found when fill it", storageID) - return nil - } - storageModel := storageObj.(*models.Storage) - storage := new(Storage) - storage.ID = storageModel.ID - storage.Name = storageModel.Name - storage.Capacity = storageModel.Capacity - storage.StorageType = storageModel.StorageType - storage.UsedCapacity = b.storageUsedCapacity(storageModel, true) - storage.WasteCapacity = b.storageUsedCapacity(storageModel, false) - storage.Cmtbound = storageModel.OverCommitBound() - storage.VCapacity = storage.GetTotalSize() - storage.FreeCapacity = storage.GetFreeSize() - rets = append(rets, storage) - - storageTypeMap[storage.StorageType] = 0 - } - - desc.Storages = rets - - for storageType := range storageTypeMap { - desc.StorageTypes = append(desc.StorageTypes, storageType) - } - - return nil +func (b *HostBuilder) fillSchedtags(desc *HostDesc, host *computemodels.SHost) error { + return desc.fillSchedtags(b.schedtags) } func (b *HostBuilder) storageUsedCapacity(storage *models.Storage, ready bool) int64 { @@ -1213,8 +1154,8 @@ func (b *HostBuilder) storageUsedCapacity(storage *models.Storage, ready bool) i return total } -func (b *HostBuilder) fillResidentGroups(desc *HostDesc, host *models.Host) error { - groups, ok := b.hostGroupCountDict[host.ID] +func (b *HostBuilder) fillResidentGroups(desc *HostDesc, host *computemodels.SHost) error { + groups, ok := b.hostGroupCountDict[host.Id] if !ok { desc.Groups = nil return nil @@ -1223,8 +1164,8 @@ func (b *HostBuilder) fillResidentGroups(desc *HostDesc, host *models.Host) erro return nil } -func (b *HostBuilder) fillMetadata(desc *HostDesc, host *models.Host) error { - metadataObjs, ok := b.hostMetadatasDict[host.ID] +func (b *HostBuilder) fillMetadata(desc *HostDesc, host *computemodels.SHost) error { + metadataObjs, ok := b.hostMetadatasDict[host.Id] if !ok { return nil } @@ -1310,9 +1251,9 @@ func (b *HostBuilder) getUnusedIsolatedDevices(hostID string) (devs []*models.Is return } -func (b *HostBuilder) fillIsolatedDevices(desc *HostDesc, host *models.Host) error { +func (b *HostBuilder) fillIsolatedDevices(desc *HostDesc, host *computemodels.SHost) error { - allDevs := b.getIsolatedDevices(host.ID) + allDevs := b.getIsolatedDevices(host.Id) if len(allDevs) == 0 { return nil } @@ -1335,9 +1276,9 @@ func (b *HostBuilder) fillIsolatedDevices(desc *HostDesc, host *models.Host) err return nil } -func (b *HostBuilder) fillCPUIOLoads(desc *HostDesc, host *models.Host) error { - desc.CPULoad = b.loadByName(host.ID, "cpu_load") - desc.IOLoad = b.loadByName(host.ID, "io_load") +func (b *HostBuilder) fillCPUIOLoads(desc *HostDesc, host *computemodels.SHost) error { + desc.CPULoad = b.loadByName(host.Id, "cpu_load") + desc.IOLoad = b.loadByName(host.Id, "io_load") return nil } diff --git a/pkg/scheduler/cache/candidate/interface.go b/pkg/scheduler/cache/candidate/interface.go index f8fde13670..0335fe7a90 100644 --- a/pkg/scheduler/cache/candidate/interface.go +++ b/pkg/scheduler/cache/candidate/interface.go @@ -17,7 +17,7 @@ type SyncGroupCacher interface { } type descer interface { - UUID() string + GetId() string } type BuildActor interface { diff --git a/pkg/scheduler/cache/db/default.go b/pkg/scheduler/cache/db/default.go index bf0e208ebb..e50c16016d 100644 --- a/pkg/scheduler/cache/db/default.go +++ b/pkg/scheduler/cache/db/default.go @@ -5,9 +5,9 @@ import ( u "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/cache" "yunion.io/x/onecloud/pkg/scheduler/db/models" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) const ( diff --git a/pkg/scheduler/cache/sync/default.go b/pkg/scheduler/cache/sync/default.go index 49627d8fd7..490c8c8141 100644 --- a/pkg/scheduler/cache/sync/default.go +++ b/pkg/scheduler/cache/sync/default.go @@ -3,9 +3,9 @@ package sync import ( u "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/cache" networks_db "yunion.io/x/onecloud/pkg/scheduler/cache/sync/networks/db" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) const ( diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index 0ec66661f8..053d0e4852 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -364,6 +364,8 @@ type Unit struct { selectPlugins []SelectPlugin LogManager *SchedLogManager + + AllocatedResources map[string]*AllocatedResource } func NewScheduleUnit(info *api.SchedInfo, schedManager interface{}) *Unit { @@ -382,6 +384,7 @@ func NewScheduleUnit(info *api.SchedInfo, schedManager interface{}) *Unit { CounterManager: NewCounterManager(), LogManager: NewSchedLogManager(), SchedulerManager: schedManager, + AllocatedResources: make(map[string]*AllocatedResource), } return unit } @@ -630,3 +633,12 @@ func (u *Unit) GetFiltedData(id string, count int64) map[string]interface{} { return nil } + +func (u *Unit) GetAllocatedResource(candidateId string) *AllocatedResource { + ret, ok := u.AllocatedResources[candidateId] + if !ok { + ret = NewAllocatedResource() + u.AllocatedResources[candidateId] = ret + } + return ret +} diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 97709c5d73..0dc59410c3 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -15,7 +15,7 @@ import ( utiltrace "yunion.io/x/pkg/util/trace" "yunion.io/x/pkg/util/workqueue" - o "yunion.io/x/onecloud/cmd/scheduler/options" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) const ( @@ -166,13 +166,14 @@ func newSchedResultByCtx(u *Unit, count int64, c Candidater) *SchedResultItem { showDetails := u.SchedInfo.ShowSuggestionDetails id := c.IndexKey() r := &SchedResultItem{ - ID: id, - Count: count, - Capacity: u.GetCapacity(id), - Name: fmt.Sprintf("%v", c.Get("Name")), - Score: u.GetScore(id).DigitString(), - Data: u.GetFiltedData(id, count), - Candidater: c, + ID: id, + Count: count, + Capacity: u.GetCapacity(id), + Name: fmt.Sprintf("%v", c.Get("Name")), + Score: u.GetScore(id).DigitString(), + Data: u.GetFiltedData(id, count), + Candidater: c, + AllocatedResource: u.GetAllocatedResource(id), } if showDetails { @@ -237,6 +238,8 @@ type SchedResultItem struct { ScoreDetails string `json:"score_details"` Candidater Candidater `json:"-"` + + *AllocatedResource } func GetCapacities(u *Unit, id string) (res map[string]int64) { diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 7deb8724b4..ab420bcaef 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -3,8 +3,9 @@ package core import ( "yunion.io/x/jsonutils" + computemodels "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core/score" - "yunion.io/x/onecloud/pkg/scheduler/db/models" ) const ( @@ -35,8 +36,19 @@ const ( KindReserved ) +type CandidatePropertyGetter interface { + Id() string + Name() string + Zone() *computemodels.SZone + Region() *computemodels.SCloudregion + HostType() string + HostSchedtags() []computemodels.SSchedtag + Storages() []*api.CandidateStorage +} + // Candidater replace host Candidate resource info type Candidater interface { + Getter() CandidatePropertyGetter // IndexKey return candidate cache item's ident, usually host ID IndexKey() string // Get return candidate cache item's value by key @@ -45,9 +57,6 @@ type Candidater interface { XGet(key string, kind Kind) interface{} Type() int - GetAggregates() []*models.Aggregate - GetHostAggregates() []*models.Aggregate - GetSchedDesc() *jsonutils.JSONDict GetGuestCount() int64 GetResourceType() string @@ -119,3 +128,18 @@ type Priority interface { // Score intervals ScoreIntervals() score.Intervals } + +type DiskAllocatedResource struct { + Index int `json:"index"` + StorageId string `json:"storage_id"` +} + +type AllocatedResource struct { + Disks []*DiskAllocatedResource `json:"disks"` +} + +func NewAllocatedResource() *AllocatedResource { + return &AllocatedResource{ + Disks: make([]*DiskAllocatedResource, 0), + } +} diff --git a/pkg/scheduler/data_manager/candidate_manager.go b/pkg/scheduler/data_manager/candidate_manager.go index 57e8afaaf2..36dd18637d 100644 --- a/pkg/scheduler/data_manager/candidate_manager.go +++ b/pkg/scheduler/data_manager/candidate_manager.go @@ -14,10 +14,10 @@ import ( ) type CandidateGetArgs struct { - ResType string - ZoneID string - PoolID string - IgnorePool bool + // ResType is candidate host_type + ResType string + RegionID string + ZoneID string } type DataManager struct { @@ -200,10 +200,28 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida return nil, err2 } - hasZone := len(args.ZoneID) > 0 - result := []core.Candidater{} + matchZone := func(r core.Candidater, zoneId string) bool { + if args.ZoneID != "" { + if r.Getter().Zone().GetId() == zoneId { + return true + } + return false + } + return true + } + + matchRegion := func(r core.Candidater, regionId string) bool { + if args.RegionID != "" { + if r.Getter().Region().GetId() == regionId { + return true + } + return false + } + return true + } + for _, c := range candidates { r := c.(core.Candidater) @@ -211,11 +229,15 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida continue } - if args.IgnorePool { - result = append(result, r) - } else if (!hasZone || r.Get("ZoneID") == args.ZoneID) && r.Get("PoolID") == args.PoolID { - result = append(result, r) + if !matchRegion(r, args.RegionID) { + continue } + + if !matchZone(r, args.ZoneID) { + continue + } + + result = append(result, r) } return result, nil diff --git a/pkg/scheduler/db/models/host.go b/pkg/scheduler/db/models/host.go index e8d2e13083..a618795e50 100644 --- a/pkg/scheduler/db/models/host.go +++ b/pkg/scheduler/db/models/host.go @@ -6,8 +6,8 @@ import ( "github.com/jinzhu/gorm" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/api" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) const ( @@ -98,7 +98,7 @@ func (h Host) CPUOverCommitBound() float64 { if h.CPUCmtbound != nil { return *h.CPUCmtbound } - return float64(o.GetOptions().DefaultCpuOvercommitBound) + return float64(o.GetOptions().DefaultCPUOvercommitBound) } func (h Host) MemOverCommitBound() float64 { diff --git a/pkg/scheduler/db/models/storage.go b/pkg/scheduler/db/models/storage.go index d21235eeb3..eacc467860 100644 --- a/pkg/scheduler/db/models/storage.go +++ b/pkg/scheduler/db/models/storage.go @@ -8,7 +8,7 @@ import ( "yunion.io/x/log" - o "yunion.io/x/onecloud/cmd/scheduler/options" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type Storage struct { diff --git a/pkg/scheduler/handler/handler.go b/pkg/scheduler/handler/handler.go index cefeb042d0..4c6e8b6394 100644 --- a/pkg/scheduler/handler/handler.go +++ b/pkg/scheduler/handler/handler.go @@ -15,6 +15,8 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/db/models" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" + + computemodels "yunion.io/x/onecloud/pkg/compute/models" ) // InstallHandler is an interface that registes route and @@ -149,13 +151,7 @@ func doSchedulerForecast(c *gin.Context) { } func doCandidateList(c *gin.Context) { - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - - args, err := api.NewCandidateListArgs(sjson) + args, err := api.NewCandidateListArgs(c.Request.Body) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -171,22 +167,22 @@ func doCandidateList(c *gin.Context) { } func doCandidateDetail(c *gin.Context, id string) { - hs, err := models.FetchHostByIDs([]string{id}) + hs, err := computemodels.HostManager.FetchById(id) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } - if len(hs) == 0 { + if hs == nil { c.AbortWithError(http.StatusNotFound, fmt.Errorf("Candidate %s not found.", id)) return } - host := hs[0].(*models.Host) + host := hs.(*computemodels.SHost) args := new(api.CandidateDetailArgs) args.ID = id - if !host.IsHypervisor() { + if host.HostType == computemodels.HOST_TYPE_BAREMETAL { args.Type = api.HostTypeBaremetal } else { args.Type = api.HostTypeHost @@ -198,7 +194,7 @@ func doCandidateDetail(c *gin.Context, id string) { return } - c.JSON(http.StatusOK, result) + SendJSON(c, http.StatusOK, result) } func doCleanup(c *gin.Context) { diff --git a/pkg/scheduler/handler/helper.go b/pkg/scheduler/handler/helper.go new file mode 100644 index 0000000000..6d3499b123 --- /dev/null +++ b/pkg/scheduler/handler/helper.go @@ -0,0 +1,27 @@ +package handler + +import ( + "net/http" + + "gopkg.in/gin-gonic/gin.v1" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/appsrv" +) + +func SendJSON(c *gin.Context, code int, obj interface{}) { + c.Render(http.StatusOK, JSON{Data: obj}) +} + +type JSON struct { + Data interface{} +} + +func (r JSON) Render(w http.ResponseWriter) error { + appsrv.SendJSON(w, jsonutils.Marshal(r.Data)) + return nil +} + +func (r JSON) WriteContentType(w http.ResponseWriter) { +} diff --git a/pkg/scheduler/handler/ping.go b/pkg/scheduler/handler/ping.go index cbe25bc93b..dce80136eb 100644 --- a/pkg/scheduler/handler/ping.go +++ b/pkg/scheduler/handler/ping.go @@ -8,8 +8,8 @@ import ( "yunion.io/x/log" - o "yunion.io/x/onecloud/cmd/scheduler/options" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) var counter = 0 diff --git a/pkg/scheduler/manager/completed_queue.go b/pkg/scheduler/manager/completed_queue.go index b65a203a94..55263b9f04 100644 --- a/pkg/scheduler/manager/completed_queue.go +++ b/pkg/scheduler/manager/completed_queue.go @@ -6,8 +6,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/api" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type CompletedManager struct { diff --git a/pkg/scheduler/manager/expire_queue.go b/pkg/scheduler/manager/expire_queue.go index 2069f0a27e..ca74a90492 100644 --- a/pkg/scheduler/manager/expire_queue.go +++ b/pkg/scheduler/manager/expire_queue.go @@ -7,8 +7,8 @@ import ( "yunion.io/x/log" u "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/api" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type ExpireManager struct { diff --git a/pkg/scheduler/manager/manager.go b/pkg/scheduler/manager/manager.go index b4f09f7155..0b43409cf3 100644 --- a/pkg/scheduler/manager/manager.go +++ b/pkg/scheduler/manager/manager.go @@ -11,12 +11,12 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" candidatecache "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/data_manager" + o "yunion.io/x/onecloud/pkg/scheduler/options" "yunion.io/x/onecloud/pkg/util/k8s" ) @@ -142,10 +142,9 @@ func getHostCandidatesList(args *api.CandidateListArgs) (*api.CandidateListResul r.Limit = args.Limit r.Offset = args.Offset cs, err := GetCandidateManager().GetCandidates(data_manager.CandidateGetArgs{ - ResType: "host", - ZoneID: args.Zone, - PoolID: args.Pool, - IgnorePool: defaultIgnorePool, + ResType: "host", + ZoneID: args.Zone, + RegionID: args.Region, }) if err != nil { return nil, fmt.Errorf("Get host candidates err: %v", err) @@ -158,10 +157,9 @@ func getBaremetalCandidatesList(args *api.CandidateListArgs) (*api.CandidateList r.Limit = args.Limit r.Offset = args.Offset cs, err := GetCandidateManager().GetCandidates(data_manager.CandidateGetArgs{ - ResType: "baremetal", - ZoneID: args.Zone, - PoolID: args.Pool, - IgnorePool: defaultIgnorePool, + ResType: "baremetal", + ZoneID: args.Zone, + RegionID: args.Region, }) if err != nil { return nil, fmt.Errorf("Get baremetal candidates err: %v", err) @@ -283,12 +281,12 @@ func GetCandidateBaremetalList( mem := api.NewResultResourceInt64( c.FreeMemSize(), 0, - c.MemSize) + int64(c.MemSize)) cpu := api.NewResultResourceInt64( c.FreeCPUCount(), 0, - c.CPUCount) + int64(c.CpuCount)) storage := api.NewResultResourceInt64( c.FreeStorageSize(), diff --git a/pkg/scheduler/manager/scheduler.go b/pkg/scheduler/manager/scheduler.go index 37b855f264..35ac8903a6 100644 --- a/pkg/scheduler/manager/scheduler.go +++ b/pkg/scheduler/manager/scheduler.go @@ -25,7 +25,6 @@ func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) args := data_manager.CandidateGetArgs{ ResType: provider.CandidateType(), ZoneID: schedData.ZoneID, - PoolID: schedData.PoolID, } hosts, err = candidateManager.GetCandidates(args) } diff --git a/pkg/scheduler/manager/task_history.go b/pkg/scheduler/manager/task_history.go index 9c234ee711..2847d083ab 100644 --- a/pkg/scheduler/manager/task_history.go +++ b/pkg/scheduler/manager/task_history.go @@ -8,7 +8,7 @@ import ( "yunion.io/x/pkg/util/wait" u "yunion.io/x/pkg/utils" - o "yunion.io/x/onecloud/cmd/scheduler/options" + o "yunion.io/x/onecloud/pkg/scheduler/options" ) type HistoryItem struct { diff --git a/pkg/scheduler/manager/task_queue.go b/pkg/scheduler/manager/task_queue.go index 06d1edd956..6fa0e167d0 100644 --- a/pkg/scheduler/manager/task_queue.go +++ b/pkg/scheduler/manager/task_queue.go @@ -188,7 +188,7 @@ func (teqm *TaskExecutorQueueManager) AddTaskExecutor( scheduler Scheduler, callback TaskExecuteCallback) *TaskExecutor { schedData := scheduler.SchedData() log.V(10).Infof("AddTaskExecutor schedData: %#v", schedData) - taskQueue := teqm.GetQueue(schedData.Type, schedData.PoolID) + taskQueue := teqm.GetQueue(schedData.Type, "") return taskQueue.AddTaskExecutor(scheduler, callback) } diff --git a/cmd/scheduler/options/options.go b/pkg/scheduler/options/options.go similarity index 59% rename from cmd/scheduler/options/options.go rename to pkg/scheduler/options/options.go index 4576913468..e8b335a274 100644 --- a/cmd/scheduler/options/options.go +++ b/pkg/scheduler/options/options.go @@ -1,19 +1,25 @@ package options import ( - "fmt" "os" - gin "gopkg.in/gin-gonic/gin.v1" - - "yunion.io/x/log" - "yunion.io/x/pkg/util/version" - "yunion.io/x/structarg" + "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/compute/options" ) type SchedulerOptions struct { - IgnoreNonRunningGuests bool `help:"Ignore non running guests when build host memory and cpu size" default:"false" alias:"ignore-nonrunning-guests"` - IgnoreFakeDeletedGuests bool `help:"Ignore fake deleted guests when build host memory and cpu size" default:"false"` + options.ComputeOptions + + SchedOptions + + // gin http framework mode + GinMode string `help:"gin http framework work mode" default:"debug" choices:"debug|release"` +} + +type SchedOptions struct { + SchedulerPort int `help:"The port that the scheduler's http service runs on" default:"8897"` + IgnoreFakeDeletedGuests bool `help:"Ignore fake deleted guests when build host memory and cpu size" default:"false"` + AlwaysCheckAllPredicates bool `help:"Excute all predicates when scheduling" default:"false"` DisableBaremetalPredicates bool `help:"Switch to trigger baremetal related predicates" default:"false"` SchedulerTestLimit int `help:"Scheduler test items' limitations" default:"100"` @@ -30,11 +36,6 @@ type SchedulerOptions struct { PredicateParallelizeSize int `help:"Number of execute predicates parallelization" default:"14"` PriorityParallelizeSize int `help:"Number of execute priority parallelization" default:"14"` - // overcommit bound options - DefaultStorageOvercommitBound int `help:"Default storage overcommit bound" default:"1"` - DefaultCpuOvercommitBound int `help:"Default cpu overcommit bound" default:"8"` - DefaultMemoryOvercommitBound int `help:"Default memory overcommit bound" default:"1"` - // expire queue options ExpireQueueConsumptionPeriod string `help:"Expire queue consumption period" default:"3s"` ExpireQueueConsumptionTimeout string `help:"Expire queue consumption timeout" default:"10s"` @@ -81,83 +82,15 @@ type SchedulerOptions struct { WireDBCachePeriod string `help:"Wire database cache period" default:"5m"` } -type Options struct { - // common options - structarg.BaseOptions - Port int `help:"The port that the scheduler's http service runs on" default:"8897" alias:"scheduler-port"` - Address string `help:"The IP address to serve on (set to 0.0.0.0 for all interfaces)" default:"0.0.0.0"` +var ( + opt SchedulerOptions +) - // mysql options - SqlConnection string `help:"SQL connection string" default:"root:root@tcp(127.0.0.1:3306)/mclouds?charset=utf8&parseTime=True"` - - // log options - LogLevel string `help:"log level" default:"info" choices:"debug|info|warn|error"` - LogVerboseLevel int `help:"log verbosity level" default:"0"` - - // gin http framework mode - GinMode string `help:"gin http framework work mode" default:"debug" choices:"debug|release"` - - // cloud auth options - Region string `help:"Region name" default:"Beijing"` - AuthURL string `help:"Keystone auth URL" default:"http://10.168.26.241:35357/v2.0" alias:"auth-uri"` - AdminUser string `help:"Admin username" default:"regionadmin"` - AdminPasswd string `help:"Admin password" default:"eBVVSNaMeyzDnD8F" alias:"admin-password"` - AdminTenant string `help:"Admin tenant" default:"system" alias:"admin-tenant-name"` - - EnableSsl bool `help:"Enable https"` - SslCertfile string `help:"ssl certification file"` - SslKeyfile string `help:"ssl certification key file"` - - // scheduler options - SchedulerOptions +func GetOptions() *SchedulerOptions { + return &opt } -var options Options - -func GetOptions() *Options { - return &options -} - -func Parse() { - parser, e := structarg.NewArgumentParser(&options, - "scheduler", - `Yunion cloud scheduler`, - `Yunion Technology @ 2018`) - if e != nil { - log.Fatalf("Error define argument parser: %v", e) - } - - e = parser.ParseArgs(os.Args[1:], false) - if e != nil { - log.Fatalf("Parse arguments error: %v", e) - } - - if len(options.Config) > 0 { - e := parser.ParseFile(options.Config) - if e != nil { - log.Fatalf("Parse configuration file: %v", e) - } - } - - if options.Help { - fmt.Println(parser.HelpString()) - os.Exit(0) - } - - if options.Version { - fmt.Printf("Yunion cloud version:\n%s", version.GetJsonString()) - os.Exit(0) - } - - // log configuration - log.SetVerboseLevel(int32(options.LogVerboseLevel)) - e = log.SetLogLevelByString(log.Logger(), options.LogLevel) - if e != nil { - log.Fatalf("Set log level %q: %v", options.LogLevel, e) - } - - log.V(10).Debugf("Parsed options: %#v", options) - - // gin http framework mode configuration - gin.SetMode(options.GinMode) +func Init() { + cloudcommon.ParseOptions(&opt, os.Args, "region.conf", "scheduler") + options.Options = opt.ComputeOptions } diff --git a/pkg/scheduler/service/service.go b/pkg/scheduler/service/service.go new file mode 100644 index 0000000000..6e744e0ae6 --- /dev/null +++ b/pkg/scheduler/service/service.go @@ -0,0 +1,95 @@ +package service + +import ( + "io/ioutil" + "net" + "net/http" + "strconv" + + "gopkg.in/gin-gonic/gin.v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/util/prometheus" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + computemodels "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/db/models" + schedhandler "yunion.io/x/onecloud/pkg/scheduler/handler" + schedman "yunion.io/x/onecloud/pkg/scheduler/manager" + o "yunion.io/x/onecloud/pkg/scheduler/options" + "yunion.io/x/onecloud/pkg/util/gin/middleware" + + _ "yunion.io/x/onecloud/pkg/scheduler/algorithmprovider" +) + +func StartService() error { + o.Init() + opts := o.GetOptions() + dbOpts := &opts.DBOptions + + // gin http framework mode configuration + gin.SetMode(opts.GinMode) + + startSched := func() { + sqlDialect, sqlConn, err := utils.TransSQLAchemyURL(opts.SqlConnection) + if err != nil { + log.Fatalf("Invalid SqlConnection: %v", err) + } + if err := models.Init(sqlDialect, sqlConn); err != nil { + log.Fatalf("DB init error: %v, dialect: %s, url: %s", err, sqlDialect, sqlConn) + } + + stopEverything := make(chan struct{}) + schedman.InitAndStart(stopEverything) + } + + opts.Port = opts.SchedulerPort + // init region compute models + cloudcommon.InitDB(dbOpts) + defer cloudcommon.CloseDB() + + db.InitAllManagers() + + if err := computemodels.InitDB(); err != nil { + log.Fatalf("InitDB fail: %s", err) + } + + commonOpts := &opts.CommonOptions + cloudcommon.InitAuth(commonOpts, func() { + log.Infof("Auth complete!!") + startSched() + }) + + //app := cloudcommon.InitApp(commonOpts, true) + + //InitHandlers(app) + return startHTTP(opts) +} + +func startHTTP(opt *o.SchedulerOptions) error { + gin.DefaultWriter = ioutil.Discard + + router := gin.Default() + router.Use(middleware.Logger()) + router.Use(middleware.ErrorHandler) + router.Use(middleware.KeystoneTokenVerifyMiddleware()) + + prometheus.InstallHandler(router) + schedhandler.InstallHandler(router) + + server := &http.Server{ + Addr: net.JoinHostPort(opt.Address, strconv.Itoa(int(opt.Port))), + Handler: router, + } + + log.Infof("Start server on: %s:%d", opt.Address, opt.Port) + + if o.GetOptions().EnableSsl { + return server.ListenAndServeTLS(o.GetOptions().SslCertfile, + o.GetOptions().SslKeyfile) + } else { + return server.ListenAndServe() + } +} diff --git a/vendor/yunion.io/x/pkg/utils/misc.go b/vendor/yunion.io/x/pkg/utils/misc.go index b79264d6ea..f2eee46389 100644 --- a/vendor/yunion.io/x/pkg/utils/misc.go +++ b/vendor/yunion.io/x/pkg/utils/misc.go @@ -13,22 +13,32 @@ import ( type selectFunc func(obj interface{}) (string, error) -func ToDict(items []interface{}, ks selectFunc) (map[string]interface{}, error) { +func ToDict(objs interface{}, ks selectFunc) (map[string]interface{}, error) { + s := reflect.ValueOf(objs) + if s.Kind() != reflect.Slice { + return nil, fmt.Errorf("Not slice") + } res := map[string]interface{}{} - for _, item := range items { - key, err := ks(item) + for i := 0; i < s.Len(); i++ { + obj := s.Index(i).Interface() + key, err := ks(obj) if err != nil { return nil, err } - res[key] = item + res[key] = obj } return res, nil } -func GroupBy(items []interface{}, ks selectFunc) (map[string][]interface{}, error) { +func GroupBy(items interface{}, ks selectFunc) (map[string][]interface{}, error) { + s := reflect.ValueOf(items) + if s.Kind() != reflect.Slice { + return nil, fmt.Errorf("Not slice") + } res := map[string][]interface{}{} - for _, item := range items { - key, err := ks(item) + for i := 0; i < s.Len(); i++ { + obj := s.Index(i).Interface() + key, err := ks(obj) if err != nil { return nil, err } @@ -36,7 +46,7 @@ func GroupBy(items []interface{}, ks selectFunc) (map[string][]interface{}, erro if !ok { values = []interface{}{} } - values = append(values, item) + values = append(values, obj) res[key] = values } return res, nil From 6df68805010b68b7d1bbd3c3e400b1131c5fa391 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 14 Mar 2019 14:58:31 +0800 Subject: [PATCH 25/26] define scheduler apis output and input structs --- Gopkg.lock | 4 +- cmd/climc/shell/disks.go | 22 +- cmd/climc/shell/schedtaghosts.go | 2 - cmd/climc/shell/schedulers.go | 2 +- cmd/climc/shell/servers.go | 45 +- pkg/apis/api.go | 12 + pkg/apis/compute/api.go | 178 ++++ pkg/apis/compute/baremetal_const.go | 54 ++ pkg/apis/compute/disk.go | 51 + pkg/apis/compute/disk_const.go | 38 + pkg/apis/compute/doc.go | 1 + pkg/apis/compute/isolated_device_const.go | 26 + pkg/apis/compute/network_const.go | 38 + pkg/apis/compute/schedtag_const.go | 13 + pkg/apis/compute/storage_const.go | 78 ++ pkg/apis/doc.go | 1 + pkg/apis/scheduler/api.go | 82 ++ pkg/apis/scheduler/doc.go | 1 + pkg/baremetal/manager.go | 3 +- pkg/baremetal/utils/raid/drivers/drivers.go | 5 +- pkg/baremetal/utils/raid/hpssactl/hpssactl.go | 19 +- pkg/baremetal/utils/raid/interface.go | 13 +- pkg/baremetal/utils/raid/megactl/megactl.go | 47 +- pkg/baremetal/utils/raid/mvcli/mvcli.go | 15 +- pkg/baremetal/utils/raid/sas2iru/sas2iru.go | 15 +- pkg/cloudcommon/cmdline/doc.go | 1 + pkg/cloudcommon/cmdline/helper.go | 406 ++++++++ pkg/cloudcommon/cmdline/helper_test.go | 87 ++ pkg/cloudcommon/cmdline/parser.go | 349 +++++++ pkg/cloudcommon/cmdline/parser_test.go | 177 ++++ pkg/compute/baremetal/diskconfig.go | 188 +--- pkg/compute/baremetal/diskconfig_test.go | 67 +- pkg/compute/baremetal/types.go | 87 +- pkg/compute/guestdrivers/aliyun.go | 34 +- pkg/compute/guestdrivers/aws.go | 24 +- pkg/compute/guestdrivers/azure.go | 34 +- pkg/compute/guestdrivers/baremetals.go | 82 +- pkg/compute/guestdrivers/container.go | 14 - pkg/compute/guestdrivers/huawei.go | 12 +- pkg/compute/guestdrivers/kvm.go | 5 +- pkg/compute/guestdrivers/openstack.go | 16 +- pkg/compute/guestdrivers/qcloud.go | 40 +- pkg/compute/guestdrivers/virtualization.go | 25 +- pkg/compute/hostdrivers/base.go | 7 +- pkg/compute/hostdrivers/kvm.go | 1 + pkg/compute/models/disks.go | 229 ++--- pkg/compute/models/dynamicschedtags.go | 4 +- pkg/compute/models/guest_actions.go | 97 +- pkg/compute/models/guestdisks.go | 12 + pkg/compute/models/guestdrivers.go | 12 +- pkg/compute/models/guestnetworks.go | 16 + pkg/compute/models/guests.go | 396 ++++---- pkg/compute/models/helper.go | 73 +- pkg/compute/models/host_recycle.go | 42 +- pkg/compute/models/hosts.go | 1 - pkg/compute/models/isolated_devices.go | 82 +- pkg/compute/models/networks.go | 162 +--- pkg/compute/models/schedpolicies.go | 33 +- pkg/compute/models/schedtags.go | 13 +- pkg/compute/models/storages.go | 95 +- .../baremetal_convert_hypervisor_task.go | 8 +- pkg/compute/tasks/disk_batch_create_task.go | 42 +- pkg/compute/tasks/guest_backup_tasks.go | 14 +- pkg/compute/tasks/guest_batch_create_task.go | 49 +- pkg/compute/tasks/guest_change_config_task.go | 9 +- pkg/compute/tasks/guest_create_disk_task.go | 84 +- pkg/compute/tasks/guest_live_migrate_task.go | 12 +- pkg/compute/tasks/schedule.go | 114 ++- pkg/mcclient/modules/mod_scheduler.go | 47 +- pkg/mcclient/options/disks.go | 49 + pkg/mcclient/options/schedule.go | 17 - pkg/mcclient/options/schedulers.go | 129 +-- pkg/mcclient/options/servers.go | 280 ++++-- .../predicates/aggregate_predicate.go | 32 +- .../predicates/baremetal/cpu_predicate.go | 4 +- .../predicates/baremetal/memory_predicate.go | 2 +- .../predicates/baremetal/network_predicate.go | 14 +- .../predicates/baremetal/storage_predicate.go | 25 +- .../predicates/disk_schedtag_predicate.go | 199 +++- .../predicates/guest/cpu_predicate.go | 4 +- .../predicates/guest/group_predicate.go | 8 +- .../guest/isolated_device_predicate.go | 12 +- .../predicates/guest/memory_predicate.go | 4 +- .../predicates/guest/migrate_predicate.go | 4 +- .../predicates/guest/nest_predicate.go | 2 +- .../predicates/guest/network_predicate.go | 20 +- .../predicates/guest/storage_predicate.go | 12 +- .../algorithm/predicates/schedtag_helper.go | 82 +- .../priorities/guest/avoid_same_host.go | 2 +- pkg/scheduler/api/history.go | 2 +- pkg/scheduler/api/sched.go | 899 ++---------------- pkg/scheduler/cache/candidate/baremetals.go | 1 - pkg/scheduler/cache/candidate/base.go | 10 +- pkg/scheduler/cache/candidate/hosts.go | 55 +- pkg/scheduler/core/context.go | 6 +- pkg/scheduler/core/generic_scheduler.go | 13 +- pkg/scheduler/core/types.go | 10 +- .../data_manager/candidate_manager.go | 19 +- pkg/scheduler/handler/backup_helper.go | 25 +- pkg/scheduler/handler/forecast_helper.go | 2 +- pkg/scheduler/handler/handler.go | 49 +- pkg/scheduler/manager/manager.go | 17 +- pkg/scheduler/manager/scheduler.go | 14 +- pkg/scheduler/manager/task_queue.go | 6 +- pkg/scheduler/options/doc.go | 1 + pkg/scheduler/options/options.go | 4 +- pkg/scheduler/service/doc.go | 1 + pkg/scheduler/service/service.go | 3 +- 108 files changed, 3495 insertions(+), 2610 deletions(-) create mode 100644 pkg/apis/api.go create mode 100644 pkg/apis/compute/api.go create mode 100644 pkg/apis/compute/baremetal_const.go create mode 100644 pkg/apis/compute/disk.go create mode 100644 pkg/apis/compute/disk_const.go create mode 100644 pkg/apis/compute/doc.go create mode 100644 pkg/apis/compute/isolated_device_const.go create mode 100644 pkg/apis/compute/network_const.go create mode 100644 pkg/apis/compute/schedtag_const.go create mode 100644 pkg/apis/compute/storage_const.go create mode 100644 pkg/apis/doc.go create mode 100644 pkg/apis/scheduler/api.go create mode 100644 pkg/apis/scheduler/doc.go create mode 100644 pkg/cloudcommon/cmdline/doc.go create mode 100644 pkg/cloudcommon/cmdline/helper.go create mode 100644 pkg/cloudcommon/cmdline/helper_test.go create mode 100644 pkg/cloudcommon/cmdline/parser.go create mode 100644 pkg/cloudcommon/cmdline/parser_test.go create mode 100644 pkg/mcclient/options/disks.go delete mode 100644 pkg/mcclient/options/schedule.go create mode 100644 pkg/scheduler/options/doc.go create mode 100644 pkg/scheduler/service/doc.go diff --git a/Gopkg.lock b/Gopkg.lock index 02e25daf62..0310a4d7a0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1703,7 +1703,7 @@ [[projects]] branch = "master" - digest = "1:04fac3de327ed7bf4490436dd3eb5fb10c7e80b919a109935d053f584fdb6566" + digest = "1:65cac3daa80d78369c59b0e5ca4d8db352bf412db23767213e04f8c19dc822e1" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1737,7 +1737,7 @@ "utils", ] pruneopts = "UT" - revision = "5d28ae811a2aaaed19efdc468573ef236b8327f9" + revision = "e9943d77232072e08a77520fdc775dd29a43462e" [[projects]] branch = "master" diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 9a6af79ebe..9fac46e66a 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -157,31 +157,15 @@ func init() { return nil }) - type DiskCreateOptions struct { - options.ScheduleOptions - NAME string `help:"Name of the disk"` - DISKDESC string `help:"Image size or size of virtual disk"` - Desc string `help:"Description" metavar:"Description"` - Storage string `help:"ID or name of storage where the disk is created"` - TaskNotify bool `help:"Setup task notify"` - } - R(&DiskCreateOptions{}, "disk-create", "Create a virtual disk", func(s *mcclient.ClientSession, args *DiskCreateOptions) error { - params, err := args.ScheduleOptions.Params() + R(&options.DiskCreateOptions{}, "disk-create", "Create a virtual disk", func(s *mcclient.ClientSession, args *options.DiskCreateOptions) error { + params, err := args.Params() if err != nil { return err } - params.Add(jsonutils.NewString(args.NAME), "name") - params.Add(jsonutils.NewString(args.DISKDESC), "disk") - if len(args.Desc) > 0 { - params.Add(jsonutils.NewString(args.Desc), "description") - } if args.TaskNotify { s.PrepareTask() } - if args.Storage != "" { - params.Add(jsonutils.NewString(args.Storage), "storage") - } - disk, err := modules.Disks.Create(s, params) + disk, err := modules.Disks.Create(s, params.JSON(params)) if err != nil { return err } diff --git a/cmd/climc/shell/schedtaghosts.go b/cmd/climc/shell/schedtaghosts.go index b68b5144e6..b2d636dfe1 100644 --- a/cmd/climc/shell/schedtaghosts.go +++ b/cmd/climc/shell/schedtaghosts.go @@ -3,8 +3,6 @@ package shell import ( "fmt" - "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" diff --git a/cmd/climc/shell/schedulers.go b/cmd/climc/shell/schedulers.go index 8c3c47b808..5bcea109b4 100644 --- a/cmd/climc/shell/schedulers.go +++ b/cmd/climc/shell/schedulers.go @@ -35,7 +35,7 @@ func init() { if err != nil { return err } - result, err := modules.SchedManager.DoForecast(s, params) + result, err := modules.SchedManager.DoForecast(s, params.JSON(params)) if err != nil { return err } diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 9ad2743bb5..0d1b2b9d10 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -8,7 +8,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -116,47 +115,31 @@ func init() { if err != nil { return err } - - if opts.GenerateName { - params.Add(jsonutils.NewString(opts.NAME), "generate_name") - } else { - params.Add(jsonutils.NewString(opts.NAME), "name") - } - if regutils.MatchSize(opts.MEMSPEC) { - params.Add(jsonutils.NewString(opts.MEMSPEC), "vmem_size") - } else { - params.Add(jsonutils.NewString(opts.MEMSPEC), "instance_type") - } - - if opts.NoAccountInit != nil && *opts.NoAccountInit { - params.Add(jsonutils.JSONFalse, "reset_password") - } - - if len(opts.UserDataFile) > 0 { - userdata, err := ioutil.ReadFile(opts.UserDataFile) - if err != nil { - return err - } - params.Add(jsonutils.NewString(string(userdata)), "user_data") - } - - count := options.IntV(opts.Count) + count := params.Count if options.BoolV(opts.DryRun) { - results, err := modules.SchedManager.DoScheduleListResult(s, params, count) + listFields := []string{"id", "name", "capacity", "count", "score", "capacity_details", "score_details"} + input, err := opts.ToScheduleInput() if err != nil { return err } - printList(results, []string{"id", "name", "rank", "capacity", "error"}) + result, err := modules.SchedManager.Test(s, input) + if err != nil { + return err + } + if err != nil { + return err + } + printList(modules.JSON2ListResult(result), listFields) } else { taskNotify := options.BoolV(opts.TaskNotify) if taskNotify { s.PrepareTask() } if count > 1 { - results := modules.Servers.BatchCreate(s, params, count) + results := modules.Servers.BatchCreate(s, params.JSON(params), count) printBatchResults(results, modules.Servers.GetColumns(s)) } else { - server, err := modules.Servers.Create(s, params) + server, err := modules.Servers.Create(s, params.JSON(params)) if err != nil { return err } @@ -384,7 +367,7 @@ func init() { if err != nil { return err } - srv, e := modules.Servers.PerformAction(s, opts.ID, "deploy", params) + srv, e := modules.Servers.PerformAction(s, opts.ID, "deploy", params.JSON(params)) if e != nil { return e } diff --git a/pkg/apis/api.go b/pkg/apis/api.go new file mode 100644 index 0000000000..2b6d540b7e --- /dev/null +++ b/pkg/apis/api.go @@ -0,0 +1,12 @@ +package apis + +import ( + "yunion.io/x/jsonutils" +) + +// Meta is embedded by every input or output params +type Meta struct{} + +func (m Meta) JSON(self interface{}) *jsonutils.JSONDict { + return jsonutils.Marshal(self).(*jsonutils.JSONDict) +} diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go new file mode 100644 index 0000000000..90a67806fc --- /dev/null +++ b/pkg/apis/compute/api.go @@ -0,0 +1,178 @@ +package compute + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" +) + +type SchedtagConfig struct { + apis.Meta + + Id string `json:"id"` + Strategy string `json:"strategy"` +} + +type NetworkConfig struct { + apis.Meta + + Index int `json:"index"` + Network string `json:"network"` + Wire string `json:"wire"` + Exit bool `json:"exit"` + Private bool `json:"private"` + Mac string `json:"mac"` + Address string `json:"address"` + Address6 string `json:"address6"` + Driver string `json:"driver"` + BwLimit int `json:"bw_limit"` + Vip bool `json:"vip"` + Reserved bool `json:"reserved"` + NetType string `json:"net_type"` + + RequireTeaming bool `json:"require_teaming"` + TryTeaming bool `json:"try_teaming"` + + StandbyPortCount int `json:"standby_port_count"` + StandbyAddrCount int `json:"standby_addr_count"` + + Project string `json:"project"` + Ifname string `json:"ifname"` +} + +type DiskConfig struct { + apis.Meta + + Index int `json:"index"` // -1 means not attach to any guests + ImageId string `json:"image_id"` + SnapshotId string `json:"snapshot_id"` + DiskType string `json:"disk_type"` + Schedtags []*SchedtagConfig `json:"schedtags"` + + SizeMb int `json:"size"` + Fs string `json:"fs"` + Format string `json:"format"` + Driver string `json:"driver"` + Cache string `json:"cache"` + Mountpoint string `json:"mountpoint"` + Backend string `json:"backend"` // stroageType + Medium string `json:"medium"` + ImageProperties map[string]string `json:"image_properties"` + + Storage string `json:"storage"` + DiskId string `json:"disk_id"` +} + +type IsolatedDeviceConfig struct { + Index int `json:"index"` + Id string `json:"id"` + DevType string `json:"dev_type"` + Model string `json:"model"` + Vendor string `json:"vendor"` +} + +type BaremetalDiskConfig struct { + //Index int `json:"index"` + // disk type + Type string `json:"type"` + // raid config + Conf string `json:"conf"` + Count int64 `json:"count"` + Range []int64 `json:"range"` + Splits string `json:"splits"` + Size []int64 `json:"size"` + Adapter *int `json:"adapter,omitempty"` + Driver string `json:"driver"` + Cachedbadbbu *bool `json:"cachedbadbbu,omitempty"` + Strip *int64 `json:"strip,omitempty"` + RA *bool `json:"ra,omitempty"` + WT *bool `json:"wt,omitempty"` + Direct *bool `json:"direct,omitempty"` +} + +type ServerConfigs struct { + // prefer options + PreferRegion string `json:"prefer_region_id"` + PreferZone string `json:"prefer_zone_id"` + PreferWire string `json:"prefer_wire_id"` + PreferHost string `json:"prefer_host_id"` + PreferBackupHost string `json:"prefer_backup_host"` + + Hypervisor string `json:"hypervisor"` + // ResourceType "shared|prepaid|dedicated"` + ResourceType string `json:"resource_type"` + Project string `json:"project"` + Backup bool `json:"backup"` + Count int `json:"count"` + + Disks []*DiskConfig `json:"disks"` + Networks []*NetworkConfig `json:"nets"` + Schedtags []*SchedtagConfig `json:"schedtags"` + IsolatedDevices []*IsolatedDeviceConfig `json:"isolated_devices"` + BaremetalDiskConfigs []*BaremetalDiskConfig `json:"baremetal_disk_configs"` + + // DEPRECATE + Suggestion bool `json:"suggestion"` +} + +type DeployConfig struct { + Action string `json:"action"` + Path string `json:"path"` + Content string `json:"content"` +} + +type ServerCreateInput struct { + apis.Meta + + *ServerConfigs + + Name string `json:"name"` + GenerateName string `json:"generate_name"` + VmemSize int `json:"vmem_size"` + VcpuCount int `json:"vcpu_count"` + InstanceType string `json:"instance_type"` + UserData string `json:"user_data"` + + Keypair string `json:"keypair"` + Password string `json:"password"` + Cdrom string `json:"cdrom"` + Vga string `json:"vga"` + Vdi string `json:"vdi"` + Bios string `json:"bios"` + Description string `json:"description"` + BootOrder string `json:"boot_order"` + ResetPassword *bool `json:"reset_password"` + DisableDelete bool `json:"disable_delete"` + ShutdownBehavior string `json:"shutdown_behavior"` + AutoStart bool `json:"auto_start"` + DeployConfigs []*DeployConfig `json:"deploy_configs"` + IsSystem bool `json:"is_system"` + Duration string `json:"duration"` + AutoPrepaidRecycle bool `json:"auto_prepaid_recycle,omitfalse"` + Secgroup string `json:"secgroup"` + EipBw int `json:"eip_bw,omitzero"` + EipChargeType string `json:"eip_charge_type,omitempty"` + Eip string `json:"eip,omitempty"` + + OsType string `json:"os_type"` + // Fill by server + OsProfile jsonutils.JSONObject `json:"__os_profile__"` + BillingType string `json:"billing_type"` + BillingCycle string `json:"billing_cycle"` + + // DEPRECATE or not used fields + Baremetal bool `json:"baremetal"` +} + +type ServerDeployInput struct { + apis.Meta + + Id string + + Keypair string `json:"keypair"` + DeleteKeypair bool `json:"__delete_keypair__"` + DeployConfigs []*DeployConfig `json:"deploy_configs"` + ResetPassword *bool `json:"reset_password"` + Password string `json:"password"` + AutoStart *bool `json:"auto_start"` +} diff --git a/pkg/apis/compute/baremetal_const.go b/pkg/apis/compute/baremetal_const.go new file mode 100644 index 0000000000..30a3d7f310 --- /dev/null +++ b/pkg/apis/compute/baremetal_const.go @@ -0,0 +1,54 @@ +package compute + +import ( + "yunion.io/x/pkg/util/sets" +) + +const ( + DISK_CONF_RAID0 = "raid0" + DISK_CONF_RAID1 = "raid1" + DISK_CONF_RAID5 = "raid5" + DISK_CONF_RAID10 = "raid10" + DISK_CONF_NONE = "none" + + DEFAULT_DISK_CONF = DISK_CONF_NONE + + DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + + DISK_DRIVER_MEGARAID = "MegaRaid" + DISK_DRIVER_LINUX = "Linux" + DISK_DRIVER_HPSARAID = "HPSARaid" + DISK_DRIVER_MPT2SAS = "Mpt2SAS" + DISK_DRIVER_MARVELRAID = "MarvelRaid" + DISK_DRIVER_PCIE = "PCIE" + + HDD_DISK_SPEC_TYPE = "HDD" + SSD_DISK_SPEC_TYPE = "SSD" +) + +var ( + BaremetalDefaultDiskConfig = BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_NONE, + Count: 0, + } + + DISK_CONFS = sets.NewString( + DISK_CONF_RAID0, + DISK_CONF_RAID1, + DISK_CONF_RAID5, + DISK_CONF_RAID10, + DISK_CONF_NONE, + ) + + DISK_DRIVERS_RAID = sets.NewString( + DISK_DRIVER_MEGARAID, + DISK_DRIVER_HPSARAID, + DISK_DRIVER_MPT2SAS, + DISK_DRIVER_MARVELRAID, + ) + + DISK_DRIVERS = sets.NewString( + DISK_DRIVER_LINUX, + DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) +) diff --git a/pkg/apis/compute/disk.go b/pkg/apis/compute/disk.go new file mode 100644 index 0000000000..34491ca2a5 --- /dev/null +++ b/pkg/apis/compute/disk.go @@ -0,0 +1,51 @@ +package compute + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type DiskCreateInput struct { + apis.Meta + + *DiskConfig + + // prefer options + PreferRegion string `json:"prefer_region_id"` + PreferZone string `json:"prefer_zone_id"` + PreferWire string `json:"prefer_wire_id"` + PreferHost string `json:"prefer_host_id"` + + Name string `json:"name"` + Description string `json:"description"` + Hypervisor string `json:"hypervisor"` + Project string `json:"project"` +} + +// ToServerCreateInput used by disk schedule +func (req *DiskCreateInput) ToServerCreateInput() *ServerCreateInput { + return &ServerCreateInput{ + ServerConfigs: &ServerConfigs{ + PreferRegion: req.PreferRegion, + PreferZone: req.PreferZone, + PreferWire: req.PreferWire, + PreferHost: req.PreferHost, + Hypervisor: req.Hypervisor, + Disks: []*DiskConfig{req.DiskConfig}, + Project: req.Project, + }, + Name: req.Name, + } +} + +func (req *ServerCreateInput) ToDiskCreateInput() *DiskCreateInput { + return &DiskCreateInput{ + DiskConfig: req.Disks[0], + PreferRegion: req.PreferRegion, + PreferHost: req.PreferHost, + PreferZone: req.PreferZone, + PreferWire: req.PreferWire, + Name: req.Name, + Project: req.Project, + Hypervisor: req.Hypervisor, + } +} diff --git a/pkg/apis/compute/disk_const.go b/pkg/apis/compute/disk_const.go new file mode 100644 index 0000000000..b864252f35 --- /dev/null +++ b/pkg/apis/compute/disk_const.go @@ -0,0 +1,38 @@ +package compute + +const ( + DISK_INIT = "init" + DISK_REBUILD = "rebuild" + DISK_ALLOC_FAILED = "alloc_failed" + DISK_STARTALLOC = "start_alloc" + DISK_BACKUP_STARTALLOC = "backup_start_alloc" + DISK_BACKUP_ALLOC_FAILED = "backup_alloc_failed" + DISK_ALLOCATING = "allocating" + DISK_READY = "ready" + DISK_RESET = "reset" + DISK_DEALLOC = "deallocating" + DISK_DEALLOC_FAILED = "dealloc_failed" + DISK_UNKNOWN = "unknown" + DISK_DETACHING = "detaching" + DISK_ATTACHING = "attaching" + + DISK_START_SAVE = "start_save" + DISK_SAVING = "saving" + + DISK_START_RESIZE = "start_resize" + DISK_RESIZING = "resizing" + + DISK_START_MIGRATE = "start_migrate" + DISK_POST_MIGRATE = "post_migrate" + DISK_MIGRATING = "migrating" + + DISK_START_SNAPSHOT = "start_snapshot" + DISK_SNAPSHOTING = "snapshoting" + + DISK_TYPE_SYS = "sys" + DISK_TYPE_SWAP = "swap" + DISK_TYPE_DATA = "data" + DISK_TYPE_VOLUME = "volume" + + DISK_BACKING_IMAGE = "image" +) diff --git a/pkg/apis/compute/doc.go b/pkg/apis/compute/doc.go new file mode 100644 index 0000000000..062f6a74e8 --- /dev/null +++ b/pkg/apis/compute/doc.go @@ -0,0 +1 @@ +package compute // import "yunion.io/x/onecloud/pkg/apis/compute" diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go new file mode 100644 index 0000000000..7f613d8146 --- /dev/null +++ b/pkg/apis/compute/isolated_device_const.go @@ -0,0 +1,26 @@ +package compute + +const ( + DIRECT_PCI_TYPE = "PCI" + GPU_HPC_TYPE = "GPU-HPC" // # for compute + GPU_VGA_TYPE = "GPU-VGA" // # for display + USB_TYPE = "USB" + NIC_TYPE = "NIC" + + NVIDIA_VENDOR_ID = "10de" + AMD_VENDOR_ID = "1002" +) + +var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} + +var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE} + +var ID_VENDOR_MAP = map[string]string{ + NVIDIA_VENDOR_ID: "NVIDIA", + AMD_VENDOR_ID: "AMD", +} + +var VENDOR_ID_MAP = map[string]string{ + "NVIDIA": NVIDIA_VENDOR_ID, + "AMD": AMD_VENDOR_ID, +} diff --git a/pkg/apis/compute/network_const.go b/pkg/apis/compute/network_const.go new file mode 100644 index 0000000000..432af60b95 --- /dev/null +++ b/pkg/apis/compute/network_const.go @@ -0,0 +1,38 @@ +package compute + +const ( + // # DEFAULT_BANDWIDTH = options.default_bandwidth + MAX_BANDWIDTH = 100000 + + NETWORK_TYPE_GUEST = "guest" + NETWORK_TYPE_BAREMETAL = "baremetal" + NETWORK_TYPE_CONTAINER = "container" + NETWORK_TYPE_PXE = "pxe" + NETWORK_TYPE_IPMI = "ipmi" + + STATIC_ALLOC = "static" + + MAX_NETWORK_NAME_LEN = 11 + + EXTRA_DNS_UPDATE_TARGETS = "__extra_dns_update_targets" + + NETWORK_STATUS_INIT = "init" + NETWORK_STATUS_PENDING = "pending" + NETWORK_STATUS_AVAILABLE = "available" + NETWORK_STATUS_FAILED = "failed" + NETWORK_STATUS_UNKNOWN = "unknown" + NETWORK_STATUS_START_DELETE = "start_delete" + NETWORK_STATUS_DELETING = "deleting" + NETWORK_STATUS_DELETED = "deleted" + NETWORK_STATUS_DELETE_FAILED = "delete_failed" +) + +var ( + ALL_NETWORK_TYPES = []string{ + NETWORK_TYPE_GUEST, + NETWORK_TYPE_BAREMETAL, + NETWORK_TYPE_CONTAINER, + NETWORK_TYPE_PXE, + NETWORK_TYPE_IPMI, + } +) diff --git a/pkg/apis/compute/schedtag_const.go b/pkg/apis/compute/schedtag_const.go new file mode 100644 index 0000000000..6e3a301207 --- /dev/null +++ b/pkg/apis/compute/schedtag_const.go @@ -0,0 +1,13 @@ +package compute + +const ( + STRATEGY_REQUIRE = "require" + STRATEGY_EXCLUDE = "exclude" + STRATEGY_PREFER = "prefer" + STRATEGY_AVOID = "avoid" + + // # container used aggregate + CONTAINER_AGGREGATE = "container" +) + +var STRATEGY_LIST = []string{STRATEGY_REQUIRE, STRATEGY_EXCLUDE, STRATEGY_PREFER, STRATEGY_AVOID} diff --git a/pkg/apis/compute/storage_const.go b/pkg/apis/compute/storage_const.go new file mode 100644 index 0000000000..82ebce7899 --- /dev/null +++ b/pkg/apis/compute/storage_const.go @@ -0,0 +1,78 @@ +package compute + +const ( + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_NFS = "nfs" + + STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + STORAGE_CLOUD_SSD = "cloud_ssd" + STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 + STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" //单块本地SSD盘, 容量最大不能超过800 GiB + + //Azure hdd and ssd storagetype + STORAGE_STANDARD_LRS = "standard_lrs" + STORAGE_STANDARDSSD_LRS = "standardssd_lrs" + STORAGE_PREMIUM_LRS = "premium_lrs" + + // aws storage type + STORAGE_GP2_SSD = "gp2" // aws general purpose ssd + STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD + STORAGE_SC1_HDD = "sc1" // aws Cold HDD + STORAGE_STANDARD_HDD = "standard" // aws Magnetic volumes + + // qcloud storage type + // STORAGE_CLOUD_SSD ="cloud_ssd" + STORAGE_LOCAL_BASIC = "local_basic" + STORAGE_LOCAL_SSD = "local_ssd" + STORAGE_CLOUD_BASIC = "cloud_basic" + STORAGE_CLOUD_PREMIUM = "cloud_premium" + + // huawei storage type + STORAGE_HUAWEI_SSD = "SSD" // 超高IO云硬盘 + STORAGE_HUAWEI_SAS = "SAS" // 高IO云硬盘 + STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘 + + // openstack + STORAGE_OPENSTACK_ISCSI = "iscsi" +) + +const ( + STORAGE_ENABLED = "enabled" + // STORAGE_DISABLED = "disabled" + STORAGE_OFFLINE = "offline" + STORAGE_ONLINE = "online" + + DISK_TYPE_ROTATE = "rotate" + DISK_TYPE_SSD = "ssd" + DISK_TYPE_HYBRID = "hybrid" +) + +var ( + DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} + STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} + STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES + STORAGE_ALL_TYPES = []string{ + STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_NFS, + } + STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, + STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_EPHEMERAL_SSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS, + STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, + STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, + STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, + STORAGE_OPENSTACK_ISCSI, + } + + STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} +) diff --git a/pkg/apis/doc.go b/pkg/apis/doc.go new file mode 100644 index 0000000000..ed44acc20d --- /dev/null +++ b/pkg/apis/doc.go @@ -0,0 +1 @@ +package apis // import "yunion.io/x/onecloud/pkg/apis" diff --git a/pkg/apis/scheduler/api.go b/pkg/apis/scheduler/api.go new file mode 100644 index 0000000000..25dfc24fb5 --- /dev/null +++ b/pkg/apis/scheduler/api.go @@ -0,0 +1,82 @@ +package scheduler + +import ( + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +type ScheduleBaseConfig struct { + BestEffort bool `json:"best_effort"` + SuggestionLimit int64 `json:"suggestion_limit"` + SuggestionAll bool `json:"suggestion_all"` + IgnoreFilters []string `json:"ignore_filters"` + SessionId string `json:"session_id"` + + // usedby test api + RecordLog bool `json:"record_to_history"` + Details bool `json:"details"` +} + +type ForGuest struct { + Id string `json:"id"` + Name string `json:"name"` +} + +type GroupRelation struct { + GroupId string `json:"group_id"` + Strategy string `json:"strategy"` + Scope string `json:"scope"` +} + +type ServerConfig struct { + *compute.ServerConfigs + + Memory int `json:"vmem_size"` + Ncpu int `json:"vcpu_count"` + Name string `json:"name"` + GuestStatus string `json:"guest_status"` + + // HostId used by migrate + HostId string `json:"host_id"` + + // DEPRECATED + Metadata map[string]string `json:"__meta__"` + ForGuests []*ForGuest `json:"for_guests"` + GroupRelations []*GroupRelation `json:"group_releations"` + Groups interface{} `json:"groups"` + Id string `json:"id"` +} + +// ScheduleInput used by scheduler sync-schedule/test/forecast api +type ScheduleInput struct { + apis.Meta + + ScheduleBaseConfig + + ServerConfig +} + +type CandidateDisk struct { + Index int `json:"index"` + StorageId string `json:"storage_id"` +} + +type CandidateResource struct { + HostId string `json:"host_id"` + Name string `json:"name"` + Disks []*CandidateDisk `json:"disks"` + + // used by backup schedule + BackupCandidate *CandidateResource `json:"backup_candidate"` + IsMaster bool `json:"is_master"` + IsSlave bool `json:"is_slave"` + + // Error means no candidate found, include reasons + Error string `json:"error"` +} + +type ScheduleOutput struct { + apis.Meta + + Candidates []*CandidateResource `json:"candidates"` +} diff --git a/pkg/apis/scheduler/doc.go b/pkg/apis/scheduler/doc.go new file mode 100644 index 0000000000..5aeecee35f --- /dev/null +++ b/pkg/apis/scheduler/doc.go @@ -0,0 +1 @@ +package scheduler // import "yunion.io/x/onecloud/pkg/apis/scheduler" diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 7b77260d83..0e2fab8e94 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/pkg/util/workqueue" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" o "yunion.io/x/onecloud/pkg/baremetal/options" "yunion.io/x/onecloud/pkg/baremetal/profiles" "yunion.io/x/onecloud/pkg/baremetal/pxe" @@ -1162,7 +1163,7 @@ func (s *SBaremetalServer) GetRootTemplateId() string { return id } -func (s *SBaremetalServer) GetDiskConfig() ([]*baremetal.BaremetalDiskConfig, error) { +func (s *SBaremetalServer) GetDiskConfig() ([]*api.BaremetalDiskConfig, error) { layouts := make([]baremetal.Layout, 0) err := s.desc.Unmarshal(&layouts, "disk_config") if err != nil { diff --git a/pkg/baremetal/utils/raid/drivers/drivers.go b/pkg/baremetal/utils/raid/drivers/drivers.go index ab7518dbcd..e97b908a9f 100644 --- a/pkg/baremetal/utils/raid/drivers/drivers.go +++ b/pkg/baremetal/utils/raid/drivers/drivers.go @@ -3,6 +3,7 @@ package drivers import ( "fmt" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" _ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/hpssactl" _ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/megactl" @@ -28,7 +29,7 @@ func GetDrivers(term *ssh.Client) []raid.IRaidDriver { return ret } -func BuildRaid(driver raid.IRaidDriver, confs []*baremetal.BaremetalDiskConfig, adapterIdx int) error { +func BuildRaid(driver raid.IRaidDriver, confs []*api.BaremetalDiskConfig, adapterIdx int) error { if err := driver.PreBuildRaid(confs, adapterIdx); err != nil { return fmt.Errorf("PreBuildRaid: %v", err) } @@ -48,7 +49,7 @@ func BuildRaid(driver raid.IRaidDriver, confs []*baremetal.BaremetalDiskConfig, return nil } -func buildRaid(adapter raid.IRaidAdapter, confs []*baremetal.BaremetalDiskConfig) error { +func buildRaid(adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) error { if err := adapter.PreBuildRaid(confs); err != nil { return fmt.Errorf("PreBuildRaid: %v", err) } diff --git a/pkg/baremetal/utils/raid/hpssactl/hpssactl.go b/pkg/baremetal/utils/raid/hpssactl/hpssactl.go index c2cfaa9587..6bc1560d73 100644 --- a/pkg/baremetal/utils/raid/hpssactl/hpssactl.go +++ b/pkg/baremetal/utils/raid/hpssactl/hpssactl.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/regutils2" @@ -105,7 +106,7 @@ func (adapter *HPSARaidAdaptor) GetIndex() int { return adapter.index } -func (adapter *HPSARaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } @@ -163,7 +164,7 @@ func (adapter *HPSARaidAdaptor) GetDevices() []*baremetal.BaremetalStorage { return ret } -func (adapter *HPSARaidAdaptor) conf2Params(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *HPSARaidAdaptor) conf2Params(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.Direct != nil { if *(conf.Direct) { @@ -195,7 +196,7 @@ func (adapter *HPSARaidAdaptor) getLastArray() (string, error) { return "", nil } -func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { labels := []string{} for _, dev := range devs { labels = append(labels, fmt.Sprintf("%s", GetSpecString(dev))) @@ -235,19 +236,19 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme return err } -func (adapter *HPSARaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("0", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("1", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("5", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("10", devs, conf) } @@ -256,7 +257,7 @@ func (adapter *HPSARaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage // WT|WB] [NORA|RA] [Direct|Cached] [CachedBadBBU|NoCachedBadBBU] useWT := true useDirect := true - if err := adapter.buildRaid("0", []*baremetal.BaremetalStorage{d}, &baremetal.BaremetalDiskConfig{WT: &useWT, Direct: &useDirect}); err != nil { + if err := adapter.buildRaid("0", []*baremetal.BaremetalStorage{d}, &api.BaremetalDiskConfig{WT: &useWT, Direct: &useDirect}); err != nil { return err } } @@ -354,7 +355,7 @@ func (r *HPSARaid) parsePhyDevs(lines []string) error { return nil } -func (r *HPSARaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *HPSARaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/baremetal/utils/raid/interface.go b/pkg/baremetal/utils/raid/interface.go index a49632c025..638493a774 100644 --- a/pkg/baremetal/utils/raid/interface.go +++ b/pkg/baremetal/utils/raid/interface.go @@ -1,6 +1,7 @@ package raid import ( + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/baremetal" ) @@ -8,21 +9,21 @@ type IRaidDriver interface { ParsePhyDevs() error GetName() string GetAdapters() []IRaidAdapter - PreBuildRaid(confs []*baremetal.BaremetalDiskConfig, adapterIdx int) error + PreBuildRaid(confs []*api.BaremetalDiskConfig, adapterIdx int) error CleanRaid() error } type IRaidAdapter interface { GetIndex() int - PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error + PreBuildRaid(confs []*api.BaremetalDiskConfig) error GetLogicVolumes() ([]int, error) RemoveLogicVolumes() error GetDevices() []*baremetal.BaremetalStorage - BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error + BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error BuildNoneRaid(devs []*baremetal.BaremetalStorage) error } diff --git a/pkg/baremetal/utils/raid/megactl/megactl.go b/pkg/baremetal/utils/raid/megactl/megactl.go index 31ade2a507..42eab7dc21 100644 --- a/pkg/baremetal/utils/raid/megactl/megactl.go +++ b/pkg/baremetal/utils/raid/megactl/megactl.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/regutils2" @@ -225,12 +226,12 @@ func (adapter *MegaRaidAdaptor) parseLogicVolumes(lines []string) []int { return lvIdx } -func (adapter *MegaRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { adapter.clearJBODDisks() return nil } -func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *api.BaremetalDiskConfig) []string { params := []string{} szStr := []string{} if len(conf.Size) > 0 { @@ -242,7 +243,7 @@ func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *baremetal.Baremetal return params } -func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.WT != nil { if *conf.WT { @@ -278,7 +279,7 @@ func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *baremetal.BaremetalDisk return params } -func conf2Params(conf *baremetal.BaremetalDiskConfig) []string { +func conf2Params(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.WT != nil { if *conf.WT { @@ -319,35 +320,35 @@ func conf2Params(conf *baremetal.BaremetalDiskConfig) []string { return params } -func (adapter *MegaRaidAdaptor) storcliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 0) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 0) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 1) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 1) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 5) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 5) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 10) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(devs)%2 != 0 { return fmt.Errorf("Odd number of %d devs", len(devs)) } @@ -367,7 +368,7 @@ func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalSt return err } -func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig, level uint) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig, level uint) error { args := []string{} args = append(args, fmt.Sprintf("/c%d", adapter.index)) args = append(args, "add", "vd", fmt.Sprintf("type=r%d", level)) @@ -387,7 +388,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStor return err } -func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig, level uint) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig, level uint) error { labels := []string{} for _, dev := range devs { labels = append(labels, GetSpecString(dev)) @@ -403,8 +404,8 @@ func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStor func cliBuildRaid( devs []*baremetal.BaremetalStorage, - conf *baremetal.BaremetalDiskConfig, - funcs ...func([]*baremetal.BaremetalStorage, *baremetal.BaremetalDiskConfig) error, + conf *api.BaremetalDiskConfig, + funcs ...func([]*baremetal.BaremetalStorage, *api.BaremetalDiskConfig) error, ) error { var err error for _, f := range funcs { @@ -416,19 +417,19 @@ func cliBuildRaid( return err } -func (adapter *MegaRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid0, adapter.storcliBuildRaid0) } -func (adapter *MegaRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid1, adapter.storcliBuildRaid1) } -func (adapter *MegaRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid5, adapter.storcliBuildRaid5) } -func (adapter *MegaRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid10, adapter.storcliBuildRaid10) } @@ -492,7 +493,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildJBOD(devs []*baremetal.BaremetalStor return nil } -func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { err := adapter.storcliBuildJBOD(devs) if err == nil { return nil @@ -513,7 +514,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt return err } -func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { err := adapter.megacliBuildJBOD(devs) if err == nil { return nil @@ -746,7 +747,7 @@ func (raid *MegaRaid) CleanRaid() error { return nil } -func (raid *MegaRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (raid *MegaRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return raid.clearForeignState() } diff --git a/pkg/baremetal/utils/raid/mvcli/mvcli.go b/pkg/baremetal/utils/raid/mvcli/mvcli.go index be712c5957..9bb612e86d 100644 --- a/pkg/baremetal/utils/raid/mvcli/mvcli.go +++ b/pkg/baremetal/utils/raid/mvcli/mvcli.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/stringutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/ssh" @@ -185,11 +186,11 @@ func (adapter *MarvelRaidAdaptor) removeLogicVolume(idx int) error { return err } -func (adapter *MarvelRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } -func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { pds := []string{} for _, dev := range devs { pds = append(pds, fmt.Sprintf("%s", GetSpecString(dev))) @@ -200,20 +201,20 @@ func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.Bare return err } -func (adapter *MarvelRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r0", devs, conf) } -func (adapter *MarvelRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r1", devs, conf) } -func (adapter *MarvelRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { //return adapter.buildRaid("-r5", devs, conf) return fmt.Errorf("BuildRaid5 not impl") } -func (adapter *MarvelRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r10", devs, conf) } @@ -276,7 +277,7 @@ func (r *MarvelRaid) parseAdapters(lines []string) error { return nil } -func (r *MarvelRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *MarvelRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/baremetal/utils/raid/sas2iru/sas2iru.go b/pkg/baremetal/utils/raid/sas2iru/sas2iru.go index bdc2f5b1f1..a146e0edf8 100644 --- a/pkg/baremetal/utils/raid/sas2iru/sas2iru.go +++ b/pkg/baremetal/utils/raid/sas2iru/sas2iru.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/stringutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/ssh" @@ -225,11 +226,11 @@ func (adapter *Mpt2SASRaidAdaptor) setBootIR() error { return nil } -func (adapter *Mpt2SASRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } -func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(conf.Size) > 1 { return fmt.Errorf("Subdivide sub-size not supported") } @@ -253,19 +254,19 @@ func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.Bar return nil } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("RAID0", devs, conf) } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("RAID1", devs, conf) } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return fmt.Errorf("Not impl") } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(devs) > 10 { return fmt.Errorf("RAID10 supports no more than 10 disks") } @@ -342,7 +343,7 @@ func (r *Mpt2SASRaid) GetCommand(args ...string) string { return raid.GetCommand(r.utility, args...) } -func (r *Mpt2SASRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *Mpt2SASRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/cloudcommon/cmdline/doc.go b/pkg/cloudcommon/cmdline/doc.go new file mode 100644 index 0000000000..af392c0b87 --- /dev/null +++ b/pkg/cloudcommon/cmdline/doc.go @@ -0,0 +1 @@ +package cmdline // import "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" diff --git a/pkg/cloudcommon/cmdline/helper.go b/pkg/cloudcommon/cmdline/helper.go new file mode 100644 index 0000000000..ee9184c081 --- /dev/null +++ b/pkg/cloudcommon/cmdline/helper.go @@ -0,0 +1,406 @@ +package cmdline + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/regutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/apis/scheduler" +) + +type resourceConfigOutput struct { + keyword string + keywordPlural string +} + +func newResouceConfigOutput(kw, kws string) *resourceConfigOutput { + return &resourceConfigOutput{keyword: kw, keywordPlural: kws} +} + +func (o resourceConfigOutput) Keyword() string { + return o.keyword +} + +func (o resourceConfigOutput) KeywordPlural() string { + return o.keywordPlural +} + +type iResourcesOutput interface { + Keyword() string + KeywordPlural() string + NewConfig() interface{} + ParseDesc(desc string, idx int) (interface{}, error) + Add(config interface{}) + Resources() interface{} +} + +func fetchResourceConfigsByJSON( + obj jsonutils.JSONObject, + output iResourcesOutput, +) error { + if obj == nil { + obj = jsonutils.NewDict() + } + keywordPlural := output.KeywordPlural() + keyword := output.Keyword() + if obj.Contains(keywordPlural) { + err := obj.Unmarshal(output.Resources(), keywordPlural) + return err + } + + configs := jsonutils.GetArrayOfPrefix(obj, keyword) + for idx, config := range configs { + configJson, ok := config.(*jsonutils.JSONDict) + resConfig := output.NewConfig() + if ok { + if err := configJson.Unmarshal(resConfig); err != nil { + return err + } + } else { + configStr, err := config.GetString() + if err != nil { + return err + } + resConfig, err = output.ParseDesc(configStr, idx) + if err != nil { + return err + } + } + output.Add(resConfig) + } + return nil +} + +type diskConfigOutput struct { + *resourceConfigOutput + disks []*compute.DiskConfig +} + +func newDiskConfigOutput() *diskConfigOutput { + return &diskConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("disk", "disks"), + } +} + +func (output *diskConfigOutput) Disks() []*compute.DiskConfig { + return output.disks +} + +func (output *diskConfigOutput) Resources() interface{} { + return &output.disks +} + +func (output *diskConfigOutput) NewConfig() interface{} { + return new(compute.DiskConfig) +} + +func (output *diskConfigOutput) Add(config interface{}) { + output.disks = append(output.disks, config.(*compute.DiskConfig)) +} + +func (output *diskConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseDiskConfig(desc, idx) +} + +func FetchDiskConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.DiskConfig, error) { + output := newDiskConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.Disks(), err +} + +type netConfigOutput struct { + *resourceConfigOutput + nets []*compute.NetworkConfig +} + +func newNetworkConfigOutput() *netConfigOutput { + return &netConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("net", "nets"), + } +} + +func (output *netConfigOutput) Networks() []*compute.NetworkConfig { + return output.nets +} + +func (output *netConfigOutput) Resources() interface{} { + return &output.nets +} + +func (output *netConfigOutput) NewConfig() interface{} { + return new(compute.NetworkConfig) +} + +func (output *netConfigOutput) Add(config interface{}) { + output.nets = append(output.nets, config.(*compute.NetworkConfig)) +} + +func (output *netConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseNetworkConfig(desc, idx) +} + +func FetchNetworkConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.NetworkConfig, error) { + output := newNetworkConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.Networks(), err +} + +type schedtagConfigOutput struct { + *resourceConfigOutput + tags []*compute.SchedtagConfig +} + +func newSchedtagConfigOutput() *schedtagConfigOutput { + return &schedtagConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("schedtag", "schedtags"), + } +} + +func (output *schedtagConfigOutput) Schedtags() []*compute.SchedtagConfig { + return output.tags +} + +func (output *schedtagConfigOutput) Resources() interface{} { + return &output.tags +} + +func (output *schedtagConfigOutput) NewConfig() interface{} { + return new(compute.SchedtagConfig) +} + +func (output *schedtagConfigOutput) Add(config interface{}) { + output.tags = append(output.tags, config.(*compute.SchedtagConfig)) +} + +func (output *schedtagConfigOutput) ParseDesc(desc string, _ int) (interface{}, error) { + return ParseSchedtagConfig(desc) +} + +func FetchSchedtagConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.SchedtagConfig, error) { + output := newSchedtagConfigOutput() + if err := fetchResourceConfigsByJSON(obj, output); err != nil { + return nil, err + } + if len(output.tags) == 0 { + // compatible with old api + schedtags, _ := obj.GetMap("aggregate_strategy") + if schedtags != nil { + for id, strategyObj := range schedtags { + strategy, _ := strategyObj.GetString() + output.tags = append(output.tags, &compute.SchedtagConfig{ + Id: id, + Strategy: strategy, + }) + } + } + } + return output.tags, nil +} + +type isoDevConfigOutput struct { + *resourceConfigOutput + devs []*compute.IsolatedDeviceConfig +} + +func newIsoDevConfigOutput() *isoDevConfigOutput { + return &isoDevConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("isolated_device", "isolated_devices"), + } +} + +func (output *isoDevConfigOutput) Devs() []*compute.IsolatedDeviceConfig { + return output.devs +} + +func (output *isoDevConfigOutput) Resources() interface{} { + return &output.devs +} + +func (output *isoDevConfigOutput) NewConfig() interface{} { + return new(compute.IsolatedDeviceConfig) +} + +func (output *isoDevConfigOutput) Add(config interface{}) { + output.devs = append(output.devs, config.(*compute.IsolatedDeviceConfig)) +} + +func (output *isoDevConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseIsolatedDevice(desc, idx) +} + +func FetchIsolatedDeviceConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.IsolatedDeviceConfig, error) { + output := newIsoDevConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.devs, err +} + +type bmDiskConfigOutput struct { + *resourceConfigOutput + configs []*compute.BaremetalDiskConfig +} + +func newbmDiskConfigOutput() *bmDiskConfigOutput { + return &bmDiskConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("baremetal_disk_config", "baremetal_disk_configs"), + } +} + +func (output *bmDiskConfigOutput) Devs() []*compute.BaremetalDiskConfig { + return output.configs +} + +func (output *bmDiskConfigOutput) Resources() interface{} { + return &output.configs +} + +func (output *bmDiskConfigOutput) NewConfig() interface{} { + return new(compute.BaremetalDiskConfig) +} + +func (output *bmDiskConfigOutput) Add(config interface{}) { + output.configs = append(output.configs, config.(*compute.BaremetalDiskConfig)) +} + +func (output *bmDiskConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseBaremetalDiskConfig(desc) +} + +func FetchBaremetalDiskConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.BaremetalDiskConfig, error) { + output := newbmDiskConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.configs, err +} + +func FetchServerConfigsByJSON(obj jsonutils.JSONObject) (*compute.ServerConfigs, error) { + conf := new(compute.ServerConfigs) + if err := obj.Unmarshal(conf); err != nil { + return nil, err + } + + var err error + conf.Disks, err = FetchDiskConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.Networks, err = FetchNetworkConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.Schedtags, err = FetchSchedtagConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.IsolatedDevices, err = FetchIsolatedDeviceConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.BaremetalDiskConfigs, err = FetchBaremetalDiskConfigsByJSON(obj) + if err != nil { + return nil, err + } + return conf, nil +} + +func FetchScheduleInputByJSON(obj jsonutils.JSONObject) (*scheduler.ScheduleInput, error) { + input := new(scheduler.ScheduleInput) + err := obj.Unmarshal(input) + if err != nil { + return nil, err + } + conf := &input.ServerConfig + conf.ServerConfigs, err = FetchServerConfigsByJSON(obj) + if err != nil { + return nil, err + } + return input, nil +} + +func FetchDeployConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.DeployConfig, error) { + deploys := make([]*compute.DeployConfig, 0) + if obj.Contains("deploy_configs") { + err := obj.Unmarshal(&deploys, "deploy_configs") + return deploys, err + } + for idx := 0; obj.Contains(fmt.Sprintf("deploy.%d.path", idx)); idx += 1 { + path, _ := obj.GetString(fmt.Sprintf("deploy.%d.path", idx)) + action, _ := obj.GetString(fmt.Sprintf("deploy.%d.action", idx)) + content, _ := obj.GetString(fmt.Sprintf("deploy.%d.content", idx)) + deploys = append(deploys, &compute.DeployConfig{Path: path, Action: action, Content: content}) + } + return deploys, nil +} + +func FetchServerCreateInputByJSON(obj jsonutils.JSONObject) (*compute.ServerCreateInput, error) { + input := new(compute.ServerCreateInput) + // compatible with vmem_size string + memSizeObj, _ := obj.Get("vmem_size") + if memSizeObj != nil { + if _, ok := memSizeObj.(*jsonutils.JSONString); ok { + memStr, err := memSizeObj.GetString() + if err != nil { + return nil, err + } + if !regutils.MatchSize(memStr) { + return nil, fmt.Errorf("Invalid memory size string: %s, Memory size must be number[+unit], like 256M, 1G or 256", memStr) + } + vmemSize, err := fileutils.GetSizeMb(memStr, 'M', 1024) + if err != nil { + return nil, err + } + obj.(*jsonutils.JSONDict).Set("vmem_size", jsonutils.NewInt(int64(vmemSize))) + } + } + + err := obj.Unmarshal(input) + if err != nil { + return nil, err + } + config, err := FetchServerConfigsByJSON(obj) + if err != nil { + return nil, err + } + input.ServerConfigs = config + + deployConfigs, err := FetchDeployConfigsByJSON(obj) + if err != nil { + return nil, err + } + input.DeployConfigs = deployConfigs + + if skuName := jsonutils.GetAnyString(obj, []string{"sku", "flavor", "instance_type"}); skuName != "" { + input.InstanceType = skuName + } + if keypair := jsonutils.GetAnyString(obj, []string{"keypair", "keypair_id"}); keypair != "" { + input.Keypair = keypair + } + if secgroup := jsonutils.GetAnyString(obj, []string{"secgroup", "secgroup_id", "secgrp_id"}); len(secgroup) != 0 { + input.Secgroup = secgroup + } + + return input, nil +} + +func FetchDiskCreateInputByJSON(data jsonutils.JSONObject) (*compute.DiskCreateInput, error) { + config := new(compute.DiskConfig) + input := &compute.DiskCreateInput{ + DiskConfig: config, + } + if err := data.Unmarshal(input); err != nil { + return nil, err + } + if data.Contains("disk") { + desc, err := data.GetString("disk") + config, err = ParseDiskConfig(desc, -1) + if err != nil { + return nil, err + } + input.DiskConfig = config + } + if storageId := jsonutils.GetAnyString(data, []string{"storage_id", "storage"}); storageId != "" { + input.Storage = storageId + } + return input, nil +} diff --git a/pkg/cloudcommon/cmdline/helper_test.go b/pkg/cloudcommon/cmdline/helper_test.go new file mode 100644 index 0000000000..a03e8c62ed --- /dev/null +++ b/pkg/cloudcommon/cmdline/helper_test.go @@ -0,0 +1,87 @@ +package cmdline + +import ( + "reflect" + "testing" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +func TestFetchServerConfigsByJSON(t *testing.T) { + type args struct { + obj jsonutils.JSONObject + } + tests := []struct { + name string + args args + want *compute.ServerConfigs + wantErr bool + }{ + { + name: "all configs", + args: args{jsonutils.Marshal(map[string]string{ + "disk.0": "1g:centos", + "net.0": "192.168.222.3:inf0", + "schedtag.0": "ssd:require", + "schedtag.1": "container:exclude", + "isolated_device.0": "vendor=nvidia:p400", + "baremetal_disk_config.0": "raid0:[1,2]", + })}, + want: &compute.ServerConfigs{ + Disks: []*compute.DiskConfig{ + { + Index: 0, + SizeMb: 1024, + ImageId: "centos", + Medium: "hybrid", + }, + }, + Networks: []*compute.NetworkConfig{ + { + Network: "inf0", + Address: "192.168.222.3", + }, + }, + Schedtags: []*compute.SchedtagConfig{ + { + Id: "ssd", + Strategy: "require", + }, + { + Id: "container", + Strategy: "exclude", + }, + }, + IsolatedDevices: []*compute.IsolatedDeviceConfig{ + { + Vendor: "nvidia", + Model: "p400", + }, + }, + BaremetalDiskConfigs: []*compute.BaremetalDiskConfig{ + { + Type: "hybrid", + Conf: "raid0", + Range: []int64{1, 2}, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FetchServerConfigsByJSON(tt.args.obj) + if (err != nil) != tt.wantErr { + t.Errorf("FetchServerConfigsByJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotJ := jsonutils.Marshal(got).String() + wantJ := jsonutils.Marshal(tt.want).String() + if !reflect.DeepEqual(gotJ, wantJ) { + t.Errorf("FetchServerConfigsByJSON() = %v, want %v", gotJ, wantJ) + } + }) + } +} diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go new file mode 100644 index 0000000000..23cd42e3f6 --- /dev/null +++ b/pkg/cloudcommon/cmdline/parser.go @@ -0,0 +1,349 @@ +package cmdline + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/apis/compute" +) + +var ( + ErrorEmptyDesc = errors.New("Empty description") +) + +// ParseSchedtagConfig desc format: : +func ParseSchedtagConfig(desc string) (*compute.SchedtagConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("Invalid desc: %s", desc) + } + strategy := parts[1] + if !utils.IsInStringArray(strategy, compute.STRATEGY_LIST) { + return nil, fmt.Errorf("Invalid strategy: %s", strategy) + } + return &compute.SchedtagConfig{ + Id: parts[0], + Strategy: parts[1], + }, nil +} + +// ParseResourceSchedtagConfig desc format: :: +func ParseResourceSchedtagConfig(desc string) (int, *compute.SchedtagConfig, error) { + if len(desc) == 0 { + return 0, nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + if len(parts) != 3 { + return 0, nil, fmt.Errorf("Invalid desc: %s", desc) + } + idx, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, nil, err + } + tag, err := ParseSchedtagConfig(fmt.Sprintf("%s:%s", parts[1], parts[2])) + if err != nil { + return 0, nil, err + } + return idx, tag, nil +} + +func ParseDiskConfig(diskStr string, idx int) (*compute.DiskConfig, error) { + if len(diskStr) == 0 { + return nil, ErrorEmptyDesc + } + diskConfig := new(compute.DiskConfig) + diskConfig.Index = idx + + // default backend and medium type + diskConfig.Backend = "" // STORAGE_LOCAL + diskConfig.Medium = compute.DISK_TYPE_HYBRID + + parts := strings.Split(diskStr, ":") + for _, p := range parts { + if len(p) == 0 { + continue + } + if regutils.MatchSize(p) { + diskConfig.SizeMb, _ = fileutils.GetSizeMb(p, 'M', 1024) + } else if utils.IsInStringArray(p, osprofile.FS_TYPES) { + diskConfig.Fs = p + } else if utils.IsInStringArray(p, osprofile.IMAGE_FORMAT_TYPES) { + diskConfig.Format = p + } else if utils.IsInStringArray(p, osprofile.DISK_DRIVERS) { + diskConfig.Driver = p + } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { + diskConfig.Cache = p + } else if utils.IsInStringArray(p, compute.DISK_TYPES) { + diskConfig.Medium = p + } else if utils.IsInStringArray(p, []string{compute.DISK_TYPE_VOLUME}) { + diskConfig.DiskType = p + } else if p[0] == '/' { + diskConfig.Mountpoint = p + } else if p == "autoextend" { + diskConfig.SizeMb = -1 + } else if strings.HasPrefix(p, "snapshot-") { + // HACK: use snapshot creat disk format snapshot-id + // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b + diskConfig.SnapshotId = p[len("snapshot-"):] + } else if len(p) > 0 { + diskConfig.ImageId = p + } + } + + return diskConfig, nil +} + +func ParseNetworkConfigByJSON(desc jsonutils.JSONObject, idx int) (*compute.NetworkConfig, error) { + if _, ok := desc.(*jsonutils.JSONString); ok { + descStr, _ := desc.GetString() + return ParseNetworkConfig(descStr, idx) + } + conf := new(compute.NetworkConfig) + conf.Index = idx + err := desc.Unmarshal(conf) + return conf, err +} + +func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + netConfig := new(compute.NetworkConfig) + netConfig.Index = idx + for _, p := range parts { + if len(p) == 0 { + continue + } + if regutils.MatchIP4Addr(p) { + netConfig.Address = p + } else if regutils.MatchIP6Addr(p) { + netConfig.Address6 = p + } else if regutils.MatchCompactMacAddr(p) { + netConfig.Mac = netutils.MacUnpackHex(p) + } else if strings.HasPrefix(p, "wire=") { + netConfig.Wire = p[len("wire="):] + } else if p == "[random_exit]" { + netConfig.Exit = true + } else if p == "[random]" { + netConfig.Exit = false + } else if p == "[private]" { + netConfig.Private = true + } else if p == "[reserved]" { + netConfig.Reserved = true + } else if p == "[teaming]" { + netConfig.RequireTeaming = true + } else if p == "[try-teaming]" { + netConfig.TryTeaming = true + } else if strings.HasPrefix(p, "standby-port=") { + netConfig.StandbyPortCount, _ = strconv.Atoi(p[len("standby-port="):]) + } else if strings.HasPrefix(p, "standby-addr=") { + netConfig.StandbyAddrCount, _ = strconv.Atoi(p[len("standby-addr="):]) + } else if utils.IsInStringArray(p, []string{"virtio", "e1000", "vmxnet3"}) { + netConfig.Driver = p + } else if regutils.MatchSize(p) { + bw, err := fileutils.GetSizeMb(p, 'M', 1000) + if err != nil { + return nil, err + } + netConfig.BwLimit = bw + } else if p == "[vip]" { + netConfig.Vip = true + } else if utils.IsInStringArray(p, compute.ALL_NETWORK_TYPES) { + netConfig.NetType = p + } else { + netConfig.Network = p + } + } + return netConfig, nil +} + +func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + if idx < 0 { + return nil, fmt.Errorf("Invalid index: %d", idx) + } + dev := new(compute.IsolatedDeviceConfig) + parts := strings.Split(desc, ":") + for _, p := range parts { + if regutils.MatchUUIDExact(p) { + dev.Id = p + } else if utils.IsInStringArray(p, compute.VALID_PASSTHROUGH_TYPES) { + dev.DevType = p + } else if strings.HasPrefix(p, "vendor=") { + dev.Vendor = p[len("vendor="):] + } else { + dev.Model = p + } + } + return dev, nil +} + +func ParseBaremetalDiskConfig(desc string) (*compute.BaremetalDiskConfig, error) { + bdc := new(compute.BaremetalDiskConfig) + bdc.Type = compute.DISK_TYPE_HYBRID + bdc.Conf = compute.DISK_CONF_NONE + bdc.Count = 0 + desc = strings.ToLower(desc) + if len(desc) == 0 { + return bdc, nil + } + + parts := strings.Split(desc, ":") + drvMap := make(map[string]string) + for _, drv := range compute.DISK_DRIVERS.List() { + drvMap[strings.ToLower(drv)] = drv + } + + for _, p := range parts { + if len(p) == 0 { + continue + } else if utils.IsInStringArray(p, compute.DISK_TYPES) { + bdc.Type = p + } else if compute.DISK_CONFS.Has(p) { + bdc.Conf = p + } else if drv, ok := drvMap[p]; ok { + bdc.Driver = drv + } else if utils.IsMatchInteger(p) { + bdc.Count, _ = strconv.ParseInt(p, 0, 0) + } else if len(p) > 2 && p[0] == '[' && p[len(p)-1] == ']' { + rg, err1 := ParseRange(p[1:(len(p) - 1)]) + if err1 != nil { + return nil, err1 + } + bdc.Range = rg + } else if len(p) > 2 && p[0] == '(' && p[len(p)-1] == ')' { + bdc.Splits = p[1 : len(p)-1] + } else if utils.HasPrefix(p, "strip") { + strip := parseStrip(p[len("strip"):], "k") + bdc.Strip = &strip + } else if utils.HasPrefix(p, "adapter") { + ada, _ := strconv.ParseInt(p[len("adapter"):], 0, 64) + pada := int(ada) + bdc.Adapter = &pada + } else if p == "ra" { + hasRA := true + bdc.RA = &hasRA + } else if p == "nora" { + noRA := false + bdc.RA = &noRA + } else if p == "wt" { + wt := true + bdc.WT = &wt + } else if p == "wb" { + wt := false + bdc.WT = &wt + } else if p == "direct" { + direct := true + bdc.Direct = &direct + } else if p == "cached" { + direct := false + bdc.Direct = &direct + } else if p == "cachedbadbbu" { + cached := true + bdc.Cachedbadbbu = &cached + } else if p == "nocachedbadbbu" { + cached := false + bdc.Cachedbadbbu = &cached + } else { + return nil, fmt.Errorf("ParseDiskConfig unkown option %q", p) + } + } + + return bdc, nil +} + +func ParseRange(rangeStr string) (ret []int64, err error) { + rss := regexp.MustCompile(`[\s,]+`).Split(rangeStr, -1) + intSet := sets.NewInt64() + + for _, rs := range rss { + r, err1 := _parseRange(rs) + if err1 != nil { + err = err1 + return + } + intSet.Insert(r...) + } + ret = intSet.List() + return +} + +// range string should be: "1-3", "3" +func _parseRange(str string) (ret []int64, err error) { + if len(str) == 0 { + return + } + + // exclude "," symbol + if len(str) == 1 && !utils.IsMatchInteger(str) { + return + } + + // add int string + if utils.IsMatchInteger(str) { + i, _ := strconv.ParseInt(str, 10, 64) + ret = append(ret, i) + return + } + + // add rang like string, "2-10" etc. + ret, err = parseRangeStr(str) + return +} + +// return bytes +func parseStrip(stripStr string, defaultSize string) int64 { + size, _ := utils.GetSize(stripStr, defaultSize, 1024) + return size / 1024 +} + +func parseRangeStr(str string) (ret []int64, err error) { + im := utils.IsMatchInteger + errGen := func(e string) error { + return fmt.Errorf("Incorrect range str: %q", e) + } + rs := strings.Split(str, "-") + if len(rs) != 2 { + err = errGen(str) + return + } + + bs, es := rs[0], rs[1] + if !im(bs) { + err = errGen(str) + return + } + if !im(es) { + err = errGen(str) + return + } + + begin, _ := strconv.ParseInt(bs, 10, 64) + end, _ := strconv.ParseInt(es, 10, 64) + + if begin > end { + begin, end = end, begin + } + + for i := begin; i <= end; i++ { + ret = append(ret, i) + } + return +} diff --git a/pkg/cloudcommon/cmdline/parser_test.go b/pkg/cloudcommon/cmdline/parser_test.go new file mode 100644 index 0000000000..807a4b6eff --- /dev/null +++ b/pkg/cloudcommon/cmdline/parser_test.go @@ -0,0 +1,177 @@ +package cmdline + +import ( + "reflect" + "testing" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +func TestParseSchedtagConfig(t *testing.T) { + type args struct { + desc string + } + tests := []struct { + name string + args args + want *compute.SchedtagConfig + wantErr bool + }{ + { + name: "empty input", + args: args{""}, + want: nil, + wantErr: true, + }, + { + name: "normal", + args: args{"ssd:require"}, + want: &compute.SchedtagConfig{Id: "ssd", Strategy: "require"}, + wantErr: false, + }, + { + name: "invalid strategy", + args: args{"ssd:require2"}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSchedtagConfig(tt.args.desc) + if (err != nil) != tt.wantErr { + t.Errorf("ParseSchedtagConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseSchedtagConfig() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseRange(t *testing.T) { + type args struct { + rangeStr string + } + tests := []struct { + name string + args args + wantRet []int64 + wantErr bool + }{ + { + name: "range number and , test", + args: args{"11-7,, 9-9, 10, 15, 1-2,4-5,,"}, + wantRet: []int64{1, 2, 4, 5, 7, 8, 9, 10, 11, 15}, + wantErr: false, + }, + { + name: "range,range", + args: args{"1-2,4-5"}, + wantRet: []int64{1, 2, 4, 5}, + wantErr: false, + }, + { + name: "numbers", + args: args{"1, 1"}, + wantRet: []int64{1}, + wantErr: false, + }, + { + name: "numbersErr", + args: args{"1-, 1"}, + wantRet: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRet, err := ParseRange(tt.args.rangeStr) + if (err != nil) != tt.wantErr { + t.Errorf("ParseRange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRet, tt.wantRet) { + t.Errorf("ParseRange() = %v, want %v", gotRet, tt.wantRet) + } + }) + } +} + +func TestFetchDiskConfigsByJSON(t *testing.T) { + type args struct { + obj jsonutils.JSONObject + } + + testDisks := []*compute.DiskConfig{ + { + SizeMb: 1024, + Medium: compute.DISK_TYPE_SSD, + }, + { + SizeMb: 2048, + Medium: compute.DISK_TYPE_ROTATE, + }, + } + + testDisksObj := map[string][]*compute.DiskConfig{"disks": testDisks} + + tests := []struct { + name string + args args + want []*compute.DiskConfig + wantErr bool + }{ + { + name: "empty", + args: args{nil}, + want: nil, + wantErr: false, + }, + { + name: "parse strings", + args: args{obj: jsonutils.Marshal( + map[string]string{ + "disk.0": "10g:/data1", + "disk.1": "20g:ext4:ssd", + }, + )}, + want: []*compute.DiskConfig{ + { + Index: 0, + SizeMb: 10 * 1024, + Mountpoint: "/data1", + Medium: compute.DISK_TYPE_HYBRID, + }, + { + Index: 1, + SizeMb: 20 * 1024, + Fs: "ext4", + Medium: compute.DISK_TYPE_SSD, + }, + }, + wantErr: false, + }, + { + name: "parse struct", + args: args{obj: jsonutils.Marshal(testDisksObj)}, + want: testDisks, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FetchDiskConfigsByJSON(tt.args.obj) + if (err != nil) != tt.wantErr { + t.Errorf("FetchDiskConfigsByJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FetchDiskConfigsByJSON() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/compute/baremetal/diskconfig.go b/pkg/compute/baremetal/diskconfig.go index 9eae1f6808..fd1cd6d4b5 100644 --- a/pkg/compute/baremetal/diskconfig.go +++ b/pkg/compute/baremetal/diskconfig.go @@ -4,171 +4,27 @@ import ( "encoding/json" "fmt" "reflect" - "regexp" "strconv" "strings" "yunion.io/x/log" "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" ) -// return bytes -func parseStrip(stripStr string, defaultSize string) int64 { - size, _ := utils.GetSize(stripStr, defaultSize, 1024) - return size / 1024 -} - -func parseRangeStr(str string) (ret []int64, err error) { - im := utils.IsMatchInteger - errGen := func(e string) error { - return fmt.Errorf("Incorrect range str: %q", e) +func ParseDiskConfig(desc string) (api.BaremetalDiskConfig, error) { + conf, err := cmdline.ParseBaremetalDiskConfig(desc) + if err != nil { + return api.BaremetalDiskConfig{}, err } - rs := strings.Split(str, "-") - if len(rs) != 2 { - err = errGen(str) - return - } - - bs, es := rs[0], rs[1] - if !im(bs) { - err = errGen(str) - return - } - if !im(es) { - err = errGen(str) - return - } - - begin, _ := strconv.ParseInt(bs, 10, 64) - end, _ := strconv.ParseInt(es, 10, 64) - - if begin > end { - begin, end = end, begin - } - - for i := begin; i <= end; i++ { - ret = append(ret, i) - } - return -} - -// range string should be: "1-3", "3" -func _parseRange(str string) (ret []int64, err error) { - if len(str) == 0 { - return - } - - // exclude "," symbol - if len(str) == 1 && !utils.IsMatchInteger(str) { - return - } - - // add int string - if utils.IsMatchInteger(str) { - i, _ := strconv.ParseInt(str, 10, 64) - ret = append(ret, i) - return - } - - // add rang like string, "2-10" etc. - ret, err = parseRangeStr(str) - return -} - -func ParseRange(rangeStr string) (ret []int64, err error) { - rss := regexp.MustCompile(`[\s,]+`).Split(rangeStr, -1) - intSet := sets.NewInt64() - - for _, rs := range rss { - r, err1 := _parseRange(rs) - if err1 != nil { - err = err1 - return - } - intSet.Insert(r...) - } - ret = intSet.List() - return -} - -func ParseDiskConfig(desc string) (bdc BaremetalDiskConfig, err error) { - bdc.Type = DISK_TYPE_HYBRID - bdc.Conf = DISK_CONF_NONE - bdc.Count = 0 - - desc = strings.ToLower(desc) - if desc == "" { - return - } - - parts := strings.Split(desc, ":") - drvMap := make(map[string]string) - for _, drv := range DISK_DRIVERS.List() { - drvMap[strings.ToLower(drv)] = drv - } - for _, p := range parts { - if len(p) == 0 { - continue - } else if DISK_TYPES.Has(p) { - bdc.Type = p - } else if DISK_CONFS.Has(p) { - bdc.Conf = p - } else if drv, ok := drvMap[p]; ok { - bdc.Driver = drv - } else if utils.IsMatchInteger(p) { - bdc.Count, _ = strconv.ParseInt(p, 0, 0) - } else if len(p) > 2 && p[0] == '[' && p[len(p)-1] == ']' { - rg, err1 := ParseRange(p[1:(len(p) - 1)]) - if err1 != nil { - err = err1 - return - } - bdc.Range = rg - } else if len(p) > 2 && p[0] == '(' && p[len(p)-1] == ')' { - bdc.Splits = p[1 : len(p)-1] - } else if utils.HasPrefix(p, "strip") { - strip := parseStrip(p[len("strip"):], "k") - bdc.Strip = &strip - } else if utils.HasPrefix(p, "adapter") { - ada, _ := strconv.ParseInt(p[len("adapter"):], 0, 64) - pada := int(ada) - bdc.Adapter = &pada - } else if p == "ra" { - hasRA := true - bdc.RA = &hasRA - } else if p == "nora" { - noRA := false - bdc.RA = &noRA - } else if p == "wt" { - wt := true - bdc.WT = &wt - } else if p == "wb" { - wt := false - bdc.WT = &wt - } else if p == "direct" { - direct := true - bdc.Direct = &direct - } else if p == "cached" { - direct := false - bdc.Direct = &direct - } else if p == "cachedbadbbu" { - cached := true - bdc.Cachedbadbbu = &cached - } else if p == "nocachedbadbbu" { - cached := false - bdc.Cachedbadbbu = &cached - } else { - err = fmt.Errorf("ParseDiskConfig unkown option %q", p) - return - } - } - - return + return *conf, nil } func isDiskConfigStorageMatch( - config *BaremetalDiskConfig, + config *api.BaremetalDiskConfig, confDriver *string, confAdapter *int, storage *BaremetalStorage, @@ -198,7 +54,7 @@ func isDiskConfigStorageMatch( return false } -func RetrieveStorages(diskConfig *BaremetalDiskConfig, storages []*BaremetalStorage) (selected, rest []*BaremetalStorage) { +func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*BaremetalStorage) (selected, rest []*BaremetalStorage) { var confDriver *string = nil var confAdapter *int = nil @@ -259,9 +115,9 @@ func RequireEvenDisks(diskConfig string) bool { } type Layout struct { - Disks []*BaremetalStorage `json:"disks"` - Conf *BaremetalDiskConfig `json:"conf"` - Size int64 `json:"size"` + Disks []*BaremetalStorage `json:"disks"` + Conf *api.BaremetalDiskConfig `json:"conf"` + Size int64 `json:"size"` } func (l Layout) String() string { @@ -280,7 +136,7 @@ func RetrieveStorageDrivers(storages []*BaremetalStorage) sets.String { } func MeetConfig( - conf *BaremetalDiskConfig, + conf *api.BaremetalDiskConfig, storages []*BaremetalStorage, ) error { storageDrvs := RetrieveStorageDrivers(storages) @@ -417,9 +273,9 @@ func ExpandNoneConf(layouts []Layout) (ret []Layout) { return ret } -func GetLayoutRaidConfig(layouts []Layout) []*BaremetalDiskConfig { +func GetLayoutRaidConfig(layouts []Layout) []*api.BaremetalDiskConfig { var disk []*BaremetalStorage - ret := make([]*BaremetalDiskConfig, 0) + ret := make([]*api.BaremetalDiskConfig, 0) for _, layout := range layouts { if layout.Conf.Conf == DISK_CONF_NONE && sets.NewString(DISK_DRIVER_LINUX, DISK_DRIVER_PCIE).Has(layout.Disks[0].Driver) { @@ -444,10 +300,10 @@ func GetLayoutRaidConfig(layouts []Layout) []*BaremetalDiskConfig { return ret } -func CalculateLayout(confs []*BaremetalDiskConfig, storages []*BaremetalStorage) (layouts []Layout, err error) { +func CalculateLayout(confs []*api.BaremetalDiskConfig, storages []*BaremetalStorage) (layouts []Layout, err error) { var confIdx = 0 for len(storages) > 0 { - var conf *BaremetalDiskConfig + var conf *api.BaremetalDiskConfig if confIdx < len(confs) { conf = confs[confIdx] confIdx += 1 @@ -544,8 +400,8 @@ func CheckDisksAllocable(layouts []Layout, disks []*Disk) bool { return true } -func NewBaremetalDiskConfigs(dss ...string) ([]*BaremetalDiskConfig, error) { - ret := make([]*BaremetalDiskConfig, 0) +func NewBaremetalDiskConfigs(dss ...string) ([]*api.BaremetalDiskConfig, error) { + ret := make([]*api.BaremetalDiskConfig, 0) for _, ds := range dss { r, err := ParseDiskConfig(ds) if err != nil { @@ -671,7 +527,7 @@ func GetDiskConfigurations(layouts []Layout) []DiskConfiguration { type DriverAdapterDiskConfig struct { Driver string Adapter int - Configs []*BaremetalDiskConfig + Configs []*api.BaremetalDiskConfig } func GroupLayoutResultsByDriverAdapter(layouts []Layout) []*DriverAdapterDiskConfig { @@ -687,7 +543,7 @@ func GroupLayoutResultsByDriverAdapter(layouts []Layout) []*DriverAdapterDiskCon item := &DriverAdapterDiskConfig{ Driver: driver, Adapter: adapter, - Configs: []*BaremetalDiskConfig{layout.Conf}, + Configs: []*api.BaremetalDiskConfig{layout.Conf}, } ret = append(ret, item) tbl[key] = item diff --git a/pkg/compute/baremetal/diskconfig_test.go b/pkg/compute/baremetal/diskconfig_test.go index 388030f4f2..03e85fb797 100644 --- a/pkg/compute/baremetal/diskconfig_test.go +++ b/pkg/compute/baremetal/diskconfig_test.go @@ -6,6 +6,8 @@ import ( "testing" "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/compute" ) func TestParseDiskConfig(t *testing.T) { @@ -22,13 +24,13 @@ func TestParseDiskConfig(t *testing.T) { tests := []struct { name string args args - wantBdc BaremetalDiskConfig + wantBdc api.BaremetalDiskConfig wantErr bool }{ { name: "rotate:[1-2,4-5]:MegaRaid", args: args{"rotate:[1-2,4-5]:MegaRaid"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_NONE, Driver: DISK_DRIVER_MEGARAID, @@ -40,7 +42,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[1-2,4,6]:raid10:marvelraid", args: args{"rotate:[1-2,4,6]:raid10:marvelraid"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Driver: DISK_DRIVER_MARVELRAID, @@ -52,7 +54,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4,6]:raid10", args: args{"rotate:[4,6]:raid10"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -63,7 +65,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4]:raid10:(40%, )", args: args{"rotate:[4]:raid10:(40%, )"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -75,7 +77,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "[12-13]:raid1:(100g,32g,):adapter0:strip64k", args: args{"[12-13]:raid1:(100g,32g,):adapter0:strip64k"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID1, Count: 0, @@ -89,7 +91,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "6:raid5:adapter2", args: args{"6:raid5:adapter2"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID5, Count: 6, @@ -112,55 +114,6 @@ func TestParseDiskConfig(t *testing.T) { } } -func TestParseRange(t *testing.T) { - type args struct { - rangeStr string - } - tests := []struct { - name string - args args - wantRet []int64 - wantErr bool - }{ - { - name: "range number and , test", - args: args{"11-7,, 9-9, 10, 15, 1-2,4-5,,"}, - wantRet: []int64{1, 2, 4, 5, 7, 8, 9, 10, 11, 15}, - wantErr: false, - }, - { - name: "range,range", - args: args{"1-2,4-5"}, - wantRet: []int64{1, 2, 4, 5}, - wantErr: false, - }, - { - name: "numbers", - args: args{"1, 1"}, - wantRet: []int64{1}, - wantErr: false, - }, - { - name: "numbersErr", - args: args{"1-, 1"}, - wantRet: nil, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotRet, err := ParseRange(tt.args.rangeStr) - if (err != nil) != tt.wantErr { - t.Errorf("ParseRange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(gotRet, tt.wantRet) { - t.Errorf("ParseRange() = %v, want %v", gotRet, tt.wantRet) - } - }) - } -} - var ( testStorages = []*BaremetalStorage{ // 0-11 disk on adapter2 @@ -516,7 +469,7 @@ func TestCheckDisksAllocable(t *testing.T) { } layout, err := CalculateLayout(confs, testStorages) - defaultLayout, err := CalculateLayout([]*BaremetalDiskConfig{&BaremetalDefaultDiskConfig}, testStorages[12:]) + defaultLayout, err := CalculateLayout([]*api.BaremetalDiskConfig{&BaremetalDefaultDiskConfig}, testStorages[12:]) if err != nil { t.Fatalf("CalculateLayout err: %v", err) } diff --git a/pkg/compute/baremetal/types.go b/pkg/compute/baremetal/types.go index 384178d9c6..9f8bc930b2 100644 --- a/pkg/compute/baremetal/types.go +++ b/pkg/compute/baremetal/types.go @@ -3,66 +3,45 @@ package baremetal import ( "errors" - "yunion.io/x/pkg/util/sets" + api "yunion.io/x/onecloud/pkg/apis/compute" ) const ( - DISK_CONF_RAID0 = "raid0" - DISK_CONF_RAID1 = "raid1" - DISK_CONF_RAID5 = "raid5" - DISK_CONF_RAID10 = "raid10" - DISK_CONF_NONE = "none" + DISK_CONF_RAID0 = api.DISK_CONF_RAID0 + DISK_CONF_RAID1 = api.DISK_CONF_RAID1 + DISK_CONF_RAID5 = api.DISK_CONF_RAID5 + DISK_CONF_RAID10 = api.DISK_CONF_RAID10 + DISK_CONF_NONE = api.DISK_CONF_NONE - DEFAULT_DISK_CONF = DISK_CONF_NONE + DEFAULT_DISK_CONF = api.DEFAULT_DISK_CONF - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" - DISK_TYPE_HYBRID = "hybrid" + DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE + DISK_TYPE_SSD = api.DISK_TYPE_SSD + DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID - DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + DEFAULT_DISK_TYPE = api.DEFAULT_DISK_TYPE - DISK_DRIVER_MEGARAID = "MegaRaid" - DISK_DRIVER_LINUX = "Linux" - DISK_DRIVER_HPSARAID = "HPSARaid" - DISK_DRIVER_MPT2SAS = "Mpt2SAS" - DISK_DRIVER_MARVELRAID = "MarvelRaid" - DISK_DRIVER_PCIE = "PCIE" + DISK_DRIVER_MEGARAID = api.DISK_DRIVER_MEGARAID + DISK_DRIVER_LINUX = api.DISK_DRIVER_LINUX + DISK_DRIVER_HPSARAID = api.DISK_DRIVER_HPSARAID + DISK_DRIVER_MPT2SAS = api.DISK_DRIVER_MPT2SAS + DISK_DRIVER_MARVELRAID = api.DISK_DRIVER_MARVELRAID + DISK_DRIVER_PCIE = api.DISK_DRIVER_PCIE - HDD_DISK_SPEC_TYPE = "HDD" - SSD_DISK_SPEC_TYPE = "SSD" + HDD_DISK_SPEC_TYPE = api.HDD_DISK_SPEC_TYPE + SSD_DISK_SPEC_TYPE = api.SSD_DISK_SPEC_TYPE ) var ( - BaremetalDefaultDiskConfig = BaremetalDiskConfig{ - Type: DISK_TYPE_HYBRID, - Conf: DISK_CONF_NONE, - Count: 0, - } + BaremetalDefaultDiskConfig = api.BaremetalDefaultDiskConfig - DISK_CONFS = sets.NewString( - DISK_CONF_RAID0, - DISK_CONF_RAID1, - DISK_CONF_RAID5, - DISK_CONF_RAID10, - DISK_CONF_NONE, - ) + DISK_CONFS = api.DISK_CONFS - DISK_TYPES = sets.NewString( - DISK_TYPE_ROTATE, - DISK_TYPE_SSD, - DISK_TYPE_HYBRID, - ) + DISK_TYPES = api.DISK_TYPES - DISK_DRIVERS_RAID = sets.NewString( - DISK_DRIVER_MEGARAID, - DISK_DRIVER_HPSARAID, - DISK_DRIVER_MPT2SAS, - DISK_DRIVER_MARVELRAID, - ) + DISK_DRIVERS_RAID = api.DISK_DRIVERS_RAID - DISK_DRIVERS = sets.NewString( - DISK_DRIVER_LINUX, - DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) + DISK_DRIVERS = api.DISK_DRIVERS ) var ( @@ -99,24 +78,6 @@ func (s BaremetalStorage) GetBlock() int64 { return s.Block } -type BaremetalDiskConfig struct { - // disk type - Type string `json:"type"` - // raid config - Conf string `json:"conf"` - Count int64 `json:"count"` - Range []int64 `json:"range"` - Splits string `json:"splits"` - Size []int64 `json:"size"` - Adapter *int `json:"adapter,omitempty"` - Driver string `json:"driver"` - Cachedbadbbu *bool `json:"cachedbadbbu,omitempty"` - Strip *int64 `json:"strip,omitempty"` - RA *bool `json:"ra,omitempty"` - WT *bool `json:"wt,omitempty"` - Direct *bool `json:"direct,omitempty"` -} - type Disk struct { Backend string `json:"backend"` ImageID string `json:"image_id"` diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index dda30729f8..95bc091a6b 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -36,6 +36,16 @@ func (self *SAliyunGuestDriver) GetMinimalSysDiskSizeGb() int { return 20 } +func (self *SAliyunGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_CLOUD_EFFICIENCY, + models.STORAGE_CLOUD_SSD, + models.STORAGE_CLOUD_ESSD, + models.STORAGE_PUBLIC_CLOUD, + models.STORAGE_EPHEMERAL_SSD, + } +} + func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -43,13 +53,7 @@ func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend st return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_CLOUD_EFFICIENCY, - models.STORAGE_CLOUD_SSD, - models.STORAGE_CLOUD_ESSD, - models.STORAGE_PUBLIC_CLOUD, - models.STORAGE_EPHEMERAL_SSD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -96,19 +100,15 @@ func (self *SAliyunGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - for i := 0; data.Contains(fmt.Sprintf("disk.%d", i)); i++ { - disk := models.SDiskConfig{} - if err := data.Unmarshal(&disk, fmt.Sprintf("disk.%d", i)); err != nil { - return nil, httperrors.NewInputParameterError("invalid diskinfo of index %d", i) - } + for i, disk := range input.Disks { if i == 0 && (disk.SizeMb < 20*1024 || disk.SizeMb > 500*1024) { return nil, httperrors.NewInputParameterError("The system disk size must be in the range of 20GB ~ 500Gb") } @@ -127,7 +127,7 @@ func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred } } } - return data, nil + return input, nil } func (self *SAliyunGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index f473b4c871..1d50455754 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -69,6 +69,16 @@ func (self *SAwsGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SAwsGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_GP2_SSD, + models.STORAGE_IO1_SSD, + models.STORAGE_ST1_HDD, + models.STORAGE_SC1_HDD, + models.STORAGE_STANDARD_HDD, + } +} + func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -77,13 +87,7 @@ func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend strin } } - for _, stype := range []string{ - models.STORAGE_GP2_SSD, - models.STORAGE_IO1_SSD, - models.STORAGE_ST1_HDD, - models.STORAGE_SC1_HDD, - models.STORAGE_STANDARD_HDD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -117,8 +121,8 @@ func (self *SAwsGuestDriver) RequestDetachDisk(ctx context.Context, guest *model return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) } func (self *SAwsGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index e1f558ab61..aef2e4b74e 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -9,6 +9,8 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -40,6 +42,14 @@ func (self *SAzureGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SAzureGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_STANDARD_LRS, + models.STORAGE_STANDARDSSD_LRS, + models.STORAGE_PREMIUM_LRS, + } +} + func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -47,11 +57,7 @@ func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend str return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_STANDARD_LRS, - models.STORAGE_STANDARDSSD_LRS, - models.STORAGE_PREMIUM_LRS, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -102,26 +108,30 @@ func (self *SAzureGuestDriver) RequestDetachDisk(ctx context.Context, guest *mod return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - return data, nil + return input, nil } func (self *SAzureGuestDriver) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) + input, err := cmdline.FetchServerCreateInputByJSON(data) if err != nil { return nil, err } - if data.Contains("name") { + input, err = self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) + if err != nil { + return nil, err + } + if input.Name != "" { return nil, httperrors.NewInputParameterError("cannot support change azure instance name") } - return data, nil + return input.JSON(input), nil } func (self *SAzureGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index 4caebf8a72..101fb44c68 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -55,32 +56,21 @@ func (self *SBaremetalGuestDriver) GetMaxVMemSizeGB() int { return 4096 } -func (self *SBaremetalGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params *jsonutils.JSONDict) error { +func (self *SBaremetalGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, confs []*api.BaremetalDiskConfig) error { baremetalStorage := models.ConvertStorageInfo2BaremetalStorages(host.StorageInfo) if baremetalStorage == nil { return fmt.Errorf("Convert storage info error") } - confs, err := params.GetArray("baremetal_disk_config") - if err != nil { + if len(confs) == 0 { parsedConf, _ := baremetal.ParseDiskConfig("") - nConfs := []*baremetal.BaremetalDiskConfig{&parsedConf} + nConfs := []*api.BaremetalDiskConfig{&parsedConf} layouts, err := baremetal.CalculateLayout(nConfs, baremetalStorage) if err != nil { return err } return host.UpdateDiskConfig(userCred, layouts) } else { - var nConfs = make([]*baremetal.BaremetalDiskConfig, 0) - for i := 0; i < len(confs); i++ { - parsedConf := &baremetal.BaremetalDiskConfig{} - err := confs[i].Unmarshal(parsedConf) - if err != nil { - log.Errorln(err) - return err - } - nConfs = append(nConfs, parsedConf) - } - layouts, err := baremetal.CalculateLayout(nConfs, baremetalStorage) + layouts, err := baremetal.CalculateLayout(confs, baremetalStorage) if err != nil { return err } @@ -104,7 +94,7 @@ func (self *SBaremetalGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk return httperrors.NewUnsupportOperationError("Cannot resize disk for baremtal") } -func (self *SBaremetalGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { +func (self *SBaremetalGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { netifs, net := host.GetNetinterfacesWithIdAndCredential(netConfig.Network, userCred, netConfig.Reserved) if netifs != nil { nicCnt := 1 @@ -135,7 +125,7 @@ func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []string { return []string{models.NETWORK_TYPE_BAREMETAL, models.NETWORK_TYPE_GUEST} } -func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { +func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { netifs := host.GetNetInterfaces() netsAvaiable := make([]models.SNetwork, 0) netifIndexs := make(map[string][]models.SNetInterface, 0) @@ -203,6 +193,10 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct return nil, fmt.Errorf("No appropriate host virtual network...") } +func (self *SBaremetalGuestDriver) GetStorageTypes() []string { + return nil +} + func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { bs := host.GetBaremetalstorage() if bs == nil { @@ -298,35 +292,37 @@ func (self *SBaremetalGuestDriver) StartGuestSyncstatusTask(guest *models.SGuest return nil } -func (self *SBaremetalGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - var confs = make([]baremetal.BaremetalDiskConfig, 0) - jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") - for i := 0; i < len(jsonArray); i += 1 { - // for data.Contains(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) { - desc, _ := jsonArray[i].GetString() // data.GetString(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) - data.Remove(fmt.Sprintf("baremetal_disk_config.%d", i)) - bmConf, err := baremetal.ParseDiskConfig(desc) - if err != nil { - return nil, httperrors.NewInputParameterError("baremetal_disk_config.%d", i) - } - confs = append(confs, bmConf) - } - if len(confs) == 0 { - bmConf, _ := baremetal.ParseDiskConfig("") - confs = append(confs, bmConf) - } - data.Set("baremetal_disk_config", jsonutils.Marshal(confs)) - if !data.Contains("disk.0") { +func (self *SBaremetalGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + /*var confs = make([]api.BaremetalDiskConfig, 0)*/ + //jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") + //for i := 0; i < len(jsonArray); i += 1 { + //// for data.Contains(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) { + //desc, _ := jsonArray[i].GetString() // data.GetString(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) + //data.Remove(fmt.Sprintf("baremetal_disk_config.%d", i)) + //bmConf, err := baremetal.ParseDiskConfig(desc) + //if err != nil { + //return nil, httperrors.NewInputParameterError("baremetal_disk_config.%d", i) + //} + //confs = append(confs, bmConf) + //} + //if len(confs) == 0 { + //bmConf, _ := baremetal.ParseDiskConfig("") + //confs = append(confs, bmConf) + //} + /*data.Set("baremetal_disk_config", jsonutils.Marshal(confs))*/ + if len(input.Disks) <= 0 { return nil, httperrors.NewInputParameterError("Root disk must be present") } - disk0, _ := data.Get("disk.0") - if !disk0.Contains("image_id") { + + disk0 := input.Disks[0] + + if disk0.ImageId == "" { return nil, httperrors.NewInputParameterError("Root disk must have templete") } - return data, nil + return input, nil } -func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { if host.HostType != models.HOST_TYPE_BAREMETAL || !host.IsBaremetal { return nil, httperrors.NewInputParameterError("Host %s is not a baremetal", bmName) } @@ -336,9 +332,9 @@ func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, u if host.GetBaremetalServer() != nil { return nil, httperrors.NewInsufficientResourceError("Baremetal %s is occupied", bmName) } - data.Set("vmem_size", jsonutils.NewInt(int64(host.MemSize))) - data.Set("vcpu_count", jsonutils.NewInt(int64(host.CpuCount))) - return data, nil + input.VmemSize = host.MemSize + input.VcpuCount = int(host.CpuCount) + return input, nil } func (self *SBaremetalGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { diff --git a/pkg/compute/guestdrivers/container.go b/pkg/compute/guestdrivers/container.go index 0be43be85b..d36fb073e6 100644 --- a/pkg/compute/guestdrivers/container.go +++ b/pkg/compute/guestdrivers/container.go @@ -28,20 +28,6 @@ func init() { models.RegisterGuestDriver(&driver) } -func (self *SContainerDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - vmemSize, vcpuCount, err := models.ValidateMemCpuData(data) - if err != nil { - return nil, err - } - if vmemSize > 0 { - data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") - } - if vcpuCount > 0 { - data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") - } - return data, nil -} - func (self *SContainerDriver) newUnsupportOperationError(option string) error { return httperrors.NewUnsupportOperationError("Container not support %s", option) } diff --git a/pkg/compute/guestdrivers/huawei.go b/pkg/compute/guestdrivers/huawei.go index 5a3831cedf..8d12b45d59 100644 --- a/pkg/compute/guestdrivers/huawei.go +++ b/pkg/compute/guestdrivers/huawei.go @@ -4,13 +4,11 @@ import ( "context" "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" ) @@ -35,6 +33,10 @@ func (self *SHuaweiGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SHuaweiGuestDriver) GetStorageTypes() []string { + return []string{models.STORAGE_HUAWEI_SATA, models.STORAGE_HUAWEI_SAS, models.STORAGE_HUAWEI_SSD} +} + func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -43,7 +45,7 @@ func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend st } } - for _, stype := range []string{models.STORAGE_HUAWEI_SATA, models.STORAGE_HUAWEI_SAS, models.STORAGE_HUAWEI_SSD} { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -77,10 +79,6 @@ func (self *SHuaweiGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SHuaweiGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) -} - func (self *SHuaweiGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { if !utils.IsInStringArray(guest.Status, []string{models.VM_RUNNING, models.VM_READY}) { return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 3594b06d32..7f1c8466ee 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -348,7 +349,9 @@ func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *mo } func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), task.GetParams(), task.GetTaskId()) + input := new(api.ServerCreateInput) + task.GetParams().Unmarshal(input) + return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), input.Disks, task.GetTaskId()) } func (self *SKVMGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { diff --git a/pkg/compute/guestdrivers/openstack.go b/pkg/compute/guestdrivers/openstack.go index 66522963ec..e96b1e6bfe 100644 --- a/pkg/compute/guestdrivers/openstack.go +++ b/pkg/compute/guestdrivers/openstack.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/compare" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -42,6 +43,10 @@ func (self *SOpenStackGuestDriver) GetMinimalSysDiskSizeGb() int { return options.Options.DefaultDiskSizeMB / 1024 } +func (self *SOpenStackGuestDriver) GetStorageTypes() []string { + return []string{models.STORAGE_OPENSTACK_ISCSI} +} + func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i++ { @@ -49,7 +54,7 @@ func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend return &storages[i] } } - for _, stype := range []string{models.STORAGE_OPENSTACK_ISCSI} { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i++ { if storages[i].StorageType == stype { return &storages[i] @@ -83,15 +88,16 @@ func (self *SOpenStackGuestDriver) GetDeployStatus() ([]string, error) { return []string{models.VM_RUNNING}, nil } -func (self *SOpenStackGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SOpenStackGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + var err error + input, err = self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) >= 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - return data, nil + return input, nil } func (self *SOpenStackGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index 79e8781542..6586791324 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -38,6 +39,16 @@ func (self *SQcloudGuestDriver) GetMinimalSysDiskSizeGb() int { return 50 } +func (self *SQcloudGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_CLOUD_BASIC, + models.STORAGE_CLOUD_PREMIUM, + models.STORAGE_CLOUD_SSD, + models.STORAGE_LOCAL_BASIC, + models.STORAGE_LOCAL_SSD, + } +} + func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i++ { @@ -45,13 +56,7 @@ func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend st return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_CLOUD_BASIC, - models.STORAGE_CLOUD_PREMIUM, - models.STORAGE_CLOUD_SSD, - models.STORAGE_LOCAL_BASIC, - models.STORAGE_LOCAL_SSD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i++ { if storages[i].StorageType == stype { return &storages[i] @@ -99,18 +104,16 @@ func (self *SQcloudGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - sysDisk := models.SDiskConfig{} - if err := data.Unmarshal(&sysDisk, "disk.0"); err != nil { - return nil, fmt.Errorf("Missing system disk information") - } + + sysDisk := input.Disks[0] switch sysDisk.Backend { case models.STORAGE_CLOUD_BASIC, models.STORAGE_CLOUD_SSD: if sysDisk.SizeMb > 500*1024 { @@ -122,11 +125,8 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred } } - for i := 1; data.Contains(fmt.Sprintf("disk.%d", i)); i++ { - disk := models.SDiskConfig{} - if err := data.Unmarshal(&disk, fmt.Sprintf("disk.%d", i)); err != nil { - return nil, httperrors.NewInputParameterError("invalid diskinfo of index %d", i) - } + for i := 1; i < len(input.Disks); i++ { + disk := input.Disks[i] switch disk.Backend { case models.STORAGE_CLOUD_BASIC: if disk.SizeMb < 10*1024 || disk.SizeMb > 16000*1024 { @@ -142,7 +142,7 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred } } } - return data, nil + return input, nil } func (self *SQcloudGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index ea7db64abe..6b903da128 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -27,12 +28,12 @@ func (self *SVirtualizedGuestDriver) GetMaxVMemSizeGB() int { return 512 } -func (self *SVirtualizedGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params *jsonutils.JSONDict) error { +func (self *SVirtualizedGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params []*api.BaremetalDiskConfig) error { // do nothing return nil } -func (self *SVirtualizedGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { +func (self *SVirtualizedGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { net, _ := host.GetNetworkWithIdAndCredential(netConfig.Network, userCred, netConfig.Reserved) nicConfs := []models.SNicConfig{ { @@ -55,7 +56,7 @@ func (self *SVirtualizedGuestDriver) GetRandomNetworkTypes() []string { return []string{models.NETWORK_TYPE_GUEST} } -func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { +func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { wirePattern = regexp.MustCompile(netConfig.Wire) @@ -115,6 +116,10 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, return gn, err } +func (self *SVirtualizedGuestDriver) GetStorageTypes() []string { + return nil +} + func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { return host.GetLeastUsedStorage(backend) } @@ -183,24 +188,24 @@ func (self *SVirtualizedGuestDriver) RequestStopGuestForDelete(ctx context.Conte return nil } -func (self *SVirtualizedGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return data, nil +func (self *SVirtualizedGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + return input, nil } -func (self *SVirtualizedGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (self *SVirtualizedGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { if host.HostStatus != models.HOST_ONLINE { return nil, httperrors.NewInvalidStatusError("Host %s is not online", bmName) } - data.Add(jsonutils.NewString(host.Id), "prefer_host_id") + input.PreferHost = host.Id if host.IsPrepaidRecycle() { - data.Set("vmem_size", jsonutils.NewInt(int64(host.MemSize))) - data.Set("vcpu_count", jsonutils.NewInt(int64(host.CpuCount))) + input.VmemSize = host.MemSize + input.VcpuCount = int(host.CpuCount) if host.GetGuestCount() >= 1 { return nil, httperrors.NewInsufficientResourceError("host has been occupied") } } - return data, nil + return input, nil } func (self *SVirtualizedGuestDriver) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, data *jsonutils.JSONDict) error { diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 417232e60c..6af7fb6b33 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/baremetal" @@ -156,17 +157,17 @@ func (self *SBaseHostDriver) GetRaidScheme(host *models.SHost, raid string) (str candidates = []string{raid} } } - var conf []*baremetal.BaremetalDiskConfig + var conf []*api.BaremetalDiskConfig for i := 0; i < len(candidates); i++ { if candidates[i] == baremetal.DISK_CONF_NONE { - conf = []*baremetal.BaremetalDiskConfig{} + conf = []*api.BaremetalDiskConfig{} } else { parsedConf, err := baremetal.ParseDiskConfig(candidates[i]) if err != nil { log.Errorf("try raid %s failed: %s", candidates[i], err.Error()) return "", err } - conf = []*baremetal.BaremetalDiskConfig{&parsedConf} + conf = []*api.BaremetalDiskConfig{&parsedConf} } baremetalStorage := models.ConvertStorageInfo2BaremetalStorages(host.StorageInfo) if baremetalStorage == nil { diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index e551cb61f2..84d4d7f1ae 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -341,6 +341,7 @@ func (self *SKVMHostDriver) PrepareConvert(host *models.SHost, image, raid strin if err != nil { return nil, err } + // TODO: use api.ServerCreateInput var sysSize = "60g" if data.Contains("baremetal_disk_config.0") { jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 2b757d7f6f..966cc2dbb1 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -13,12 +13,12 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/fileutils" - "yunion.io/x/pkg/util/osprofile" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -33,40 +33,40 @@ import ( ) const ( - DISK_INIT = "init" - DISK_REBUILD = "rebuild" - DISK_ALLOC_FAILED = "alloc_failed" - DISK_STARTALLOC = "start_alloc" - DISK_BACKUP_STARTALLOC = "backup_start_alloc" - DISK_BACKUP_ALLOC_FAILED = "backup_alloc_failed" - DISK_ALLOCATING = "allocating" - DISK_READY = "ready" - DISK_RESET = "reset" - DISK_DEALLOC = "deallocating" - DISK_DEALLOC_FAILED = "dealloc_failed" - DISK_UNKNOWN = "unknown" - DISK_DETACHING = "detaching" - DISK_ATTACHING = "attaching" + DISK_INIT = api.DISK_INIT + DISK_REBUILD = api.DISK_REBUILD + DISK_ALLOC_FAILED = api.DISK_ALLOC_FAILED + DISK_STARTALLOC = api.DISK_STARTALLOC + DISK_BACKUP_STARTALLOC = api.DISK_BACKUP_STARTALLOC + DISK_BACKUP_ALLOC_FAILED = api.DISK_BACKUP_ALLOC_FAILED + DISK_ALLOCATING = api.DISK_ALLOCATING + DISK_READY = api.DISK_READY + DISK_RESET = api.DISK_RESET + DISK_DEALLOC = api.DISK_DEALLOC + DISK_DEALLOC_FAILED = api.DISK_DEALLOC_FAILED + DISK_UNKNOWN = api.DISK_UNKNOWN + DISK_DETACHING = api.DISK_DETACHING + DISK_ATTACHING = api.DISK_ATTACHING - DISK_START_SAVE = "start_save" - DISK_SAVING = "saving" + DISK_START_SAVE = api.DISK_START_SAVE + DISK_SAVING = api.DISK_SAVING - DISK_START_RESIZE = "start_resize" - DISK_RESIZING = "resizing" + DISK_START_RESIZE = api.DISK_START_RESIZE + DISK_RESIZING = api.DISK_RESIZING - DISK_START_MIGRATE = "start_migrate" - DISK_POST_MIGRATE = "post_migrate" - DISK_MIGRATING = "migrating" + DISK_START_MIGRATE = api.DISK_START_MIGRATE + DISK_POST_MIGRATE = api.DISK_POST_MIGRATE + DISK_MIGRATING = api.DISK_MIGRATING - DISK_START_SNAPSHOT = "start_snapshot" - DISK_SNAPSHOTING = "snapshoting" + DISK_START_SNAPSHOT = api.DISK_START_SNAPSHOT + DISK_SNAPSHOTING = api.DISK_SNAPSHOTING - DISK_TYPE_SYS = "sys" - DISK_TYPE_SWAP = "swap" - DISK_TYPE_DATA = "data" - DISK_TYPE_VOLUME = "volume" + DISK_TYPE_SYS = api.DISK_TYPE_SYS + DISK_TYPE_SWAP = api.DISK_TYPE_SWAP + DISK_TYPE_DATA = api.DISK_TYPE_DATA + DISK_TYPE_VOLUME = api.DISK_TYPE_VOLUME - DISK_BACKING_IMAGE = "image" + DISK_BACKING_IMAGE = api.DISK_BACKING_IMAGE ) type SDiskManager struct { @@ -344,12 +344,11 @@ func (self *SDisk) GetRuningGuestCount() int { } func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - diskConfig := SDiskConfig{} - if err := data.Unmarshal(&diskConfig, "disk"); err != nil { + input := new(api.DiskCreateInput) + if err := data.Unmarshal(input); err != nil { return err - } else { - self.fetchDiskInfo(&diskConfig) } + self.fetchDiskInfo(input.DiskConfig) return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) } @@ -379,17 +378,18 @@ func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok } func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - disk, err := data.Get("disk") + input, err := cmdline.FetchDiskCreateInputByJSON(data) if err != nil { - return nil, httperrors.NewMissingParameterError("disk") + return nil, httperrors.NewInputParameterError("parse disk input: %v", err) } - - diskConfig, err := parseDiskInfo(ctx, userCred, disk) + diskConfig := input.DiskConfig + diskConfig, err = parseDiskInfo(ctx, userCred, diskConfig) if err != nil { return nil, err } + input.Project = ownerProjId - storageID := jsonutils.GetAnyString(data, []string{"storage_id", "storage"}) + storageID := input.Storage if storageID != "" { storageObj, err := StorageManager.FetchByIdOrName(nil, storageID) if err != nil { @@ -404,16 +404,15 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc if err != nil { return nil, err } - data.Add(jsonutils.NewString(storage.Id), "storage_id") + input.Storage = storage.Id } else { diskConfig.Backend = STORAGE_LOCAL - hypervisor, _ := data.GetString("hypervisor") - data, err = ValidateScheduleCreateData(ctx, userCred, data, hypervisor) + serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor) if err != nil { return nil, err } + input = serverInput.ToDiskCreateInput() } - data.Add(jsonutils.Marshal(diskConfig), "disk") if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil { return nil, err @@ -422,10 +421,10 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } - return data, nil + return input.JSON(input), nil } -func (manager *SDiskManager) validateDiskOnStorage(diskConfig *SDiskConfig, storage *SStorage) error { +func (manager *SDiskManager) validateDiskOnStorage(diskConfig *api.DiskConfig, storage *SStorage) error { if !storage.Enabled { return httperrors.NewInputParameterError("Cannot create disk with disabled storage[%s]", storage.Name) } @@ -455,7 +454,27 @@ func (manager *SDiskManager) validateDiskOnStorage(diskConfig *SDiskConfig, stor return nil } -func (disk *SDisk) SetStorageByHost(hostId string, diskConfig *SDiskConfig) error { +func (disk *SDisk) SetStorage(storageId string, diskConfig *api.DiskConfig) error { + backend := diskConfig.Backend + if backend == "" { + return fmt.Errorf("Backend is empty") + } + storage := StorageManager.FetchStorageById(storageId) + if storage == nil { + return fmt.Errorf("Not found backend %s storage %s", backend, storageId) + } + err := DiskManager.validateDiskOnStorage(diskConfig, storage) + if err != nil { + return err + } + _, err = db.Update(disk, func() error { + disk.StorageId = storage.Id + return nil + }) + return err +} + +func (disk *SDisk) SetStorageByHost(hostId string, diskConfig *api.DiskConfig) error { host := HostManager.FetchHostById(hostId) backend := diskConfig.Backend if backend == "" { @@ -495,16 +514,16 @@ func getDiskResourceRequirements(ctx context.Context, userCred mcclient.TokenCre } } -func (manager *SDiskManager) convertToBatchCreateData(data jsonutils.JSONObject) *jsonutils.JSONDict { +/*func (manager *SDiskManager) convertToBatchCreateData(data jsonutils.JSONObject) *jsonutils.JSONDict { diskConfig, _ := data.Get("disk") newData := data.(*jsonutils.JSONDict).CopyExcludes("disk") newData.Add(diskConfig, "disk.0") return newData -} +}*/ func (manager *SDiskManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { pendingUsage := getDiskResourceRequirements(ctx, userCred, data, len(items)) - RunBatchCreateTask(ctx, items, userCred, manager.convertToBatchCreateData(data), pendingUsage, "DiskBatchCreateTask") + RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "DiskBatchCreateTask") } func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { @@ -1142,97 +1161,28 @@ func totalDiskSize(projectId string, active tristate.TriState, ready tristate.Tr return size } -type SDiskConfig struct { - ImageId string `json:"image_id"` - - SnapshotId string `json:"snapshot_id"` - DiskType string `json:"disk_type"` // sys, data, swap, volume - - // ImageDiskFormat string - SizeMb int `json:"size"` // MB - Fs string `json:"fs"` // file system - Format string `json:"format"` // - Driver string `json:"driver"` // - Cache string `json:"cache"` // - Mountpoint string `json:"mountpoint"` // - Backend string `json:"backend"` // stroageType - Medium string `json:"medium"` - ImageProperties map[string]string `json:"image_properties"` - - Schedtags map[string]string `json:"schedtags"` - - DiskId string `json:"-"` // import only -} - -func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SDiskConfig, error) { - diskConfig := SDiskConfig{} - - diskJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := diskJson.Unmarshal(&diskConfig) - if err != nil { +func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info *api.DiskConfig) (*api.DiskConfig, error) { + if info.SnapshotId != "" { + if err := fillDiskConfigBySnapshot(userCred, info, info.SnapshotId); err != nil { return nil, err } - return &diskConfig, nil } - - // default backend and medium type - diskConfig.Backend = "" // STORAGE_LOCAL - diskConfig.Medium = DISK_TYPE_HYBRID - - diskStr, err := info.GetString() - if err != nil { - log.Errorf("invalid diskinfo format %s", err) - return nil, err - } - parts := strings.Split(diskStr, ":") - for _, p := range parts { - if len(p) == 0 { - continue - } - if regutils.MatchSize(p) { - diskConfig.SizeMb, _ = fileutils.GetSizeMb(p, 'M', 1024) - } else if utils.IsInStringArray(p, osprofile.FS_TYPES) { - diskConfig.Fs = p - } else if utils.IsInStringArray(p, osprofile.IMAGE_FORMAT_TYPES) { - diskConfig.Format = p - } else if utils.IsInStringArray(p, osprofile.DISK_DRIVERS) { - diskConfig.Driver = p - } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { - diskConfig.Cache = p - } else if utils.IsInStringArray(p, DISK_TYPES) { - diskConfig.Medium = p - } else if utils.IsInStringArray(p, []string{DISK_TYPE_VOLUME}) { - diskConfig.DiskType = p - } else if p[0] == '/' { - diskConfig.Mountpoint = p - } else if p == "autoextend" { - diskConfig.SizeMb = -1 - } else if storageType, exist := StorageManager.IsStorageTypeExist(p); exist { - diskConfig.Backend = storageType - } else if strings.HasPrefix(p, "snapshot-") { - // HACK: use snapshot creat disk format snapshot-id - // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b - if err := fillDiskConfigBySnapshot(userCred, &diskConfig, p[len("snapshot-"):]); err != nil { - return nil, err - } - } else if len(p) > 0 { - if err := fillDiskConfigByImage(ctx, userCred, &diskConfig, p); err != nil { - return nil, err - } + if info.ImageId != "" { + if err := fillDiskConfigByImage(ctx, userCred, info, info.ImageId); err != nil { + return nil, err } } // XXX: do not set default disk size here, set it by each hypervisor driver // if len(diskConfig.ImageId) > 0 && diskConfig.SizeMb == 0 { // diskConfig.SizeMb = options.Options.DefaultDiskSize // MB // else - if len(diskConfig.ImageId) == 0 && diskConfig.SizeMb == 0 { + if len(info.ImageId) == 0 && info.SizeMb == 0 { return nil, httperrors.NewInputParameterError("Diskinfo not contains either imageID or size") } - return &diskConfig, nil + return info, nil } -func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDiskConfig, snapshotId string) error { +func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *api.DiskConfig, snapshotId string) error { iSnapshot, err := SnapshotManager.FetchByIdOrName(userCred, snapshotId) if err != nil { if err == sql.ErrNoRows { @@ -1260,7 +1210,7 @@ func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDi } func fillDiskConfigByImage(ctx context.Context, userCred mcclient.TokenCredential, - diskConfig *SDiskConfig, imageId string) error { + diskConfig *api.DiskConfig, imageId string) error { if userCred == nil { diskConfig.ImageId = imageId } else { @@ -1298,7 +1248,7 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, imageI return image, nil } -func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { +func (self *SDisk) fetchDiskInfo(diskConfig *api.DiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId self.DiskType = DISK_TYPE_SYS @@ -1343,6 +1293,7 @@ type DiskInfo struct { DiskType string } +// DEPRECATE: will be remove in future, use ToDiskConfig func (self *SDisk) ToDiskInfo() DiskInfo { ret := DiskInfo{ ImageId: self.GetTemplateId(), @@ -1362,6 +1313,26 @@ func (self *SDisk) ToDiskInfo() DiskInfo { return ret } +func (self *SDisk) ToDiskConfig() *api.DiskConfig { + ret := &api.DiskConfig{ + Index: -1, + ImageId: self.GetTemplateId(), + Fs: self.GetFsFormat(), + Mountpoint: self.GetMountPoint(), + Format: self.DiskFormat, + SizeMb: self.DiskSize, + DiskType: self.DiskType, + } + storage := self.GetStorage() + if storage == nil { + return ret + } + ret.Storage = storage.Id + ret.Backend = storage.StorageType + ret.Medium = storage.MediumType + return ret +} + func (self *SDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { // override log.Infof("disk delete do nothing") diff --git a/pkg/compute/models/dynamicschedtags.go b/pkg/compute/models/dynamicschedtags.go index a2c0059c3f..5fe7c7903e 100644 --- a/pkg/compute/models/dynamicschedtags.go +++ b/pkg/compute/models/dynamicschedtags.go @@ -193,7 +193,7 @@ func (self *SDynamicschedtag) PerformEvaluate(ctx context.Context, userCred mccl hostDesc := jsonutils.Marshal(host) params := jsonutils.NewDict() - params.Add(srvDesc, "server") + params.Add(srvDesc.JSON(srvDesc), "server") params.Add(hostDesc, "host") meet, err := conditionparser.Eval(self.Condition, params) @@ -201,7 +201,7 @@ func (self *SDynamicschedtag) PerformEvaluate(ctx context.Context, userCred mccl return nil, err } result := jsonutils.NewDict() - result.Add(srvDesc, "server") + result.Add(srvDesc.JSON(srvDesc), "server") result.Add(hostDesc, "host") if meet { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 8e9192e232..88b93a234b 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -18,6 +18,9 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -703,8 +706,8 @@ func (self *SGuest) StartGueststartTask(ctx context.Context, userCred mcclient.T return nil } -func (self *SGuest) StartGuestCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error { - return self.GetDriver().StartGuestCreateTask(self, ctx, userCred, params, pendingUsage, parentTaskId) +func (self *SGuest) StartGuestCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, pendingUsage quotas.IQuota, parentTaskId string) error { + return self.GetDriver().StartGuestCreateTask(self, ctx, userCred, input.JSON(input), pendingUsage, parentTaskId) } func (self *SGuest) StartSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { @@ -1118,10 +1121,13 @@ func (self *SGuest) AllowPerformCreatedisk(ctx context.Context, userCred mcclien func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { diskSize := 0 - disksConf := jsonutils.NewDict() + disksConf := make([]*api.DiskConfig, 0) diskSizes := make(map[string]int, 0) - diskDefArray := jsonutils.GetArrayOfPrefix(data, "disk") + diskDefArray, err := cmdline.FetchDiskConfigsByJSON(data) + if err != nil { + return nil, err + } if len(diskDefArray) == 0 { logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, "No Disk Info Provided", userCred, false) return nil, httperrors.NewBadRequestError("No Disk Info Provided") @@ -1136,7 +1142,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok if len(diskInfo.Backend) == 0 { diskInfo.Backend = self.getDefaultStorageType() } - disksConf.Set(fmt.Sprintf("disk.%d", diskIdx), jsonutils.Marshal(diskInfo)) + disksConf = append(disksConf, diskInfo) if _, ok := diskSizes[diskInfo.Backend]; !ok { diskSizes[diskInfo.Backend] = diskInfo.SizeMb } else { @@ -1163,7 +1169,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok pendingUsage := &SQuota{ Storage: diskSize, } - err := QuotaManager.CheckSetPendingQuota(ctx, userCred, self.ProjectId, pendingUsage) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, self.ProjectId, pendingUsage) if err != nil { logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) return nil, httperrors.NewOutOfQuotaError(err.Error()) @@ -1172,7 +1178,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false) + err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false, nil) if err != nil { QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) @@ -1182,7 +1188,9 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok return nil, err } -func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error { +func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, disks []*api.DiskConfig, parentTaskId string) error { + data := jsonutils.NewDict() + data.Add(jsonutils.Marshal(disks), "disks") task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateDiskTask", self, userCred, data, parentTaskId, "", nil) if err != nil { return err @@ -1394,7 +1402,8 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T log.Errorf("net_desc not found") return nil, httperrors.NewMissingParameterError("net_desc") } - conf, err := parseNetworkInfo(userCred, netDesc) + conf, err := cmdline.ParseNetworkConfigByJSON(netDesc, -1) + conf, err = parseNetworkInfo(userCred, conf) if err != nil { log.Errorf("parseNetworkInfo fail %s", err) return nil, err @@ -1523,7 +1532,12 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. if err != nil { return nil, httperrors.NewBadRequestError(err.Error()) } - conf, err := parseNetworkInfo(userCred, netDesc) + conf, err := cmdline.ParseNetworkConfigByJSON(netDesc, -1) + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + + conf, err = parseNetworkInfo(userCred, conf) if err != nil { return nil, err } @@ -1706,15 +1720,16 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T var diskIdx = 1 var newDiskIdx = 0 var diskSizes = make(map[string]int, 0) - var newDisks = jsonutils.NewDict() + var newDisks = make([]*api.DiskConfig, 0) var resizeDisks = jsonutils.NewArray() - for { - diskNum := fmt.Sprintf("disk.%d", diskIdx) - diskDesc, err := data.Get(diskNum) - if err != nil || diskDesc == jsonutils.JSONNull { - break - } - diskConf, err := parseDiskInfo(ctx, userCred, diskDesc) + + inputDisks, err := cmdline.FetchDiskConfigsByJSON(data) + if err != nil { + return nil, err + } + + for _, diskConf := range inputDisks { + diskConf, err = parseDiskInfo(ctx, userCred, diskConf) if err != nil { return nil, httperrors.NewBadRequestError("Parse disk info error: %s", err) } @@ -1723,7 +1738,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } if diskConf.SizeMb > 0 { if diskIdx >= len(disks) { - newDisks.Add(jsonutils.Marshal(diskConf), fmt.Sprintf("disk.%d", newDiskIdx)) + newDisks = append(newDisks, diskConf) newDiskIdx += 1 addDisk += diskConf.SizeMb storage := host.GetLeastUsedStorage(diskConf.Backend) @@ -1772,8 +1787,8 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetId()) } - if newDisks.Length() > 0 { - confs.Add(newDisks, "create") + if len(newDisks) > 0 { + confs.Add(jsonutils.Marshal(newDisks), "create") } if resizeDisks.Length() > 0 { confs.Add(resizeDisks, "resize") @@ -1815,8 +1830,8 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } } - if newDisks.Length() > 0 { - err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false) + if len(newDisks) > 0 { + err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false, nil) if err != nil { QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err) @@ -1826,17 +1841,23 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T return nil, nil } -func (self *SGuest) confToSchedDesc(addCpu, addMem, addDisk int) *jsonutils.JSONDict { - desc := jsonutils.NewDict() - desc.Set("vmem_size", jsonutils.NewInt(int64(addMem))) - desc.Set("vcpu_count", jsonutils.NewInt(int64(addCpu))) +func (self *SGuest) confToSchedDesc(addCpu, addMem, addDisk int) *schedapi.ScheduleInput { guestDisks := self.GetDisks() - diskInfo := guestDisks[0].ToDiskInfo() - diskInfo.Size = int64(addDisk) - desc.Set("disk.0", jsonutils.Marshal(diskInfo)) - desc.Set("prefer_host_id", jsonutils.NewString(self.HostId)) - desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) - desc.Set("owner_tenant_id", jsonutils.NewString(self.ProjectId)) + diskInfo := guestDisks[0].ToDiskConfig() + diskInfo.SizeMb = addDisk + + desc := &schedapi.ScheduleInput{ + ServerConfig: schedapi.ServerConfig{ + ServerConfigs: &api.ServerConfigs{ + Hypervisor: self.Hypervisor, + Project: self.ProjectId, + PreferHost: self.HostId, + Disks: []*api.DiskConfig{diskInfo}, + }, + Memory: addMem, + Ncpu: addCpu, + }, + } return desc } @@ -2700,8 +2721,8 @@ type SImportNic struct { Ifname string `json:"ifname"` } -func (n SImportNic) ToNetConfig(net *SNetwork) *SNetworkConfig { - return &SNetworkConfig{ +func (n SImportNic) ToNetConfig(net *SNetwork) *api.NetworkConfig { + return &api.NetworkConfig{ Network: net.Id, Wire: net.WireId, Address: n.Ip, @@ -2725,8 +2746,8 @@ type SImportDisk struct { TemplateId string `json:"template_id"` } -func (d SImportDisk) ToDiskConfig() *SDiskConfig { - ret := &SDiskConfig{ +func (d SImportDisk) ToDiskConfig() *api.DiskConfig { + ret := &api.DiskConfig{ SizeMb: d.SizeMb, ImageId: d.TemplateId, Format: d.Format, @@ -2849,7 +2870,7 @@ func (self *SGuest) importDisks(ctx context.Context, userCred mcclient.TokenCred return httperrors.NewInputParameterError("Empty import disks") } for _, disk := range disks { - disk, err := self.createDiskOnHost(ctx, userCred, self.GetHost(), disk.ToDiskConfig(), nil, true, true) + disk, err := self.createDiskOnHost(ctx, userCred, self.GetHost(), disk.ToDiskConfig(), nil, true, true, nil) if err != nil { return err } diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index cde845baba..94815caf87 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -247,6 +248,7 @@ func (self *SGuestdisk) Detach(ctx context.Context, userCred mcclient.TokenCrede return db.DetachJoint(ctx, userCred, self) } +// DEPRECATE: will be remove in future, use ToDiskConfig func (self *SGuestdisk) ToDiskInfo() DiskInfo { disk := self.GetDisk() if disk == nil { @@ -257,3 +259,13 @@ func (self *SGuestdisk) ToDiskInfo() DiskInfo { info.Cache = self.CacheMode return info } + +func (self *SGuestdisk) ToDiskConfig() *api.DiskConfig { + disk := self.GetDisk() + if disk == nil { + return nil + } + conf := disk.ToDiskConfig() + conf.Index = int(self.Index) + return conf +} diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 34790a0d91..478fe10105 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -30,19 +31,20 @@ type IGuestDriver interface { GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, host *SHost) jsonutils.JSONObject - ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) + ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *api.ServerCreateInput) (*api.ServerCreateInput, error) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) - ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) + ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) - PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *SHost, params *jsonutils.JSONDict) error + PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *SHost, params []*api.BaremetalDiskConfig) error - GetNamedNetworkConfiguration(guest *SGuest, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig) (*SNetwork, []SNicConfig, IPAddlocationDirection) + GetNamedNetworkConfiguration(guest *SGuest, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig) (*SNetwork, []SNicConfig, IPAddlocationDirection) - Attach2RandomNetwork(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) + Attach2RandomNetwork(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) GetRandomNetworkTypes() []string + GetStorageTypes() []string ChooseHostStorage(host *SHost, backend string) *SStorage StartGuestCreateTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 220f17acad..0b326e8427 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" @@ -629,3 +630,18 @@ func (self *SGuestnetwork) GetShortDesc(ctx context.Context) *jsonutils.JSONDict } return desc } + +func (self *SGuestnetwork) ToNetworkConfig() *api.NetworkConfig { + net := self.GetNetwork() + if net == nil { + return nil + } + ret := &api.NetworkConfig{ + Index: int(self.Index), + Network: net.Id, + Wire: net.GetWire().Id, + Address: self.IpAddr, + Project: net.ProjectId, + } + return ret +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0f2f09831c..fd50ef79ce 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "fmt" - "strconv" "strings" "time" @@ -13,7 +12,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" @@ -22,6 +20,9 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -717,47 +718,30 @@ func (guest *SGuest) ValidateResizeDisk(disk *SDisk, storage *SStorage) error { return guest.GetDriver().ValidateResizeDisk(guest, disk, storage) } -func ValidateMemCpuData(data jsonutils.JSONObject) (int, int, error) { - vmemSize := 0 - vcpuCount := 0 - var err error - - hypervisor, _ := data.GetString("hypervisor") +func ValidateMemCpuData(vmemSize, vcpuCount int, hypervisor string) (int, int, error) { if len(hypervisor) == 0 { hypervisor = HYPERVISOR_DEFAULT } driver := GetDriver(hypervisor) - vmemStr, _ := data.GetString("vmem_size") - if len(vmemStr) > 0 { - if !regutils.MatchSize(vmemStr) { - return 0, 0, httperrors.NewInputParameterError("Memory size must be number[+unit], like 256M, 1G or 256") - } - vmemSize, err = fileutils.GetSizeMb(vmemStr, 'M', 1024) - if err != nil { - return 0, 0, err - } + if vmemSize > 0 { maxVmemGb := driver.GetMaxVMemSizeGB() if vmemSize < 8 || vmemSize > maxVmemGb*1024 { return 0, 0, httperrors.NewInputParameterError("Memory size must be 8MB ~ %d GB", maxVmemGb) } } - vcpuStr, _ := data.GetString("vcpu_count") - if len(vcpuStr) > 0 { - if !regutils.MatchInteger(vcpuStr) { - return 0, 0, httperrors.NewInputParameterError("CPU core count must be integer") - } - vcpuCount, _ = strconv.Atoi(vcpuStr) - maxVcpuCount := driver.GetMaxVCpuCount() - if vcpuCount < 1 || vcpuCount > maxVcpuCount { - return 0, 0, httperrors.NewInputParameterError("CPU core count must be 1 ~ %d", maxVcpuCount) - } + + maxVcpuCount := driver.GetMaxVCpuCount() + if vcpuCount < 1 || vcpuCount > maxVcpuCount { + return 0, 0, httperrors.NewInputParameterError("CPU core count must be 1 ~ %d", maxVcpuCount) } return vmemSize, vcpuCount, nil } func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - vmemSize, vcpuCount, err := ValidateMemCpuData(data) + memSize, _ := data.Int("vmem_size") + cpuCount, _ := data.Int("vcpu_count") + vmemSize, vcpuCount, err := ValidateMemCpuData(int(memSize), int(cpuCount), self.Hypervisor) if err != nil { return nil, err } @@ -797,26 +781,32 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To } func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - resetPassword := jsonutils.QueryBoolean(data, "reset_password", true) - passwd, _ := data.GetString("password") + input, err := cmdline.FetchServerCreateInputByJSON(data) + if err != nil { + return nil, err + } + resetPassword := true + if input.ResetPassword != nil { + resetPassword = *input.ResetPassword + } + + passwd := input.Password if resetPassword && len(passwd) > 0 { if !seclib2.MeetComplxity(passwd) { return nil, httperrors.NewWeakPasswordError() } } - var err error var hypervisor string // var rootStorageType string var osProf osprofile.SOSProfile - hypervisor, _ = data.GetString("hypervisor") + hypervisor = input.Hypervisor if hypervisor != HYPERVISOR_CONTAINER { - - disk0Json, _ := data.Get("disk.0") - if disk0Json == nil || disk0Json == jsonutils.JSONNull { + if len(input.Disks) == 0 { return nil, httperrors.NewInputParameterError("No disk information provided") } - diskConfig, err := parseDiskInfo(ctx, userCred, disk0Json) + diskConfig := input.Disks[0] + diskConfig, err = parseDiskInfo(ctx, userCred, diskConfig) if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) } @@ -833,19 +823,16 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m // } // rootStorageType = diskConfig.Backend - data.Add(jsonutils.Marshal(diskConfig), "disk.0") + input.Disks[0] = diskConfig imgProperties := diskConfig.ImageProperties - if data.Contains("cdrom") { - cdromStr, err := data.GetString("cdrom") - if err != nil { - return nil, httperrors.NewInputParameterError("invalid cdrom device description %s", err) - } + if input.Cdrom != "" { + cdromStr := input.Cdrom image, err := parseIsoInfo(ctx, userCred, cdromStr) if err != nil { return nil, httperrors.NewInputParameterError("parse cdrom device info error %s", err) } - data.Add(jsonutils.NewString(image.Id), "cdrom") + input.Cdrom = image.Id if len(imgProperties) == 0 { imgProperties = image.Properties } @@ -855,7 +842,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m imgProperties = map[string]string{"os_type": "Linux"} } - osType, _ := data.GetString("os_type") + osType := input.OsType osProf, err = osprofile.GetOSProfileFromImageProperties(imgProperties, hypervisor) if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) @@ -863,36 +850,36 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(osProf.Hypervisor) > 0 && len(hypervisor) == 0 { hypervisor = osProf.Hypervisor - data.Add(jsonutils.NewString(osProf.Hypervisor), "hypervisor") + input.Hypervisor = hypervisor } if len(osProf.OSType) > 0 && len(osType) == 0 { osType = osProf.OSType - data.Add(jsonutils.NewString(osProf.OSType), "os_type") + input.OsType = osType } - data.Add(jsonutils.Marshal(osProf), "__os_profile__") + input.OsProfile = jsonutils.Marshal(osProf) } - data, err = ValidateScheduleCreateData(ctx, userCred, data, hypervisor) + input, err = ValidateScheduleCreateData(ctx, userCred, input, hypervisor) if err != nil { return nil, err } - hypervisor, _ = data.GetString("hypervisor") + hypervisor = input.Hypervisor if hypervisor != HYPERVISOR_CONTAINER { // support sku here var sku *SServerSku - skuName := jsonutils.GetAnyString(data, []string{"sku", "flavor", "instance_type"}) + skuName := input.InstanceType if len(skuName) > 0 { sku, err := ServerSkuManager.FetchSkuByNameAndHypervisor(skuName, hypervisor, true) if err != nil { return nil, err } - data.Add(jsonutils.NewString(sku.Name), "instance_type") - data.Add(jsonutils.NewInt(int64(sku.MemorySizeMB)), "vmem_size") - data.Add(jsonutils.NewInt(int64(sku.CpuCoreCount)), "vcpu_count") + input.InstanceType = sku.Name + input.VmemSize = sku.MemorySizeMB + input.VcpuCount = sku.CpuCoreCount } else { - vmemSize, vcpuCount, err := ValidateMemCpuData(data) + vmemSize, vcpuCount, err := ValidateMemCpuData(input.VmemSize, input.VcpuCount, input.Hypervisor) if err != nil { return nil, err } @@ -903,28 +890,28 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if vcpuCount == 0 { vcpuCount = 1 } - data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") - data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") + input.VmemSize = vmemSize + input.VcpuCount = vcpuCount } - dataDiskDefs := []jsonutils.JSONObject{} + dataDiskDefs := []*api.DiskConfig{} if sku != nil && sku.AttachedDiskCount > 0 { for i := 0; i < sku.AttachedDiskCount; i += 1 { - dataDisk := jsonutils.Marshal(map[string]interface{}{ - "size_mb": sku.AttachedDiskSizeGB * 1024, - "backend": strings.ToLower(sku.AttachedDiskType), - }) + dataDisk := &api.DiskConfig{ + SizeMb: sku.AttachedDiskSizeGB * 1024, + Backend: strings.ToLower(sku.AttachedDiskType), + } dataDiskDefs = append(dataDiskDefs, dataDisk) } } // start from data disk - jsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 1; idx < len(jsonArray); idx += 1 { // data.Contains(fmt.Sprintf("disk.%d", idx)) - dataDiskDefs = append(dataDiskDefs, jsonArray[idx]) + disks := input.Disks + for idx := 1; idx < len(disks); idx += 1 { + dataDiskDefs = append(dataDiskDefs, disks[idx]) } - rootDiskConfig, err := parseDiskInfo(ctx, userCred, jsonArray[0]) + rootDiskConfig, err := parseDiskInfo(ctx, userCred, disks[0]) if err != nil { return nil, httperrors.NewGeneralError(err) // should no error } @@ -935,7 +922,8 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m rootDiskConfig.SizeMb = GetDriver(hypervisor).GetMinimalSysDiskSizeGb() * 1024 } log.Debugf("ROOT DISK: %#v", rootDiskConfig) - data.Set("disk.0", jsonutils.Marshal(rootDiskConfig)) + input.Disks[0] = rootDiskConfig + //data.Set("disk.0", jsonutils.Marshal(rootDiskConfig)) for i := 0; i < len(dataDiskDefs); i += 1 { diskConfig, err := parseDiskInfo(ctx, userCred, dataDiskDefs[i]) @@ -951,11 +939,11 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(diskConfig.Driver) == 0 { diskConfig.Driver = osProf.DiskDriver } - data.Set(fmt.Sprintf("disk.%d", i+1), jsonutils.Marshal(diskConfig)) + input.Disks[i+1] = diskConfig } - resourceTypeStr := jsonutils.GetAnyString(data, []string{"resource_type"}) - durationStr := jsonutils.GetAnyString(data, []string{"duration"}) + resourceTypeStr := input.ResourceType + durationStr := input.Duration if len(durationStr) > 0 { @@ -976,18 +964,18 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m return nil, httperrors.NewInputParameterError("unsupported duration %s", durationStr) } - data.Add(jsonutils.NewString(BILLING_TYPE_PREPAID), "billing_type") - data.Add(jsonutils.NewString(billingCycle.String()), "billing_cycle") + input.BillingType = BILLING_TYPE_PREPAID + input.BillingCycle = billingCycle.String() // expired_at will be set later by callback // data.Add(jsonutils.NewTimeString(billingCycle.EndAt(time.Time{})), "expired_at") - data.Set("duration", jsonutils.NewString(billingCycle.String())) + input.Duration = billingCycle.String() } } - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - for idx := 0; idx < len(netJsonArray); idx += 1 { // .Contains(fmt.Sprintf("net.%d", idx)); idx += 1 { - netConfig, err := parseNetworkInfo(userCred, netJsonArray[idx]) + netArray := input.Networks + for idx := 0; idx < len(netArray); idx += 1 { + netConfig, err := parseNetworkInfo(userCred, netArray[idx]) if err != nil { return nil, httperrors.NewInputParameterError("parse network description error %s", err) } @@ -998,12 +986,12 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(netConfig.Driver) == 0 { netConfig.Driver = osProf.NetDriver } - data.Set(fmt.Sprintf("net.%d", idx), jsonutils.Marshal(netConfig)) + input.Networks[idx] = netConfig } - isoDevArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") + isoDevArray := input.IsolatedDevices for idx := 0; idx < len(isoDevArray); idx += 1 { // .Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - if jsonutils.QueryBoolean(data, "backup", false) { + if input.Backup { return nil, httperrors.NewBadRequestError("Cannot create backup with isolated device") } devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, isoDevArray[idx]) @@ -1014,36 +1002,33 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, err } - data.Set(fmt.Sprintf("isolated_device.%d", idx), jsonutils.Marshal(devConfig)) + input.IsolatedDevices[idx] = devConfig } - keypairId, _ := data.GetString("keypair") - if len(keypairId) == 0 { - keypairId, _ = data.GetString("keypair_id") - } + keypairId := input.Keypair if len(keypairId) > 0 { keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Keypair %s not found", keypairId) } - data.Add(jsonutils.NewString(keypairObj.GetId()), "keypair_id") + input.Keypair = keypairObj.GetId() } else { - data.Add(jsonutils.NewString("None"), "keypair_id") + input.Keypair = "None" // TODO: ??? None? } - if data.Contains("secgroup") { - secGrpId, _ := data.GetString("secgroup") + if input.Secgroup != "" { + secGrpId := input.Secgroup secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred, secGrpId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Secgroup %s not found", secGrpId) } - data.Add(jsonutils.NewString(secGrpObj.GetId()), "secgrp_id") + input.Secgroup = secGrpObj.GetId() } else { - data.Add(jsonutils.NewString("default"), "secgrp_id") + input.Secgroup = "default" } - eipStr, _ := data.GetString("eip") - eipBw, _ := data.Int("eip_bw") + eipStr := input.Eip + eipBw := input.EipBw if len(eipStr) > 0 || eipBw > 0 { if !GetDriver(hypervisor).IsSupportEip() { return nil, httperrors.NewNotImplementedError("eip not supported for %s", hypervisor) @@ -1065,14 +1050,14 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if eip.IsAssociated() { return nil, httperrors.NewResourceBusyError("eip %s has been associated", eipStr) } - data.Set("eip_id", jsonutils.NewString(eipObj.GetId())) + input.Eip = eipObj.GetId() eipRegion := eip.GetRegion() preferRegionId, _ := data.GetString("prefer_region_id") if len(preferRegionId) > 0 && preferRegionId != eipRegion.Id { return nil, httperrors.NewConflictError("cannot assoicate with eip %s: different region", eipStr) } - data.Set("prefer_region_id", jsonutils.NewString(eipRegion.Id)) + input.PreferRegion = eipRegion.Id } else { // create new eip } @@ -1085,30 +1070,33 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m }*/ - data, err = GetDriver(hypervisor).ValidateCreateData(ctx, userCred, data) + input, err = GetDriver(hypervisor).ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) + data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, input.JSON(input)) if err != nil { return nil, err } + if err := data.Unmarshal(input); err != nil { + return nil, err + } - if !jsonutils.QueryBoolean(data, "is_system", false) { - err = manager.checkCreateQuota(ctx, userCred, ownerProjId, data, - jsonutils.QueryBoolean(data, "backup", false)) + if !input.IsSystem { + err = manager.checkCreateQuota(ctx, userCred, ownerProjId, input, + input.Backup) if err != nil { return nil, err } } - data.Add(jsonutils.NewString(ownerProjId), "owner_tenant_id") - return data, nil + input.Project = ownerProjId + return input.JSON(input), nil } -func (manager *SGuestManager) checkCreateQuota(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, data *jsonutils.JSONDict, hasBackup bool) error { - req := getGuestResourceRequirements(ctx, userCred, data, 1, hasBackup) +func (manager *SGuestManager) checkCreateQuota(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, input *api.ServerCreateInput, hasBackup bool) error { + req := getGuestResourceRequirements(ctx, userCred, input, 1, hasBackup) err := QuotaManager.CheckSetPendingQuota(ctx, userCred, ownerProjId, &req) if err != nil { return httperrors.NewOutOfQuotaError(err.Error()) @@ -1133,35 +1121,27 @@ func (self *SGuest) checkUpdateQuota(ctx context.Context, userCred mcclient.Toke return err } -func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, count int, hasBackup bool) SQuota { - vcpuCount, _ := data.Int("vcpu_count") +func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, count int, hasBackup bool) SQuota { + vcpuCount := input.VcpuCount if vcpuCount == 0 { vcpuCount = 1 } - vmemSize, _ := data.Int("vmem_size") + vmemSize := input.VmemSize diskSize := 0 - diskJsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 0; idx < len(diskJsonArray); idx += 1 { // data.Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { - diskConfig, _ := parseDiskInfo(ctx, userCred, diskJsonArray[idx]) + for _, diskConfig := range input.Disks { diskSize += diskConfig.SizeMb } - isoDevArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") - devCount := len(isoDevArray) - // for idx := 0; data.Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - // devCount += 1 - //} + devCount := len(input.IsolatedDevices) eNicCnt := 0 iNicCnt := 0 eBw := 0 iBw := 0 - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - for idx := 0; idx < len(netJsonArray); idx += 1 { // .Contains(fmt.Sprintf("net.%d", idx)); idx += 1 { - netConfig, _ := parseNetworkInfo(userCred, netJsonArray[idx]) + for _, netConfig := range input.Networks { if isExitNetworkInfo(netConfig) { eNicCnt += 1 eBw += netConfig.BwLimit @@ -1177,7 +1157,7 @@ func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCr } eipCnt := 0 - eipBw, _ := data.Int("eip_bw") + eipBw := input.EipBw if eipBw > 0 { eipCnt = 1 } @@ -1233,8 +1213,9 @@ func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred } func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { - pendingUsage := getGuestResourceRequirements(ctx, userCred, data, len(items), - jsonutils.QueryBoolean(data, "backup", false)) + input := new(api.ServerCreateInput) + data.Unmarshal(input) + pendingUsage := getGuestResourceRequirements(ctx, userCred, input, len(items), input.Backup) RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "GuestBatchCreateTask") } @@ -2515,23 +2496,22 @@ func totalGuestResourceCount( return stat } -func (self *SGuest) getDefaultNetworkConfig() *SNetworkConfig { - netConf := SNetworkConfig{} +func (self *SGuest) getDefaultNetworkConfig() *api.NetworkConfig { + netConf := api.NetworkConfig{} netConf.BwLimit = options.Options.DefaultBandwidth osProf := self.getOSProfile() netConf.Driver = osProf.NetDriver return &netConf } -func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, data *jsonutils.JSONDict, pendingUsage quotas.IQuota) error { - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - if len(netJsonArray) == 0 { +func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netArray []*api.NetworkConfig, pendingUsage quotas.IQuota) error { + if len(netArray) == 0 { netConfig := self.getDefaultNetworkConfig() _, err := self.attach2RandomNetwork(ctx, userCred, host, netConfig, pendingUsage) return err } - for idx := 0; idx < len(netJsonArray); idx += 1 { - netConfig, err := parseNetworkInfo(userCred, netJsonArray[idx]) + for _, netConfig := range netArray { + netConfig, err := parseNetworkInfo(userCred, netConfig) if err != nil { return err } @@ -2543,7 +2523,7 @@ func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient. return nil } -func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { var gns []SGuestnetwork var err1, err2 error if len(netConfig.Network) > 0 { @@ -2563,7 +2543,7 @@ func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.To } } -func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { driver := self.GetDriver() net, nicConfs, allocDir := driver.GetNamedNetworkConfiguration(self, userCred, host, netConfig) if net != nil { @@ -2582,33 +2562,38 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie } } -func (self *SGuest) attach2RandomNetwork(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2RandomNetwork(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { driver := self.GetDriver() return driver.Attach2RandomNetwork(self, ctx, userCred, host, netConfig, pendingUsage) } -func (self *SGuest) CreateDisksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, - data *jsonutils.JSONDict, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) error { - diskJsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 0; idx < len(diskJsonArray); idx += 1 { // .Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { - diskConfig, err := parseDiskInfo(ctx, userCred, diskJsonArray[idx]) +func (self *SGuest) CreateDisksOnHost( + ctx context.Context, + userCred mcclient.TokenCredential, + host *SHost, + disks []*api.DiskConfig, + pendingUsage quotas.IQuota, + inheritBilling bool, + isWithServerCreate bool, + candidate *schedapi.CandidateResource, +) error { + for idx := 0; idx < len(disks); idx += 1 { + diskConfig, err := parseDiskInfo(ctx, userCred, disks[idx]) if err != nil { return err } - disk, err := self.createDiskOnHost(ctx, userCred, host, diskConfig, pendingUsage, inheritBilling, isWithServerCreate) + disk, err := self.createDiskOnHost(ctx, userCred, host, diskConfig, pendingUsage, inheritBilling, isWithServerCreate, candidate.Disks[idx]) if err != nil { return err } - data.Add(jsonutils.NewString(disk.Id), fmt.Sprintf("disk.%d.id", idx)) - if len(diskConfig.SnapshotId) > 0 { - data.Add(jsonutils.NewString(diskConfig.SnapshotId), fmt.Sprintf("disk.%d.snapshot", idx)) - } + diskConfig.DiskId = disk.Id + disks[idx] = diskConfig } return nil } func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.TokenCredential, storage *SStorage, - diskConfig *SDiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { + diskConfig *api.DiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { lockman.LockObject(ctx, storage) defer lockman.ReleaseObject(ctx, storage) @@ -2642,9 +2627,26 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T return disk, nil } -func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, - diskConfig *SDiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { - storage := self.GetDriver().ChooseHostStorage(host, diskConfig.Backend) +func (self *SGuest) ChooseHostStorage(host *SHost, backend string, candidate *schedapi.CandidateDisk) *SStorage { + log.Errorf("==========candidate %#v", candidate) + if candidate == nil { + return self.GetDriver().ChooseHostStorage(host, backend) + } + return StorageManager.FetchStorageById(candidate.StorageId) +} + +func (self *SGuest) createDiskOnHost( + ctx context.Context, + userCred mcclient.TokenCredential, + host *SHost, + diskConfig *api.DiskConfig, + pendingUsage quotas.IQuota, + inheritBilling bool, + isWithServerCreate bool, + candidate *schedapi.CandidateDisk, +) (*SDisk, error) { + storage := self.ChooseHostStorage(host, diskConfig.Backend, candidate) + log.Debugf("Choose storage %s:%s for disk %#v", storage.Name, storage.Id, diskConfig) if storage == nil { return nil, fmt.Errorf("No storage on %s to create disk for %s", host.GetName(), diskConfig.Backend) } @@ -2652,6 +2654,7 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke if err != nil { return nil, err } + // TODO: use scheduler candidate storage if len(self.BackupHostId) > 0 { backupHost := HostManager.FetchHostById(self.BackupHostId) backupStorage := self.GetDriver().ChooseHostStorage(backupHost, diskConfig.Backend) @@ -2669,14 +2672,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke return disk, err } -func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, data *jsonutils.JSONDict, pendingUsage quotas.IQuota) error { - devJsonArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") - for idx := 0; idx < len(devJsonArray); idx += 1 { // .Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, devJsonArray[idx]) - if err != nil { - return err - } - err = self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) +func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { + for idx := 0; idx < len(devs); idx += 1 { + devConfig := devs[idx] + err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) if err != nil { return err } @@ -2684,7 +2683,7 @@ func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcc return nil } -func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *SIsolatedDeviceConfig, pendingUsage quotas.IQuota) error { +func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { lockman.LockClass(ctx, QuotaManager, self.ProjectId) defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId) @@ -2866,24 +2865,15 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To return nil } -type SDeployConfig struct { - Path string - Action string - Content string -} - func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, params *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { config := jsonutils.NewDict() desc := self.GetDriver().GetJsonDescAtHost(ctx, userCred, self, host) config.Add(desc, "desc") - deploys := make([]SDeployConfig, 0) - for idx := 0; params.Contains(fmt.Sprintf("deploy.%d.path", idx)); idx += 1 { - path, _ := params.GetString(fmt.Sprintf("deploy.%d.path", idx)) - action, _ := params.GetString(fmt.Sprintf("deploy.%d.action", idx)) - content, _ := params.GetString(fmt.Sprintf("deploy.%d.content", idx)) - deploys = append(deploys, SDeployConfig{Path: path, Action: action, Content: content}) + deploys, err := cmdline.FetchDeployConfigsByJSON(params) + if err != nil { + return nil, err } if len(deploys) > 0 { @@ -3856,34 +3846,39 @@ func (self *SGuest) getDefaultStorageType() string { return STORAGE_LOCAL } -func (self *SGuest) getSchedDesc() jsonutils.JSONObject { - desc := jsonutils.NewDict() +func (self *SGuest) getSchedDesc() *schedapi.ScheduleInput { + desc := new(schedapi.ScheduleInput) - desc.Add(jsonutils.NewString(self.Id), "id") - desc.Add(jsonutils.NewString(self.Name), "name") - desc.Add(jsonutils.NewInt(int64(self.VmemSize)), "vmem_size") - desc.Add(jsonutils.NewInt(int64(self.VcpuCount)), "vcpu_count") + desc.Id = self.Id + desc.Name = self.Name + desc.Memory = self.VmemSize + desc.Ncpu = int(self.VcpuCount) gds := self.GetDisks() if gds != nil { for i := 0; i < len(gds); i += 1 { - desc.Add(jsonutils.Marshal(gds[i].ToDiskInfo()), fmt.Sprintf("disk.%d", i)) + conf := new(api.DiskConfig) + jsonutils.Marshal(gds[i].ToDiskInfo()).Unmarshal(conf) + desc.Disks = append(desc.Disks, conf) } } gns, _ := self.GetNetworks("") if gns != nil { for i := 0; i < len(gns); i += 1 { - desc.Add(jsonutils.NewString(fmt.Sprintf("%s:%s", gns[i].NetworkId, gns[i].IpAddr)), fmt.Sprintf("net.%d", i)) + desc.Networks = append(desc.Networks, &api.NetworkConfig{ + Network: gns[i].NetworkId, + Address: gns[i].IpAddr, + }) } } if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { - desc.Add(jsonutils.NewString(self.HostId), "host_id") + desc.HostId = self.HostId } - desc.Add(jsonutils.NewString(self.ProjectId), "owner_tenant_id") - desc.Add(jsonutils.NewString(self.GetHypervisor()), "hypervisor") + desc.Project = self.ProjectId + desc.Hypervisor = self.GetHypervisor() return desc } @@ -3896,28 +3891,31 @@ func (self *SGuest) GetApptags() []string { return nil } -func (self *SGuest) ToSchedDesc() *jsonutils.JSONDict { - desc := jsonutils.NewDict() - desc.Set("id", jsonutils.NewString(self.Id)) - desc.Set("name", jsonutils.NewString(self.Name)) - desc.Set("vmem_size", jsonutils.NewInt(int64(self.VmemSize))) - desc.Set("vcpu_count", jsonutils.NewInt(int64(self.VcpuCount))) - self.FillGroupSchedDesc(desc) - self.FillDiskSchedDesc(desc) - self.FillNetSchedDesc(desc) - if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { - desc.Set("host_id", jsonutils.NewString(self.HostId)) +func (self *SGuest) ToSchedDesc() *schedapi.ScheduleInput { + desc := new(schedapi.ScheduleInput) + config := &schedapi.ServerConfig{ + Name: self.Name, + Memory: self.VmemSize, + Ncpu: int(self.VcpuCount), } - desc.Set("owner_tenant_id", jsonutils.NewString(self.ProjectId)) - tags := self.GetApptags() + //self.FillGroupSchedDesc(desc) + self.FillDiskSchedDesc(config) + self.FillNetSchedDesc(config) + if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { + config.HostId = self.HostId + } + config.Project = self.ProjectId + /*tags := self.GetApptags() for i := 0; i < len(tags); i++ { desc.Set(tags[i], jsonutils.JSONTrue) - } - desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) + }*/ + + config.Hypervisor = self.GetHypervisor() + desc.ServerConfig = *config return desc } -func (self *SGuest) FillGroupSchedDesc(desc *jsonutils.JSONDict) { +/*func (self *SGuest) FillGroupSchedDesc(desc *schedapi.ServerConfig) { groups := make([]SGroupguest, 0) err := GroupguestManager.Query().Equals("guest_id", self.Id).All(&groups) if err != nil { @@ -3928,31 +3926,29 @@ func (self *SGuest) FillGroupSchedDesc(desc *jsonutils.JSONDict) { desc.Set(fmt.Sprintf("srvtag.%d", i), jsonutils.NewString(fmt.Sprintf("%s:%s", groups[i].SrvtagId, groups[i].Tag))) } -} +}*/ -func (self *SGuest) FillDiskSchedDesc(desc *jsonutils.JSONDict) { +func (self *SGuest) FillDiskSchedDesc(desc *schedapi.ServerConfig) { guestDisks := make([]SGuestdisk, 0) err := GuestdiskManager.Query().Equals("guest_id", self.Id).All(&guestDisks) if err != nil { - log.Errorln(err) + log.Errorln("FillDiskSchedDesc: %v", err) return } for i := 0; i < len(guestDisks); i++ { - desc.Set(fmt.Sprintf("disk.%d", i), jsonutils.Marshal(guestDisks[i].ToDiskInfo())) + desc.Disks = append(desc.Disks, guestDisks[i].ToDiskConfig()) } } -func (self *SGuest) FillNetSchedDesc(desc *jsonutils.JSONDict) { +func (self *SGuest) FillNetSchedDesc(desc *schedapi.ServerConfig) { guestNetworks := make([]SGuestnetwork, 0) err := GuestnetworkManager.Query().Equals("guest_id", self.Id).All(&guestNetworks) if err != nil { - log.Errorln(err) + log.Errorln("FillNetSchedDesc: %v", err) return } for i := 0; i < len(guestNetworks); i++ { - desc.Set(fmt.Sprintf("net.%d", i), - jsonutils.NewString(fmt.Sprintf("%s:%s", - guestNetworks[i].NetworkId, guestNetworks[i].IpAddr))) + desc.Networks = append(desc.Networks, guestNetworks[i].ToNetworkConfig()) } } diff --git a/pkg/compute/models/helper.go b/pkg/compute/models/helper.go index f573990cd4..6a1a628a9b 100644 --- a/pkg/compute/models/helper.go +++ b/pkg/compute/models/helper.go @@ -3,12 +3,12 @@ package models import ( "context" "database/sql" - "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -38,23 +38,20 @@ func RunBatchCreateTask( } } -func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, hypervisor string) (*jsonutils.JSONDict, error) { +func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, hypervisor string) (*api.ServerCreateInput, error) { var err error - if jsonutils.QueryBoolean(data, "baremetal", false) { + if input.Baremetal { hypervisor = HYPERVISOR_BAREMETAL } // base validate_create_data - if (data.Contains("prefer_baremetal") || data.Contains("prefer_host")) && hypervisor != HYPERVISOR_CONTAINER { + if (input.PreferHost != "") && hypervisor != HYPERVISOR_CONTAINER { if !userCred.IsAdminAllow(consts.GetServiceType(), GuestManager.KeywordPlural(), policy.PolicyActionPerform, "assign-host") { return nil, httperrors.NewNotSufficientPrivilegeError("Only system admin can specify preferred host") } - bmName, _ := data.GetString("prefer_host") - if len(bmName) == 0 { - bmName, _ = data.GetString("prefer_baremetal") - } + bmName := input.PreferHost bmObj, err := HostManager.FetchByIdOrName(nil, bmName) if err != nil { if err == sql.ErrNoRows { @@ -80,45 +77,35 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred hypervisor = HYPERVISOR_DEFAULT } - _, err = GetDriver(hypervisor).ValidateCreateHostData(ctx, userCred, bmName, baremetal, data) + _, err = GetDriver(hypervisor).ValidateCreateHostData(ctx, userCred, bmName, baremetal, input) if err != nil { return nil, err } - data.Set("prefer_baremetal_id", jsonutils.NewString(baremetal.Id)) - data.Set("prefer_host_id", jsonutils.NewString(baremetal.Id)) + input.PreferHost = baremetal.Id zone := baremetal.GetZone() - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) + input.PreferRegion = region.Id } else { schedtags := make(map[string]string) - if data.Contains("aggregate_strategy") { - err = data.Unmarshal(&schedtags, "aggregate_strategy") - if err != nil { - return nil, httperrors.NewInputParameterError("invalid aggregate_strategy") - } - } - aggArray := jsonutils.GetArrayOfPrefix(data, "schedtag") - for idx := 0; idx < len(aggArray); idx += 1 { // data.Contains(fmt.Sprintf("schedtag.%d", idx)); idx += 1 { - aggStr, _ := aggArray[idx].GetString() // .GetString(fmt.Sprintf("schedtag.%d", idx)) - if len(aggStr) > 0 { - parts := strings.Split(aggStr, ":") - if len(parts) >= 2 && len(parts[0]) > 0 && len(parts[1]) > 0 { - schedtags[parts[0]] = parts[1] - } - } + for _, tag := range input.Schedtags { + schedtags[tag.Id] = tag.Strategy } if len(schedtags) > 0 { schedtags, err = SchedtagManager.ValidateSchedtags(userCred, schedtags) if err != nil { return nil, httperrors.NewInputParameterError("invalid aggregate_strategy: %s", err) } - data.Add(jsonutils.Marshal(schedtags), "aggregate_strategy") + tags := make([]*api.SchedtagConfig, 0) + for name, strategy := range schedtags { + tags = append(tags, &api.SchedtagConfig{Id: name, Strategy: strategy}) + } + input.Schedtags = tags } - if data.Contains("prefer_wire") { - wireStr, _ := data.GetString("prefer_wire") + if input.PreferWire != "" { + wireStr := input.PreferWire wireObj, err := WireManager.FetchById(wireStr) if err != nil { if err == sql.ErrNoRows { @@ -128,13 +115,13 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } wire := wireObj.(*SWire) - data.Set("prefer_wire_id", jsonutils.NewString(wire.Id)) + input.PreferWire = wire.Id zone := wire.GetZone() - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) - } else if data.Contains("prefer_zone") { - zoneStr, _ := data.GetString("prefer_zone") + input.PreferRegion = region.Id + } else if input.PreferZone != "" { + zoneStr := input.PreferZone zoneObj, err := ZoneManager.FetchById(zoneStr) if err != nil { if err == sql.ErrNoRows { @@ -144,11 +131,11 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } zone := zoneObj.(*SZone) - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) - } else if data.Contains("prefer_region") { - regionStr, _ := data.GetString("prefer_region") + input.PreferRegion = region.Id + } else if input.PreferRegion != "" { + regionStr := input.PreferRegion regionObj, err := CloudregionManager.FetchById(regionStr) if err != nil { if err == sql.ErrNoRows { @@ -158,7 +145,7 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } region := regionObj.(*SCloudregion) - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) + input.PreferRegion = region.Id } } @@ -171,6 +158,6 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred return nil, httperrors.NewInputParameterError("Hypervisor %s not supported", hypervisor) } - data.Add(jsonutils.NewString(hypervisor), "hypervisor") - return data, nil + input.Hypervisor = hypervisor + return input, nil } diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index c54e7a9381..83bc5867fe 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -13,6 +13,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -534,7 +535,7 @@ func (self *SHost) BorrowIpAddrsFromGuest(ctx context.Context, userCred mcclient return nil } -func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { ihost, err := host.GetIHost() if err != nil { log.Errorf("host.GetIHost fail %s", err) @@ -555,6 +556,7 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC netifs := host.GetNetInterfaces() netIdx := 0 + input.Networks = make([]*api.NetworkConfig, 0) for i := 0; i < len(netifs); i += 1 { hn := netifs[i].GetBaremetalNetwork() if hn != nil { @@ -563,12 +565,15 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC return nil, err } packedMac := strings.Replace(netifs[i].Mac, ":", "", -1) - netDesc := fmt.Sprintf("%s:%s:%s:[reserved]", packedMac, hn.IpAddr, hn.NetworkId) - params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.NewString(netDesc)) + input.Networks = append(input.Networks, &api.NetworkConfig{ + Network: hn.NetworkId, + Mac: packedMac, + Address: hn.IpAddr, + }) netIdx += 1 } } - params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.JSONNull) + //params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.JSONNull) for i := 0; i < len(idisks); i += 1 { /*istorage, err := idisks[i].GetIStorage() @@ -577,31 +582,34 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC return nil, err }*/ - key := fmt.Sprintf("disk.%d", i) - jsonConf, _ := params.Get(key) - if jsonConf != nil && jsonConf != jsonutils.JSONNull { - conf, err := parseDiskInfo(ctx, userCred, jsonConf) + var diskConfig *api.DiskConfig + if i < len(input.Disks) { + diskConfig = input.Disks[i] + diskConfig, err := parseDiskInfo(ctx, userCred, diskConfig) if err != nil { - log.Debugf("parseDiskInfo %s fail %s", jsonConf, err) + log.Debugf("parseDiskInfo %s fail %s", diskConfig, err) return nil, err } - conf.SizeMb = idisks[i].GetDiskSizeMB() - conf.Backend = STORAGE_LOCAL - params.Set(key, jsonutils.Marshal(conf)) + diskConfig.SizeMb = idisks[i].GetDiskSizeMB() + diskConfig.Backend = STORAGE_LOCAL + input.Disks[i] = diskConfig } else { - strConf := fmt.Sprintf("%s:%d", STORAGE_LOCAL, idisks[i].GetDiskSizeMB()) - conf, err := parseDiskInfo(ctx, userCred, jsonutils.NewString(strConf)) + conf := &api.DiskConfig{ + SizeMb: idisks[i].GetDiskSizeMB(), + Backend: STORAGE_LOCAL, + } + conf, err = parseDiskInfo(ctx, userCred, conf) if err != nil { return nil, err } - params.Set(key, jsonutils.Marshal(conf)) + input.Disks = append(input.Disks, conf) } } - params.Set(fmt.Sprintf("disk.%d", len(idisks)), jsonutils.JSONNull) + //params.Set(fmt.Sprintf("disk.%d", len(idisks)), jsonutils.JSONNull) // log.Debugf("params after rebuid: %s", params.String()) - return params, nil + return input, nil } func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 10bd0a64e5..5f88d90464 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3020,7 +3020,6 @@ func (self *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCrede if bn != nil { return nil } - log.Errorf("==========EnableNetif %#v, net: %s, ipAddr: %s, allocDir: %s, reserve: %v, requireDesignatedIp: %v", netif, network, ipAddr, allocDir, reserve, requireDesignatedIp) var net *SNetwork var err error if len(ipAddr) > 0 { diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 13221ef09d..c8358842d7 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -9,39 +9,33 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) const ( - DIRECT_PCI_TYPE = "PCI" - GPU_HPC_TYPE = "GPU-HPC" // # for compute - GPU_VGA_TYPE = "GPU-VGA" // # for display - USB_TYPE = "USB" - NIC_TYPE = "NIC" + DIRECT_PCI_TYPE = api.DIRECT_PCI_TYPE + GPU_HPC_TYPE = api.GPU_HPC_TYPE // # for compute + GPU_VGA_TYPE = api.GPU_VGA_TYPE // # for display + USB_TYPE = api.USB_TYPE + NIC_TYPE = api.NIC_TYPE - NVIDIA_VENDOR_ID = "10de" - AMD_VENDOR_ID = "1002" + NVIDIA_VENDOR_ID = api.NVIDIA_VENDOR_ID + AMD_VENDOR_ID = api.AMD_VENDOR_ID ) -var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} +var VALID_GPU_TYPES = api.VALID_GPU_TYPES -var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE} +var VALID_PASSTHROUGH_TYPES = api.VALID_PASSTHROUGH_TYPES -var ID_VENDOR_MAP = map[string]string{ - NVIDIA_VENDOR_ID: "NVIDIA", - AMD_VENDOR_ID: "AMD", -} +var ID_VENDOR_MAP = api.ID_VENDOR_MAP -var VENDOR_ID_MAP = map[string]string{ - "NVIDIA": NVIDIA_VENDOR_ID, - "AMD": AMD_VENDOR_ID, -} +var VENDOR_ID_MAP = api.VENDOR_ID_MAP type SIsolatedDeviceManager struct { db.SStandaloneResourceBaseManager @@ -228,43 +222,15 @@ func (self *SIsolatedDevice) isGpu() bool { return strings.HasPrefix(self.DevType, "GPU") } -type SIsolatedDeviceConfig struct { - Id string - DevType string - Model string - Vendor string -} - -func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SIsolatedDeviceConfig, error) { - devConfig := SIsolatedDeviceConfig{} - - devJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := devJson.Unmarshal(&devConfig) - if err != nil { - return nil, err - } - return &devConfig, nil - } - devStr, err := info.GetString() - if err != nil { - log.Errorf("invalid isolated device info format %s", err) - return nil, err - } +func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) { var devId, devType, devVendor string var matchDev *SIsolatedDevice - parts := strings.Split(devStr, ":") - for _, p := range parts { - if regutils.MatchUUIDExact(p) { - devId = p - } else if utils.IsInStringArray(p, VALID_PASSTHROUGH_TYPES) { - devType = p - } else if strings.HasPrefix(p, "vendor=") { - devVendor = p[len("vendor="):] - } else { - matchDev = manager.fuzzyMatchModel(p) - } - } + + devId = devConfig.Id + matchDev = manager.fuzzyMatchModel(devConfig.Model) + devVendor = devConfig.Vendor + devType = devConfig.DevType + if len(devId) == 0 { if matchDev == nil { return nil, fmt.Errorf("Isolated device info not contains either deviceID or model name") @@ -301,10 +267,10 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr if len(devType) > 0 { devConfig.DevType = devType } - return &devConfig, nil + return devConfig, nil } -func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *SIsolatedDeviceConfig) error { +func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *api.IsolatedDeviceConfig) error { if len(config.Id) > 0 { devObj, err := manager.FetchById(config.Id) if err != nil { @@ -318,7 +284,7 @@ func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *SIsolatedDevice return nil } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Id) > 0 { return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred) } else { @@ -326,7 +292,7 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context } } -func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.Context, guest *SGuest, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.Context, guest *SGuest, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { devObj, err := manager.FetchById(devConfig.Id) if devObj == nil { return fmt.Errorf("Device %s not found: %s", devConfig.Id, err) @@ -338,7 +304,7 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C return guest.attachIsolatedDevice(ctx, userCred, dev) } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Model) == 0 { return fmt.Errorf("Not found model from info: %s", devConfig) } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 01c79bd2ed..1236484d48 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -4,20 +4,18 @@ import ( "context" "database/sql" "fmt" - "strconv" - "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -31,39 +29,33 @@ import ( const ( // # DEFAULT_BANDWIDTH = options.default_bandwidth - MAX_BANDWIDTH = 100000 + MAX_BANDWIDTH = api.MAX_BANDWIDTH - NETWORK_TYPE_GUEST = "guest" - NETWORK_TYPE_BAREMETAL = "baremetal" - NETWORK_TYPE_CONTAINER = "container" - NETWORK_TYPE_PXE = "pxe" - NETWORK_TYPE_IPMI = "ipmi" + NETWORK_TYPE_GUEST = api.NETWORK_TYPE_GUEST + NETWORK_TYPE_BAREMETAL = api.NETWORK_TYPE_BAREMETAL + NETWORK_TYPE_CONTAINER = api.NETWORK_TYPE_CONTAINER + NETWORK_TYPE_PXE = api.NETWORK_TYPE_PXE + NETWORK_TYPE_IPMI = api.NETWORK_TYPE_IPMI - STATIC_ALLOC = "static" + STATIC_ALLOC = api.STATIC_ALLOC - MAX_NETWORK_NAME_LEN = 11 + MAX_NETWORK_NAME_LEN = api.MAX_NETWORK_NAME_LEN - EXTRA_DNS_UPDATE_TARGETS = "__extra_dns_update_targets" + EXTRA_DNS_UPDATE_TARGETS = api.EXTRA_DNS_UPDATE_TARGETS - NETWORK_STATUS_INIT = "init" - NETWORK_STATUS_PENDING = "pending" - NETWORK_STATUS_AVAILABLE = "available" - NETWORK_STATUS_FAILED = "failed" - NETWORK_STATUS_UNKNOWN = "unknown" - NETWORK_STATUS_START_DELETE = "start_delete" - NETWORK_STATUS_DELETING = "deleting" - NETWORK_STATUS_DELETED = "deleted" - NETWORK_STATUS_DELETE_FAILED = "delete_failed" + NETWORK_STATUS_INIT = api.NETWORK_STATUS_INIT + NETWORK_STATUS_PENDING = api.NETWORK_STATUS_PENDING + NETWORK_STATUS_AVAILABLE = api.NETWORK_STATUS_AVAILABLE + NETWORK_STATUS_FAILED = api.NETWORK_STATUS_FAILED + NETWORK_STATUS_UNKNOWN = api.NETWORK_STATUS_UNKNOWN + NETWORK_STATUS_START_DELETE = api.NETWORK_STATUS_START_DELETE + NETWORK_STATUS_DELETING = api.NETWORK_STATUS_DELETING + NETWORK_STATUS_DELETED = api.NETWORK_STATUS_DELETED + NETWORK_STATUS_DELETE_FAILED = api.NETWORK_STATUS_DELETE_FAILED ) var ( - ALL_NETWORK_TYPES = []string{ - NETWORK_TYPE_GUEST, - NETWORK_TYPE_BAREMETAL, - NETWORK_TYPE_CONTAINER, - NETWORK_TYPE_PXE, - NETWORK_TYPE_IPMI, - } + ALL_NETWORK_TYPES = api.ALL_NETWORK_TYPES ) type IPAddlocationDirection string @@ -706,119 +698,39 @@ func (manager *SNetworkManager) TotalPortCount( return NetworkPortStat{Count: ct, CountExt: ctExt} } -type SNetworkConfig struct { - Network string - Wire string - Exit bool - Private bool - Mac string - Address string - Address6 string - Driver string - BwLimit int - Vip bool - Reserved bool - Ifname string - NetType string - - RequireTeaming bool - TryTeaming bool - - StandbyPortCount int - StandbyAddrCount int -} - type SNicConfig struct { Mac string Index int8 Ifname string } -func parseNetworkInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SNetworkConfig, error) { - netConfig := SNetworkConfig{} - - netJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := netJson.Unmarshal(&netConfig) - if err != nil { +func parseNetworkInfo(userCred mcclient.TokenCredential, info *api.NetworkConfig) (*api.NetworkConfig, error) { + netObj, err := NetworkManager.FetchByIdOrName(userCred, info.Network) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(NetworkManager.Keyword(), info.Network) + } else { return nil, err } - return &netConfig, nil } - netStr, err := info.GetString() - if err != nil { - log.Errorf("invalid networkinfo format %s", err) - return nil, err + net := netObj.(*SNetwork) + if net.IsOwner(userCred) || net.IsPublic || db.IsAdminAllowGet(userCred, net) { + info.Network = netObj.GetId() + } else { + return nil, httperrors.NewForbiddenError("no allow to access network %s", info.Network) } - parts := strings.Split(netStr, ":") - for _, p := range parts { - if len(p) == 0 { - continue - } - if regutils.MatchIP4Addr(p) { - netConfig.Address = p - } else if regutils.MatchIP6Addr(p) { - netConfig.Address6 = p - } else if regutils.MatchCompactMacAddr(p) { - netConfig.Mac = netutils.MacUnpackHex(p) - } else if strings.HasPrefix(p, "wire=") { - netConfig.Wire = p[len("wire="):] - } else if p == "[random_exit]" { - netConfig.Exit = true - } else if p == "[random]" { - netConfig.Exit = false - } else if p == "[private]" { - netConfig.Private = true - } else if p == "[reserved]" { - netConfig.Reserved = true - } else if p == "[teaming]" { - netConfig.RequireTeaming = true - } else if p == "[try-teaming]" { - netConfig.TryTeaming = true - } else if strings.HasPrefix(p, "standby-port=") { - netConfig.StandbyPortCount, _ = strconv.Atoi(p[len("standby-port="):]) - } else if strings.HasPrefix(p, "standby-addr=") { - netConfig.StandbyAddrCount, _ = strconv.Atoi(p[len("standby-addr="):]) - } else if utils.IsInStringArray(p, []string{"virtio", "e1000", "vmxnet3"}) { - netConfig.Driver = p - } else if regutils.MatchSize(p) { - bw, err := fileutils.GetSizeMb(p, 'M', 1000) - if err != nil { - return nil, err - } - netConfig.BwLimit = bw - } else if p == "[vip]" { - netConfig.Vip = true - } else if utils.IsInStringArray(p, ALL_NETWORK_TYPES) { - netConfig.NetType = p - } else { - netObj, err := NetworkManager.FetchByIdOrName(userCred, p) - if err != nil { - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(NetworkManager.Keyword(), p) - } else { - return nil, err - } - } - net := netObj.(*SNetwork) - if net.IsOwner(userCred) || net.IsPublic || db.IsAdminAllowGet(userCred, net) { - netConfig.Network = netObj.GetId() - } else { - return nil, httperrors.NewForbiddenError("no allow to access network %s", p) - } - } + + if info.BwLimit == 0 { + info.BwLimit = options.Options.DefaultBandwidth } - if netConfig.BwLimit == 0 { - netConfig.BwLimit = options.Options.DefaultBandwidth - } - return &netConfig, nil + return info, nil } func (self *SNetwork) getFreeAddressCount() int { return self.getIPRange().AddressCount() - self.GetTotalNicCount() } -func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkConfig) error { +func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *api.NetworkConfig) error { if len(netConfig.Network) > 0 { netObj, err := NetworkManager.FetchByIdOrName(userCred, netConfig.Network) if err != nil { @@ -862,7 +774,7 @@ func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkCo return nil } -func isExitNetworkInfo(netConfig *SNetworkConfig) bool { +func isExitNetworkInfo(netConfig *api.NetworkConfig) bool { if len(netConfig.Network) > 0 { netObj, _ := NetworkManager.FetchById(netConfig.Network) net := netObj.(*SNetwork) diff --git a/pkg/compute/models/schedpolicies.go b/pkg/compute/models/schedpolicies.go index 5f2c957342..a70b0bf1fc 100644 --- a/pkg/compute/models/schedpolicies.go +++ b/pkg/compute/models/schedpolicies.go @@ -8,6 +8,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -160,14 +162,14 @@ func (self *SSchedpolicy) PerformEvaluate(ctx context.Context, userCred mcclient desc := server.getSchedDesc() params := jsonutils.NewDict() - params.Add(desc, "server") + params.Add(desc.JSON(desc), "server") meet, err := conditionparser.Eval(self.Condition, params) if err != nil { return nil, err } result := jsonutils.NewDict() - result.Add(desc, "server") + result.Add(desc.JSON(desc), "server") if meet { result.Add(jsonutils.JSONTrue, "result") } else { @@ -176,26 +178,25 @@ func (self *SSchedpolicy) PerformEvaluate(ctx context.Context, userCred mcclient return result, nil } -func ApplySchedPolicies(params *jsonutils.JSONDict) *jsonutils.JSONDict { +func ApplySchedPolicies(params *schedapi.ScheduleInput) *schedapi.ScheduleInput { policies := SchedpolicyManager.getAllEnabledPolicies() if policies == nil { log.Errorf("getAllEnabledPolicies fail") + //return jsonutils.Marshal(params).(*jsonutils.JSONDict) return params } schedtags := make(map[string]string) - if params.Contains("aggregate_strategy") { - err := params.Unmarshal(&schedtags, "aggregate_strategy") - if err != nil { - log.Errorf("unmarshall aggregate_strategy fail %s", err) - return params + if len(params.ServerConfig.Schedtags) != 0 { + for _, tag := range params.ServerConfig.Schedtags { + schedtags[tag.Id] = tag.Strategy } log.Infof("original sched tag %#v", schedtags) } input := jsonutils.NewDict() - input.Add(params, "server") + input.Add(jsonutils.Marshal(params), "server") for i := 0; i < len(policies); i += 1 { meet, err := conditionparser.Eval(policies[i].Condition, input) @@ -207,11 +208,15 @@ func ApplySchedPolicies(params *jsonutils.JSONDict) *jsonutils.JSONDict { } } - newSchedtags := jsonutils.Marshal(schedtags) - log.Infof("updated sched tag %s", newSchedtags) + log.Infof("updated sched tag %s", schedtags) - ret := jsonutils.NewDict() - ret.Add(newSchedtags, "aggregate_strategy") + params.ServerConfig.Schedtags = make([]*api.SchedtagConfig, 0) + for name, strategy := range schedtags { + params.ServerConfig.Schedtags = append(params.ServerConfig.Schedtags, &api.SchedtagConfig{ + Id: name, + Strategy: strategy, + }) + } - return ret + return params } diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index a5a3c68024..692d33c7cd 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -19,16 +20,16 @@ import ( type SchedStrategyType string const ( - STRATEGY_REQUIRE = "require" - STRATEGY_EXCLUDE = "exclude" - STRATEGY_PREFER = "prefer" - STRATEGY_AVOID = "avoid" + STRATEGY_REQUIRE = api.STRATEGY_REQUIRE + STRATEGY_EXCLUDE = api.STRATEGY_EXCLUDE + STRATEGY_PREFER = api.STRATEGY_PREFER + STRATEGY_AVOID = api.STRATEGY_AVOID // # container used aggregate - CONTAINER_AGGREGATE = "container" + CONTAINER_AGGREGATE = api.CONTAINER_AGGREGATE ) -var STRATEGY_LIST = []string{STRATEGY_REQUIRE, STRATEGY_EXCLUDE, STRATEGY_PREFER, STRATEGY_AVOID} +var STRATEGY_LIST = api.STRATEGY_LIST type ISchedtagJointManager interface { db.IJointModelManager diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 6357910f10..263c203a70 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -21,80 +22,68 @@ import ( ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - STORAGE_NFS = "nfs" + STORAGE_LOCAL = api.STORAGE_LOCAL + STORAGE_BAREMETAL = api.STORAGE_BAREMETAL + STORAGE_SHEEPDOG = api.STORAGE_SHEEPDOG + STORAGE_RBD = api.STORAGE_RBD + STORAGE_DOCKER = api.STORAGE_DOCKER + STORAGE_NAS = api.STORAGE_NAS + STORAGE_VSAN = api.STORAGE_VSAN + STORAGE_NFS = api.STORAGE_NFS - STORAGE_PUBLIC_CLOUD = "cloud" - STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" - STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 - STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" //单块本地SSD盘, 容量最大不能超过800 GiB + STORAGE_PUBLIC_CLOUD = api.STORAGE_PUBLIC_CLOUD + STORAGE_CLOUD_EFFICIENCY = api.STORAGE_CLOUD_EFFICIENCY + STORAGE_CLOUD_SSD = api.STORAGE_CLOUD_SSD + STORAGE_CLOUD_ESSD = api.STORAGE_CLOUD_ESSD //增强型(Enhanced)SSD 云盘 + STORAGE_EPHEMERAL_SSD = api.STORAGE_EPHEMERAL_SSD //单块本地SSD盘, 容量最大不能超过800 GiB //Azure hdd and ssd storagetype - STORAGE_STANDARD_LRS = "standard_lrs" - STORAGE_STANDARDSSD_LRS = "standardssd_lrs" - STORAGE_PREMIUM_LRS = "premium_lrs" + STORAGE_STANDARD_LRS = api.STORAGE_STANDARD_LRS + STORAGE_STANDARDSSD_LRS = api.STORAGE_STANDARDSSD_LRS + STORAGE_PREMIUM_LRS = api.STORAGE_PREMIUM_LRS // aws storage type - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_HDD = "sc1" // aws Cold HDD - STORAGE_STANDARD_HDD = "standard" // aws Magnetic volumes + STORAGE_GP2_SSD = api.STORAGE_GP2_SSD // aws general purpose ssd + STORAGE_IO1_SSD = api.STORAGE_IO1_SSD // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = api.STORAGE_ST1_HDD // aws Throughput Optimized HDD + STORAGE_SC1_HDD = api.STORAGE_SC1_HDD // aws Cold HDD + STORAGE_STANDARD_HDD = api.STORAGE_STANDARD_HDD // aws Magnetic volumes // qcloud storage type // STORAGE_CLOUD_SSD ="cloud_ssd" - STORAGE_LOCAL_BASIC = "local_basic" - STORAGE_LOCAL_SSD = "local_ssd" - STORAGE_CLOUD_BASIC = "cloud_basic" - STORAGE_CLOUD_PREMIUM = "cloud_premium" + STORAGE_LOCAL_BASIC = api.STORAGE_LOCAL_BASIC + STORAGE_LOCAL_SSD = api.STORAGE_LOCAL_SSD + STORAGE_CLOUD_BASIC = api.STORAGE_CLOUD_BASIC + STORAGE_CLOUD_PREMIUM = api.STORAGE_CLOUD_PREMIUM // huawei storage type - STORAGE_HUAWEI_SSD = "SSD" // 超高IO云硬盘 - STORAGE_HUAWEI_SAS = "SAS" // 高IO云硬盘 - STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘 + STORAGE_HUAWEI_SSD = api.STORAGE_HUAWEI_SSD // 超高IO云硬盘 + STORAGE_HUAWEI_SAS = api.STORAGE_HUAWEI_SAS // 高IO云硬盘 + STORAGE_HUAWEI_SATA = api.STORAGE_HUAWEI_SATA // 普通IO云硬盘 // openstack - STORAGE_OPENSTACK_ISCSI = "iscsi" + STORAGE_OPENSTACK_ISCSI = api.STORAGE_OPENSTACK_ISCSI ) const ( - STORAGE_ENABLED = "enabled" + STORAGE_ENABLED = api.STORAGE_ENABLED // STORAGE_DISABLED = "disabled" - STORAGE_OFFLINE = "offline" - STORAGE_ONLINE = "online" + STORAGE_OFFLINE = api.STORAGE_OFFLINE + STORAGE_ONLINE = api.STORAGE_ONLINE - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" - DISK_TYPE_HYBRID = "hybrid" + DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE + DISK_TYPE_SSD = api.DISK_TYPE_SSD + DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID ) var ( - DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} - STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} + DISK_TYPES = api.DISK_TYPES + STORAGE_LOCAL_TYPES = api.STORAGE_LOCAL_TYPES STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES - STORAGE_ALL_TYPES = []string{ - STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, - STORAGE_NFS, - } - STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, - STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_EPHEMERAL_SSD, STORAGE_CLOUD_EFFICIENCY, - STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS, - STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, - STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, - STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, - STORAGE_OPENSTACK_ISCSI, - } + STORAGE_ALL_TYPES = api.STORAGE_ALL_TYPES + STORAGE_TYPES = api.STORAGE_TYPES - STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} + STORAGE_LIMITED_TYPES = api.STORAGE_LIMITED_TYPES ) type SStorageManager struct { @@ -847,7 +836,7 @@ func (manager *SStorageManager) TotalCapacity(rangeObj db.IStandaloneModel, host return res1 } -func (self *SStorage) createDisk(name string, diskConfig *SDiskConfig, userCred mcclient.TokenCredential, +func (self *SStorage) createDisk(name string, diskConfig *api.DiskConfig, userCred mcclient.TokenCredential, ownerProjId string, autoDelete bool, isSystem bool, billingType string, billingCycle string, ) (*SDisk, error) { diff --git a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go index e6314034c5..1d1e47b49e 100644 --- a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go +++ b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -35,7 +36,12 @@ func (self *BaremetalConvertHypervisorTask) OnInit(ctx context.Context, obj db.I paramsDict := params.(*jsonutils.JSONDict) pendingUsage := models.SQuota{} self.GetPendingUsage(&pendingUsage) - guest.StartGuestCreateTask(ctx, self.UserCred, paramsDict, &pendingUsage, self.GetId()) + input, err := cmdline.FetchServerCreateInputByJSON(paramsDict) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + guest.StartGuestCreateTask(ctx, self.UserCred, input, &pendingUsage, self.GetId()) } func (self *BaremetalConvertHypervisorTask) OnGuestDeployComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { diff --git a/pkg/compute/tasks/disk_batch_create_task.go b/pkg/compute/tasks/disk_batch_create_task.go index 7948067510..af3ebc1750 100644 --- a/pkg/compute/tasks/disk_batch_create_task.go +++ b/pkg/compute/tasks/disk_batch_create_task.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -44,13 +46,30 @@ func (self *DiskBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandalo StartScheduleObjects(ctx, self, toSchedDisks) } +func (self *DiskBatchCreateTask) GetCreateInput() (*api.DiskCreateInput, error) { + input := new(api.DiskCreateInput) + err := self.GetParams().Unmarshal(input) + return input, err +} + +func (self *DiskBatchCreateTask) GetSchedParams() (*schedapi.ScheduleInput, error) { + input, err := self.GetCreateInput() + if err != nil { + return nil, err + } + ret := new(schedapi.ScheduleInput) + srvInput := input.ToServerCreateInput() + err = srvInput.JSON(srvInput).Unmarshal(ret) + return ret, err +} + func (self *DiskBatchCreateTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) { self.SSchedTask.OnScheduleFailCallback(ctx, obj, reason) disk := obj.(*models.SDisk) log.Errorf("Schedule disk %s failed", disk.Name) } -func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { var err error disk := obj.(*models.SDisk) pendingUsage := models.SQuota{} @@ -58,10 +77,25 @@ func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj ISc if err != nil { log.Errorf("GetPendingUsage fail %s", err) } - diskConfig := models.SDiskConfig{} - self.GetParams().Unmarshal(&diskConfig, "disk.0") + quotaStorage := models.SQuota{Storage: disk.DiskSize} - err = disk.SetStorageByHost(hostId, &diskConfig) + + onError := func(err error) { + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, disk.ProjectId, &pendingUsage, "aStorage) + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, err.Error()) + self.SetStageFailed(ctx, err.Error()) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE_FAIL, err, self.UserCred) + notifyclient.NotifySystemError(disk.Id, disk.Name, models.DISK_ALLOC_FAILED, err.Error()) + } + + diskConfig, err := self.GetFirstDisk() + if err != nil { + onError(err) + return + } + + //err = disk.SetStorageByHost(hostId, &diskConfig) + err = disk.SetStorage(candidate.Disks[0].StorageId, diskConfig) if err != nil { models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, disk.ProjectId, &pendingUsage, "aStorage) disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, err.Error()) diff --git a/pkg/compute/tasks/guest_backup_tasks.go b/pkg/compute/tasks/guest_backup_tasks.go index 3d88edbae7..1d90930cdc 100644 --- a/pkg/compute/tasks/guest_backup_tasks.go +++ b/pkg/compute/tasks/guest_backup_tasks.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -183,15 +184,15 @@ func (self *GuestCreateBackupTask) OnStartSchedule(obj IScheduleModel) { db.OpsLog.LogEvent(guest, db.ACT_START_CREATE_BACKUP, "", self.UserCred) } -func (self *GuestCreateBackupTask) GetSchedParams() *jsonutils.JSONDict { +func (self *GuestCreateBackupTask) GetSchedParams() (*schedapi.ScheduleInput, error) { obj := self.GetObject() guest := obj.(*models.SGuest) schedDesc := guest.ToSchedDesc() if self.Params.Contains("prefer_host_id") { - preferHostId, _ := self.Params.Get("prefer_host_id") - schedDesc.Set("prefer_host_id", preferHostId) + preferHostId, _ := self.Params.GetString("prefer_host_id") + schedDesc.ServerConfig.PreferHost = preferHostId } - return schedDesc + return schedDesc, nil } func (self *GuestCreateBackupTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) { @@ -204,9 +205,10 @@ func (self *GuestCreateBackupTask) OnScheduleFailed(ctx context.Context, reason self.TaskFailed(ctx, guest, reason) } -func (self *GuestCreateBackupTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, targetHostId string) { +func (self *GuestCreateBackupTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { guest := obj.(*models.SGuest) - targetHost := models.HostManager.FetchHostById(targetHostId) + targetHostId := candidate.HostId + targetHost := models.HostManager.FetchHostById(candidate.HostId) if targetHost == nil { self.TaskFailed(ctx, guest, "target host not found?") return diff --git a/pkg/compute/tasks/guest_batch_create_task.go b/pkg/compute/tasks/guest_batch_create_task.go index 398c751e31..54f3e2bb3f 100644 --- a/pkg/compute/tasks/guest_batch_create_task.go +++ b/pkg/compute/tasks/guest_batch_create_task.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -21,6 +23,12 @@ func init() { taskman.RegisterTask(GuestBatchCreateTask{}) } +func (self *GuestBatchCreateTask) GetCreateInput() (*api.ServerCreateInput, error) { + input := new(api.ServerCreateInput) + err := self.GetParams().Unmarshal(input) + return input, err +} + func (self *GuestBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { StartScheduleObjects(ctx, self, objs) } @@ -33,13 +41,13 @@ func (self *GuestBatchCreateTask) OnScheduleFailCallback(ctx context.Context, ob } } -func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) { +func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) { guest := obj.(*models.SGuest) - guest.SetHostIdWithBackup(self.UserCred, master, slave) + guest.SetHostIdWithBackup(self.UserCred, master.HostId, slave.HostId) self.SaveScheduleResult(ctx, obj, master) } -func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest *models.SGuest) error { +func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest *models.SGuest, candidate *schedapi.CandidateResource) error { pendingUsage := models.SQuota{} err := self.GetPendingUsage(&pendingUsage) if err != nil { @@ -51,18 +59,25 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest host := guest.GetHost() + input, err := self.GetCreateInput() + if host.IsPrepaidRecycle() { - params, err := host.SetGuestCreateNetworkAndDiskParams(ctx, self.UserCred, self.Params) + input, err = host.SetGuestCreateNetworkAndDiskParams(ctx, self.UserCred, input) if err != nil { log.Errorf("host.SetGuestCreateNetworkAndDiskParams fail %s", err) guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) return err } - self.Params = params + params := input.JSON(input) self.SaveParams(params) } - err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage) + input, err = self.GetCreateInput() + if err != nil { + guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) + return err + } + err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingUsage) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -71,8 +86,8 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - guest.GetDriver().PrepareDiskRaidConfig(self.UserCred, host, self.Params) - err = guest.CreateDisksOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage, true, true) + guest.GetDriver().PrepareDiskRaidConfig(self.UserCred, host, input.BaremetalDiskConfigs) + err = guest.CreateDisksOnHost(ctx, self.UserCred, host, input.Disks, &pendingUsage, true, true, candidate) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -81,7 +96,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - err = guest.CreateIsolatedDeviceOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage) + err = guest.CreateIsolatedDeviceOnHost(ctx, self.UserCred, host, input.IsolatedDevices, &pendingUsage) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -100,9 +115,12 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - autoStart := jsonutils.QueryBoolean(self.Params, "auto_start", false) - resetPassword := jsonutils.QueryBoolean(self.Params, "reset_password", true) - passwd, _ := self.Params.GetString("password") + autoStart := input.AutoStart + resetPassword := true + if input.ResetPassword != nil { + resetPassword = *input.ResetPassword + } + passwd := input.Password err = guest.StartRebuildRootTask(ctx, self.UserCred, "", false, autoStart, passwd, resetPassword, true) if err != nil { log.Errorf("start guest create task fail %s", err) @@ -112,7 +130,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return nil } - err = guest.StartGuestCreateTask(ctx, self.UserCred, self.Params, nil, self.GetId()) + err = guest.StartGuestCreateTask(ctx, self.UserCred, input, nil, self.GetId()) if err != nil { log.Errorf("start guest create task fail %s", err) guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) @@ -121,14 +139,15 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return nil } -func (self *GuestBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *GuestBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { var err error + hostId := candidate.HostId guest := obj.(*models.SGuest) if len(guest.HostId) == 0 { guest.OnScheduleToHost(ctx, self.UserCred, hostId) } - err = self.allocateGuestOnHost(ctx, guest) + err = self.allocateGuestOnHost(ctx, guest, candidate) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, err, self.UserCred) logclient.AddActionLogWithStartable(self, obj, logclient.ACT_ALLOCATE, err.Error(), self.GetUserCred(), false) diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go index d8b3446b9f..cb51f32eb5 100644 --- a/pkg/compute/tasks/guest_change_config_task.go +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -95,14 +96,14 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob } func (self *GuestChangeConfigTask) DoCreateDisksTask(ctx context.Context, guest *models.SGuest) { - iCreateData, err := self.Params.Get("create") - if err != nil || iCreateData == nil { + disks := make([]*api.DiskConfig, 0) + err := self.Params.Unmarshal(disks, "create") + if err != nil || len(disks) == 0 { self.OnCreateDisksComplete(ctx, guest, nil) return } - data := (iCreateData).(*jsonutils.JSONDict) self.SetStage("OnCreateDisksComplete", nil) - guest.StartGuestCreateDiskTask(ctx, self.UserCred, data, self.GetTaskId()) + guest.StartGuestCreateDiskTask(ctx, self.UserCred, disks, self.GetTaskId()) } func (self *GuestChangeConfigTask) OnCreateDisksCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 48f4b4a679..034af4e92a 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -39,7 +40,7 @@ func (self *GuestCreateDiskTask) OnDiskPreparedFailed(ctx context.Context, obj d /* --------------------------------------------- */ type KVMGuestCreateDiskTask struct { - SGuestBaseTask + SGuestCreateDiskBaseTask } func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -48,11 +49,16 @@ func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandalo } func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - var diskIndex = 0 var diskReady = true - for { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + + for _, d := range disks { + diskId := d.DiskId + if !diskReady { break } iDisk, err := models.DiskManager.FetchById(diskId) @@ -66,7 +72,7 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapshotId, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + snapshotId := d.SnapshotId err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshotId, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -75,14 +81,12 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d diskReady = false break } - diskIndex += 1 } - diskIndex = 0 - for { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { + for _, d := range disks { + if !diskReady { break } + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -97,7 +101,6 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d diskReady = false break } - diskIndex += 1 } if diskReady { guest := obj.(*models.SGuest) @@ -117,24 +120,34 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob self.SetStageComplete(ctx, nil) } -type ManagedGuestCreateDiskTask struct { +type SGuestCreateDiskBaseTask struct { SGuestBaseTask } +func (self *SGuestCreateDiskBaseTask) GetInputDisks() ([]api.DiskConfig, error) { + disks := make([]api.DiskConfig, 0) + err := self.GetParams().Unmarshal(&disks, "disks") + return disks, err +} + +type ManagedGuestCreateDiskTask struct { + SGuestCreateDiskBaseTask +} + func (self *ManagedGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { self.SetStage("on_managed_disk_prepared", nil) self.OnManagedDiskPrepared(ctx, obj, data) } func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - var diskIndex = 0 + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } - for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if err != nil { - self.SetStageFailed(ctx, err.Error()) - return - } + for _, d := range disks { + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -142,25 +155,20 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapInfo, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) - err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + snapshot := d.SnapshotId + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshot, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) return } return } - diskIndex += 1 } - diskIndex = 0 guest := obj.(*models.SGuest) - for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if err != nil { - return - } + for _, d := range disks { + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -185,7 +193,6 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex return } time.Sleep(time.Second * 5) - diskIndex += 1 } self.SetStageComplete(ctx, nil) @@ -198,7 +205,7 @@ func (self *ManagedGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context */ type ESXiGuestCreateDiskTask struct { - SGuestBaseTask + SGuestCreateDiskBaseTask } func (self *ESXiGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -209,14 +216,13 @@ func (self *ESXiGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandal return } - diskIndex := 0 - for { - diskKey := fmt.Sprintf("disk.%d.id", diskIndex) - if !self.Params.Contains(diskKey) { - break - } - diskId, _ := self.Params.GetString(diskKey) - diskIndex += 1 + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + for _, d := range disks { + diskId := d.DiskId guestDisk := guest.GetGuestDisk(diskId) if guestDisk == nil { self.SetStageFailed(ctx, "fail to find guestdisk") diff --git a/pkg/compute/tasks/guest_live_migrate_task.go b/pkg/compute/tasks/guest_live_migrate_task.go index 60b48a5724..bad4bf6e70 100644 --- a/pkg/compute/tasks/guest_live_migrate_task.go +++ b/pkg/compute/tasks/guest_live_migrate_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -31,15 +32,15 @@ func (self *GuestMigrateTask) OnInit(ctx context.Context, obj db.IStandaloneMode StartScheduleObjects(ctx, self, []db.IStandaloneModel{obj}) } -func (self *GuestMigrateTask) GetSchedParams() *jsonutils.JSONDict { +func (self *GuestMigrateTask) GetSchedParams() (*schedapi.ScheduleInput, error) { obj := self.GetObject() guest := obj.(*models.SGuest) schedDesc := guest.ToSchedDesc() if self.Params.Contains("prefer_host_id") { - preferHostId, _ := self.Params.Get("prefer_host_id") - schedDesc.Set("prefer_host_id", preferHostId) + preferHostId, _ := self.Params.GetString("prefer_host_id") + schedDesc.ServerConfig.PreferHost = preferHostId } - return schedDesc + return schedDesc, nil } func (self *GuestMigrateTask) OnStartSchedule(obj IScheduleModel) { @@ -58,7 +59,8 @@ func (self *GuestMigrateTask) OnScheduleFailed(ctx context.Context, reason strin self.TaskFailed(ctx, guest, reason) } -func (self *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, targetHostId string) { +func (self *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, target *schedapi.CandidateResource) { + targetHostId := target.HostId guest := obj.(*models.SGuest) targetHost := models.HostManager.FetchHostById(targetHostId) if targetHost == nil { diff --git a/pkg/compute/tasks/schedule.go b/pkg/compute/tasks/schedule.go index fc275ab941..f21d4a4b9c 100644 --- a/pkg/compute/tasks/schedule.go +++ b/pkg/compute/tasks/schedule.go @@ -7,6 +7,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -33,7 +36,7 @@ type IScheduleModel interface { type IScheduleTask interface { GetUserCred() mcclient.TokenCredential - GetSchedParams() *jsonutils.JSONDict + GetSchedParams() (*schedapi.ScheduleInput, error) GetPendingUsage(quota quotas.IQuota) error SetStage(stageName string, data *jsonutils.JSONDict) error SetStageFailed(ctx context.Context, reason string) @@ -41,17 +44,42 @@ type IScheduleTask interface { OnStartSchedule(obj IScheduleModel) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) OnScheduleComplete(ctx context.Context, items []db.IStandaloneModel, data *jsonutils.JSONDict) - SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) - SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) + SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) + SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) OnScheduleFailed(ctx context.Context, reason string) } type SSchedTask struct { taskman.STask + input *schedapi.ScheduleInput } -func (self *SSchedTask) GetSchedParams() *jsonutils.JSONDict { - return self.GetParams() +func (self *SSchedTask) GetSchedParams() (*schedapi.ScheduleInput, error) { + params := self.GetParams() + input, err := cmdline.FetchScheduleInputByJSON(params) + if err != nil { + return nil, fmt.Errorf("Unmarsh to schedule input: %v", err) + } + return input, err +} + +func (self *SSchedTask) GetDisks() ([]*api.DiskConfig, error) { + input, err := self.GetSchedParams() + if err != nil { + return nil, err + } + return input.Disks, nil +} + +func (self *SSchedTask) GetFirstDisk() (*api.DiskConfig, error) { + disks, err := self.GetDisks() + if err != nil { + return nil, err + } + if len(disks) == 0 { + return nil, fmt.Errorf("Empty disks to schedule") + } + return disks[0], nil } func (self *SSchedTask) OnStartSchedule(obj IScheduleModel) { @@ -70,11 +98,11 @@ func (self *SSchedTask) OnScheduleComplete(ctx context.Context, items []db.IStan self.SetStageComplete(ctx, nil) } -func (self *SSchedTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *SSchedTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { // ... } -func (self *SSchedTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) { +func (self *SSchedTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) { // ... } @@ -101,18 +129,23 @@ func doScheduleObjects( task IScheduleTask, objs []IScheduleModel, ) { - parmas := task.GetSchedParams() - schedtags := models.ApplySchedPolicies(parmas) - - task.SetStage("OnScheduleComplete", schedtags) - - s := auth.GetAdminSession(ctx, options.Options.Region, "") - results, err := modules.SchedManager.DoSchedule(s, parmas, len(objs)) + schedInput, err := task.GetSchedParams() if err != nil { onSchedulerRequestFail(ctx, task, objs, fmt.Sprintf("Scheduler fail: %s", err)) return } - onSchedulerResults(ctx, task, objs, results) + schedInput = models.ApplySchedPolicies(schedInput) + + params := jsonutils.Marshal(schedInput).(*jsonutils.JSONDict) + task.SetStage("OnScheduleComplete", params) + + s := auth.GetAdminSession(ctx, options.Options.Region, "") + output, err := modules.SchedManager.DoSchedule(s, schedInput, len(objs)) + if err != nil { + onSchedulerRequestFail(ctx, task, objs, fmt.Sprintf("Scheduler fail: %s", err)) + return + } + onSchedulerResults(ctx, task, objs, output.Candidates) } func cancelPendingUsage(ctx context.Context, task IScheduleTask) { @@ -122,7 +155,12 @@ func cancelPendingUsage(ctx context.Context, task IScheduleTask) { log.Errorf("Taks GetPendingUsage fail %s", err) return } - ownerProjectId, _ := task.GetSchedParams().GetString("owner_tenant_id") + schedInput, err := task.GetSchedParams() + if err != nil { + log.Errorf("cancelPendingUsage GetSchedParams fail: %s", err) + return + } + ownerProjectId := schedInput.ServerConfig.Project err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), ownerProjectId, &pendingUsage, &pendingUsage) if err != nil { log.Errorf("cancelpendingusage error %s", err) @@ -162,31 +200,24 @@ func onSchedulerResults( ctx context.Context, task IScheduleTask, objs []IScheduleModel, - results []jsonutils.JSONObject, + results []*schedapi.CandidateResource, ) { succCount := 0 for idx := 0; idx < len(objs); idx += 1 { obj := objs[idx] result := results[idx] - if result.Contains("candidate", "id") { - hostId, _ := result.GetString("candidate", "id") - onScheduleSucc(ctx, task, obj, hostId) - succCount += 1 - } else if result.Contains("candidate", "master_id") { - master, _ := result.GetString("candidate", "master_id") - slave, _ := result.GetString("candidate", "slave_id") - if len(master) == 0 || len(slave) == 0 { - onObjScheduleFail(ctx, task, obj, "Scheduler candidates not match") - } else { - onMasterSlaveScheduleSucc(ctx, task, obj, master, slave) - } - } else if result.Contains("error") { - msg, _ := result.Get("error") - onObjScheduleFail(ctx, task, obj, fmt.Sprintf("%s", msg)) + + if len(result.Error) != 0 { + onObjScheduleFail(ctx, task, obj, fmt.Sprintf("%s", result.Error)) + continue + } + + if result.BackupCandidate == nil { + // normal schedule + onScheduleSucc(ctx, task, obj, result) } else { - msg := fmt.Sprintf("Unknown scheduler result %s", result) - onObjScheduleFail(ctx, task, obj, msg) - return + // backup schedule + onMasterSlaveScheduleSucc(ctx, task, obj, result, result.BackupCandidate) } } if succCount == 0 { @@ -199,24 +230,25 @@ func onMasterSlaveScheduleSucc( ctx context.Context, task IScheduleTask, obj IScheduleModel, - master, slave string, + master, slave *schedapi.CandidateResource, ) { lockman.LockObject(ctx, obj) defer lockman.ReleaseObject(ctx, obj) task.SaveScheduleResultWithBackup(ctx, obj, master, slave) - models.HostManager.ClearSchedDescCache(master) - models.HostManager.ClearSchedDescCache(slave) + models.HostManager.ClearSchedDescCache(master.HostId) + models.HostManager.ClearSchedDescCache(slave.HostId) } func onScheduleSucc( ctx context.Context, task IScheduleTask, obj IScheduleModel, - hostId string, + candidate *schedapi.CandidateResource, ) { + hostId := candidate.HostId lockman.LockRawObject(ctx, models.HostManager.KeywordPlural(), hostId) defer lockman.ReleaseRawObject(ctx, models.HostManager.KeywordPlural(), hostId) - task.SaveScheduleResult(ctx, obj, hostId) - models.HostManager.ClearSchedDescCache(hostId) + task.SaveScheduleResult(ctx, obj, candidate) + models.HostManager.ClearSchedDescCache(candidate.HostId) } diff --git a/pkg/mcclient/modules/mod_scheduler.go b/pkg/mcclient/modules/mod_scheduler.go index aa54d66164..6561215861 100644 --- a/pkg/mcclient/modules/mod_scheduler.go +++ b/pkg/mcclient/modules/mod_scheduler.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" + + api "yunion.io/x/onecloud/pkg/apis/scheduler" ) var ( @@ -22,50 +24,31 @@ type SchedulerManager struct { ResourceManager } -func (this *SchedulerManager) DoScheduleListResult(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) (*ListResult, error) { - candidates, err := this.DoSchedule(s, params, count) - if err != nil { - return nil, err - } - ret := ListResult{Data: make([]jsonutils.JSONObject, len(candidates))} - for i, candidate := range candidates { - host, err := candidate.Get("candidate") - if err == nil { - ret.Data[i] = host - } else { - ret.Data[i] = candidate - } - } - return &ret, nil -} - -func (this *SchedulerManager) DoSchedule(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) ([]jsonutils.JSONObject, error) { +func (this *SchedulerManager) DoSchedule(s *mcclient.ClientSession, input *api.ScheduleInput, count int) (*api.ScheduleOutput, error) { url := fmt.Sprintf("/%s", this.Keyword) - body := jsonutils.NewDict() - body.Add(params, this.Keyword) if count <= 0 { count = 1 } - body.Add(jsonutils.NewInt(int64(count)), "count") - cands, err := this._post(s, url, body, this.Keyword) + input.Count = count + body := input.JSON(input) + ret, err := this._post(s, url, body, "") if err != nil { return nil, err } - candidates, err := cands.GetArray() + output := new(api.ScheduleOutput) + err = ret.Unmarshal(output) if err != nil { - return nil, fmt.Errorf("Not a valid response") + return nil, fmt.Errorf("Not a valid response: %v", err) } - return candidates, nil + return output, nil } -func (this *SchedulerManager) DoScheduleForecast(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) (bool, error) { - body := jsonutils.NewDict() - body.Add(params, this.Keyword) +func (this *SchedulerManager) DoScheduleForecast(s *mcclient.ClientSession, params *api.ScheduleInput, count int) (bool, error) { if count <= 0 { count = 1 } - body.Add(jsonutils.NewInt(int64(count)), "count") - res, err := this.DoForecast(s, body) + params.Count = count + res, err := this.DoForecast(s, params.JSON(params)) if err != nil { return false, err } @@ -81,9 +64,9 @@ func newSchedIdentURL(action, ident string) string { return fmt.Sprintf("%s/%s", newSchedURL(action), ident) } -func (this *SchedulerManager) Test(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (this *SchedulerManager) Test(s *mcclient.ClientSession, params *api.ScheduleInput) (jsonutils.JSONObject, error) { url := newSchedURL("test") - _, obj, err := this.jsonRequest(s, "POST", url, nil, params) + _, obj, err := this.jsonRequest(s, "POST", url, nil, params.JSON(params)) if err != nil { return nil, err } diff --git a/pkg/mcclient/options/disks.go b/pkg/mcclient/options/disks.go new file mode 100644 index 0000000000..9a937ffa50 --- /dev/null +++ b/pkg/mcclient/options/disks.go @@ -0,0 +1,49 @@ +package options + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" +) + +type DiskCreateOptions struct { + Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` + Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` + Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` + Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` + + NAME string `help:"Name of the disk"` + DISKDESC string `help:"Image size or size of virtual disk"` + Desc string `help:"Description" metavar:"Description"` + Storage string `help:"ID or name of storage where the disk is created"` + Hypervisor string `help:"Hypervisor of this disk, used by schedule"` + Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` + TaskNotify bool `help:"Setup task notify"` +} + +func (o DiskCreateOptions) Params() (*api.DiskCreateInput, error) { + config, err := cmdline.ParseDiskConfig(o.DISKDESC, 0) + if err != nil { + return nil, err + } + for _, desc := range o.Schedtag { + tag, err := cmdline.ParseSchedtagConfig(desc) + if err != nil { + return nil, err + } + config.Schedtags = append(config.Schedtags, tag) + } + params := &api.DiskCreateInput{ + PreferRegion: o.Region, + PreferZone: o.Zone, + PreferWire: o.Wire, + PreferHost: o.Host, + Description: o.Desc, + Name: o.NAME, + DiskConfig: config, + Hypervisor: o.Hypervisor, + } + if o.Storage != "" { + params.Storage = o.Storage + } + return params, nil +} diff --git a/pkg/mcclient/options/schedule.go b/pkg/mcclient/options/schedule.go deleted file mode 100644 index 599708a82e..0000000000 --- a/pkg/mcclient/options/schedule.go +++ /dev/null @@ -1,17 +0,0 @@ -package options - -import "yunion.io/x/jsonutils" - -type ScheduleOptions struct { - Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` - Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` - Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` - Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` - Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei"` - ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` -} - -func (opts *ScheduleOptions) Params() (*jsonutils.JSONDict, error) { - return optionsStructToParams(opts) -} diff --git a/pkg/mcclient/options/schedulers.go b/pkg/mcclient/options/schedulers.go index 85832ef394..f22a37f18c 100644 --- a/pkg/mcclient/options/schedulers.go +++ b/pkg/mcclient/options/schedulers.go @@ -1,90 +1,39 @@ package options import ( - "fmt" - - "yunion.io/x/jsonutils" - + "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/modules" ) type SchedulerTestBaseOptions struct { - Mem int64 `help:"Memory size (MB), default 512" metavar:"MEMORY" default:"512"` - Ncpu int64 `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` - Disk []string `help:"Disk descriptions" nargs:"+"` - DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` - BaremetalDiskConfig []string `help:"Baremetal disk layout configuration"` - Net []string `help:"Network descriptions" metavar:"NETWORK"` - IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` - Schedtag []string `help:"Schedule policy, key = SchedTag name, value = require|exclude|prefer|avoid" metavar:""` - Zone string `help:"Preferred zone where virtual server should be created"` - Host string `help:"Preferred host where virtual server should be created"` - Project string `help:"Owner project ID or Name"` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun"` - Count int64 `help:"Create multiple simultaneously, default 1" default:"1"` - Log bool `help:"Record to schedule history"` + *ServerConfigs + + Mem int `help:"Memory size (MB), default 512" metavar:"MEMORY" default:"512"` + Ncpu int `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` + Log bool `help:"Record to schedule history"` } -func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { - data := jsonutils.NewDict() +func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*scheduler.ServerConfig, error) { + config, err := o.ServerConfigs.Data() + if err != nil { + return nil, err + } + + data := new(scheduler.ServerConfig) + data.ServerConfigs = config + if o.Mem > 0 { - data.Add(jsonutils.NewInt(o.Mem), "vmem_size") + data.Memory = o.Mem } if o.Ncpu > 0 { - data.Add(jsonutils.NewInt(o.Ncpu), "vcpu_count") - } - for i, d := range o.Disk { - data.Add(jsonutils.NewString(d), fmt.Sprintf("disk.%d", i)) - } - for i, n := range o.Net { - data.Add(jsonutils.NewString(n), fmt.Sprintf("net.%d", i)) - } - for i, g := range o.IsolatedDevice { - data.Add(jsonutils.NewString(g), fmt.Sprintf("isolated_device.%d", i)) - } - if len(o.Host) > 0 { - data.Add(jsonutils.NewString(o.Host), "prefer_host") - } else { - if len(o.Zone) > 0 { - data.Add(jsonutils.NewString(o.Zone), "prefer_zone") - } - if len(o.Schedtag) > 0 { - for i, aggr := range o.Schedtag { - data.Add(jsonutils.NewString(aggr), fmt.Sprintf("aggregate.%d", i)) - } - } - } - if len(o.Project) > 0 { - ret, err := modules.Projects.Get(s, o.Project, nil) - if err != nil { - return nil, err - } - projectId, err := ret.GetString("id") - if err != nil { - return nil, err - } - data.Add(jsonutils.NewString(projectId), "owner_tenant_id") - } - if len(o.Hypervisor) > 0 { - data.Add(jsonutils.NewString(o.Hypervisor), "hypervisor") - if o.Hypervisor == "baremetal" { - for i, c := range o.BaremetalDiskConfig { - data.Add(jsonutils.NewString(c), fmt.Sprintf("baremetal_disk_config.%d", i)) - } - } + data.Ncpu = o.Ncpu } return data, nil } -func (o SchedulerTestBaseOptions) options() *jsonutils.JSONDict { - opt := jsonutils.NewDict() - opt.Add(jsonutils.NewInt(o.Count), "count") - if o.Log { - opt.Add(jsonutils.JSONTrue, "record_to_history") - } else { - opt.Add(jsonutils.JSONFalse, "record_to_history") - } +func (o SchedulerTestBaseOptions) options() *scheduler.ScheduleBaseConfig { + opt := new(scheduler.ScheduleBaseConfig) + opt.RecordLog = o.Log return opt } @@ -95,38 +44,34 @@ type SchedulerTestOptions struct { Details bool `help:"Show suggestion details"` } -func (o SchedulerTestOptions) Params(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { +func (o *SchedulerTestOptions) Params(s *mcclient.ClientSession) (*scheduler.ScheduleInput, error) { data, err := o.data(s) if err != nil { - return data, err - } - params := o.options() - params.Add(jsonutils.NewInt(o.SuggestionLimit), "suggestion_limit") - if o.SuggestionAll { - params.Add(jsonutils.JSONTrue, "suggestion_all") - } else { - params.Add(jsonutils.JSONFalse, "suggestion_all") + return nil, err } + opts := o.options() + input := new(scheduler.ScheduleInput) + input.ServerConfig = *data + input.ScheduleBaseConfig = *opts + input.SuggestionLimit = o.SuggestionLimit + input.SuggestionAll = o.SuggestionAll + input.Details = o.Details - if o.Details { - params.Add(jsonutils.JSONTrue, "suggestion_details") - } else { - params.Add(jsonutils.JSONFalse, "suggestion_details") - } - params.Add(data, "scheduler") - return params, nil + return input, nil } type SchedulerForecastOptions struct { SchedulerTestBaseOptions } -func (o SchedulerForecastOptions) Params(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { +func (o SchedulerForecastOptions) Params(s *mcclient.ClientSession) (*scheduler.ScheduleInput, error) { data, err := o.data(s) if err != nil { - return data, err + return nil, err } - params := o.options() - params.Add(data, "scheduler") - return params, nil + opts := o.options() + input := new(scheduler.ScheduleInput) + input.ServerConfig = *data + input.ScheduleBaseConfig = *opts + return input, nil } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 179304db20..0c138cf9d7 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -3,9 +3,16 @@ package options import ( "fmt" "io/ioutil" + "strconv" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/regutils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" ) type ServerListOptions struct { @@ -61,14 +68,8 @@ type ServerShowOptions struct { WithMeta *bool `help:"With meta data"` } -type ServerDeployInfo struct { - Action string - Path string - Content string -} - -func ParseServerDeployInfo(info string) (*ServerDeployInfo, error) { - sdi := &ServerDeployInfo{} +func ParseServerDeployInfo(info string) (*computeapi.DeployConfig, error) { + sdi := &computeapi.DeployConfig{} colon := strings.IndexByte(info, ':') if colon <= 0 { return nil, fmt.Errorf("malformed deploy info: %s", info) @@ -90,33 +91,117 @@ func ParseServerDeployInfo(info string) (*ServerDeployInfo, error) { return sdi, nil } -func ParseServerDeployInfoList(list []string) (*jsonutils.JSONDict, error) { - params := jsonutils.NewDict() - for i, info := range list { - ret, err := ParseServerDeployInfo(info) +func ParseServerDeployInfoList(list []string) ([]*computeapi.DeployConfig, error) { + ret := make([]*computeapi.DeployConfig, 0) + for _, info := range list { + deployInfo, err := ParseServerDeployInfo(info) if err != nil { return nil, err } - params.Set(fmt.Sprintf("deploy.%d.action", i), jsonutils.NewString(ret.Action)) - params.Set(fmt.Sprintf("deploy.%d.path", i), jsonutils.NewString(ret.Path)) - params.Set(fmt.Sprintf("deploy.%d.content", i), jsonutils.NewString(ret.Content)) + ret = append(ret, deployInfo) } - return params, nil + return ret, nil +} + +type ServerConfigs struct { + Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` + Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` + Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` + Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` + BackupHost string `help:"Perfered host where virtual backup server should be created"` + + Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei"` + ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` + Backup bool `help:"Create server with backup server"` + + Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` + Disk []string `help:"Disk descriptions" nargs:"+"` + DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` + Net []string `help:"Network descriptions" metavar:"NETWORK"` + IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` + RaidConfig []string `help:"Baremetal raid config" json:"-"` + Project string `help:"'Owner project ID or Name" json:"tenant"` + User string `help:"Owner user ID or Name"` + Count int `help:"Create multiple simultaneously" default:"1"` +} + +func (o ServerConfigs) Data() (*computeapi.ServerConfigs, error) { + data := &computeapi.ServerConfigs{ + PreferRegion: o.Region, + PreferZone: o.Zone, + PreferWire: o.Wire, + PreferHost: o.Host, + PreferBackupHost: o.BackupHost, + Hypervisor: o.Hypervisor, + ResourceType: o.ResourceType, + Project: o.Project, + Backup: o.Backup, + Count: o.Count, + } + for i, d := range o.Disk { + disk, err := cmdline.ParseDiskConfig(d, i) + if err != nil { + return nil, err + } + data.Disks = append(data.Disks, disk) + } + for _, dtag := range o.DiskSchedtag { + idx, tag, err := cmdline.ParseResourceSchedtagConfig(dtag) + if err != nil { + return nil, fmt.Errorf("ParseDiskSchedtag: %v", err) + } + if idx >= len(data.Disks) { + return nil, fmt.Errorf("Invalid disk index: %d", idx) + } + d := data.Disks[idx] + d.Schedtags = append(d.Schedtags, tag) + } + for i, n := range o.Net { + net, err := cmdline.ParseNetworkConfig(n, i) + if err != nil { + return nil, err + } + data.Networks = append(data.Networks, net) + } + for i, g := range o.IsolatedDevice { + dev, err := cmdline.ParseIsolatedDevice(g, i) + if err != nil { + return nil, err + } + data.IsolatedDevices = append(data.IsolatedDevices, dev) + } + if len(o.RaidConfig) > 0 { + if data.Hypervisor != "baremetal" { + return nil, fmt.Errorf("RaidConfig is applicable to baremetal ONLY") + } + for _, conf := range o.RaidConfig { + raidConf, err := cmdline.ParseBaremetalDiskConfig(conf) + if err != nil { + return nil, err + } + data.BaremetalDiskConfigs = append(data.BaremetalDiskConfigs, raidConf) + } + } + for _, tag := range o.Schedtag { + schedtag, err := cmdline.ParseSchedtagConfig(tag) + if err != nil { + return nil, err + } + data.Schedtags = append(data.Schedtags, schedtag) + } + return data, nil } type ServerCreateOptions struct { - ScheduleOptions + ServerConfigs NAME string `help:"Name of server" json:"-"` MEMSPEC string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"` - Disk []string `help:"Disk descriptions" nargs:"+"` - Net []string `help:"Network descriptions" metavar:"NETWORK"` - IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` Keypair string `help:"SSH Keypair"` Password string `help:"Default user password"` Iso string `help:"ISO image ID" metavar:"IMAGE_ID" json:"cdrom"` - VcpuCount *int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` + VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl"` Vdi string `help:"VDI protocool" choices:"vnc|spice"` Bios string `help:"BIOS" choices:"BIOS|UEFI"` @@ -125,18 +210,12 @@ type ServerCreateOptions struct { NoAccountInit *bool `help:"Not reset account password"` AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` ShutdownBehavior string `help:"Behavior after VM server shutdown, stop or terminate server" metavar:"" choices:"stop|terminate"` - AutoStart *bool `help:"Auto start server after it is created"` - Backup *bool `help:"Create server with backup server" json:"backup"` - BackupHost string `help:"Perfered host where virtual backup server should be created" json:"prefer_backup_host"` + AutoStart bool `help:"Auto start server after it is created"` Deploy []string `help:"Specify deploy files in virtual server file system" json:"-"` Group []string `help:"Group of virtual server"` - Project string `help:"'Owner project ID or Name" json:"tenant"` - User string `help:"Owner user ID or Name"` - System *bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` + System bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` TaskNotify *bool `help:"Setup task notify" json:"-"` - Count *int `help:"Create multiple simultaneously" default:"1" json:"-"` DryRun *bool `help:"Dry run to test scheduler" json:"-"` - RaidConfig []string `help:"Baremetal raid config" json:"-"` UserDataFile string `help:"user_data file path" json:"-"` Duration string `help:"valid duration of the server, e.g. 1H, 1D, 1W, 1M, 1Y, ADMIN ONLY option"` @@ -150,50 +229,130 @@ type ServerCreateOptions struct { Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` } -func (opts *ServerCreateOptions) Params() (*jsonutils.JSONDict, error) { - schedParams, err := opts.ScheduleOptions.Params() +func (o *ServerCreateOptions) ToScheduleInput() (*schedapi.ScheduleInput, error) { + data := new(schedapi.ServerConfig) + + // only support digit number as for now + memSize, err := strconv.Atoi(o.MEMSPEC) if err != nil { return nil, err } - - params, err := optionsStructToParams(opts) - if err != nil { - return nil, err + data.Memory = memSize + if o.VcpuCount > 0 { + data.Ncpu = o.VcpuCount } - params.Update(schedParams) - - { - deployParams, err := ParseServerDeployInfoList(opts.Deploy) + for i, d := range o.Disk { + disk, err := cmdline.ParseDiskConfig(d, i) if err != nil { return nil, err } - params.Update(deployParams) + data.Disks = append(data.Disks, disk) } + for i, n := range o.Net { + net, err := cmdline.ParseNetworkConfig(n, i) + if err != nil { + return nil, err + } + data.Networks = append(data.Networks, net) + } + for i, g := range o.IsolatedDevice { + dev, err := cmdline.ParseIsolatedDevice(g, i) + if err != nil { + return nil, err + } + data.IsolatedDevices = append(data.IsolatedDevices, dev) + } + count := 1 + if o.Count > 1 { + count = o.Count + } + input := new(schedapi.ScheduleInput) + input.Count = count + input.ServerConfig = *data + if o.DryRun != nil && *o.DryRun { + input.Details = true + } + return input, nil +} + +func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error) { + config, err := opts.ServerConfigs.Data() + if err != nil { + return nil, err + } + + params := &computeapi.ServerCreateInput{ + ServerConfigs: config, + VcpuCount: opts.VcpuCount, + Keypair: opts.Keypair, + Password: opts.Password, + Cdrom: opts.Iso, + Vga: opts.Vga, + Vdi: opts.Vdi, + Bios: opts.Bios, + Description: opts.Desc, + ShutdownBehavior: opts.ShutdownBehavior, + AutoStart: opts.AutoStart, + IsSystem: opts.System, + Duration: opts.Duration, + AutoPrepaidRecycle: opts.AutoPrepaidRecycle, + EipBw: opts.EipBw, + EipChargeType: opts.EipChargeType, + Eip: opts.Eip, + } + + if opts.GenerateName { + params.GenerateName = opts.NAME + } else { + params.Name = opts.NAME + } + if regutils.MatchSize(opts.MEMSPEC) { + memSize, err := fileutils.GetSizeMb(opts.MEMSPEC, 'M', 1024) + if err != nil { + return nil, err + } + params.VmemSize = memSize + } else { + params.InstanceType = opts.MEMSPEC + } + + deployInfos, err := ParseServerDeployInfoList(opts.Deploy) + if err != nil { + return nil, err + } + params.DeployConfigs = deployInfos if len(opts.Boot) > 0 { if opts.Boot == "disk" { - params.Set("boot_order", jsonutils.NewString("cdn")) + params.BootOrder = "cdn" } else { - params.Set("boot_order", jsonutils.NewString("dcn")) + params.BootOrder = "dcn" } } + resetPasswd := false + if opts.NoAccountInit != nil && *opts.NoAccountInit { + params.ResetPassword = &resetPasswd + } else { + params.ResetPassword = nil + } + + if len(opts.UserDataFile) > 0 { + userdata, err := ioutil.ReadFile(opts.UserDataFile) + if err != nil { + return nil, err + } + params.UserData = string(userdata) + } + if BoolV(opts.AllowDelete) { - params.Set("disable_delete", jsonutils.JSONFalse) - } - - if len(opts.RaidConfig) > 0 { - if opts.Hypervisor != "baremetal" { - return nil, fmt.Errorf("RaidConfig is applicable to baremetal ONLY") - } - for i, conf := range opts.RaidConfig { - params.Set(fmt.Sprintf("baremetal_disk_config.%d", i), jsonutils.NewString(conf)) - } + params.DisableDelete = false } if BoolV(opts.DryRun) { - params.Set("suggestion", jsonutils.JSONTrue) + params.Suggestion = true } + return params, nil } @@ -254,25 +413,22 @@ type ServerDeployOptions struct { AutoStart *bool `help:"Auto start server after deployed"` } -func (opts *ServerDeployOptions) Params() (*jsonutils.JSONDict, error) { - params, err := optionsStructToParams(opts) - if err != nil { - return nil, err - } +func (opts *ServerDeployOptions) Params() (*computeapi.ServerDeployInput, error) { + params := new(computeapi.ServerDeployInput) { deleteKeyPair := BoolV(opts.DeleteKeypair) if deleteKeyPair { - params.Add(jsonutils.JSONTrue, "__delete_keypair__") + params.DeleteKeypair = true } else if len(opts.Keypair) > 0 { - params.Add(jsonutils.NewString(opts.Keypair), "keypair") + params.Keypair = opts.Keypair } } { - deployParams, err := ParseServerDeployInfoList(opts.Deploy) + deployInfos, err := ParseServerDeployInfoList(opts.Deploy) if err != nil { return nil, err } - params.Update(deployParams) + params.DeployConfigs = deployInfos } return params, nil } diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go index bcd477b839..6c4b796fd0 100644 --- a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -63,7 +63,7 @@ func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []c type schedtagCandidateW struct { core.Candidater - schedData *api.SchedData + schedData *api.SchedInfo } func (w schedtagCandidateW) GetDynamicSchedDesc() *jsonutils.JSONDict { @@ -96,25 +96,27 @@ func (p *AggregatePredicate) exec(h *PredicateHelper) string { return "" } +func SetCandidateScoreBySchedtag(u *core.Unit, c core.Candidater, aggCountMap map[string]int, postiveScore bool) { + stepScore := core.PriorityStep + if !postiveScore { + stepScore = -stepScore + } + for n, count := range aggCountMap { + u.SetFrontScore( + c.IndexKey(), + score.NewScore(score.TScore(count*stepScore), n), + ) + } +} + func (p *AggregatePredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { hostAggs := c.Getter().HostSchedtags() avoidCountMap := GetSchedtagCount(p.SchedtagPredicate.GetAvoidTags(), hostAggs, api.AggregateStrategyAvoid) preferCountMap := GetSchedtagCount(p.SchedtagPredicate.GetPreferTags(), hostAggs, api.AggregateStrategyPrefer) - setScore := func(aggCountMap map[string]int, postiveScore bool) { - stepScore := core.PriorityStep - if !postiveScore { - stepScore = -stepScore - } - for n, count := range aggCountMap { - u.SetFrontScore( - c.IndexKey(), - score.NewScore(score.TScore(count*stepScore), n), - ) - } - } + setScore := SetCandidateScoreBySchedtag - setScore(preferCountMap, true) - setScore(avoidCountMap, false) + setScore(u, c, preferCountMap, true) + setScore(u, c, avoidCountMap, false) } diff --git a/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go index f1346eefd6..493b3cf45c 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go @@ -22,8 +22,8 @@ func (p *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.Pr d := u.SchedData() freeCPUCount := h.GetInt64("FreeCPUCount", 0) - reqCPUCount := d.VCPUCount - if freeCPUCount < d.VCPUCount { + reqCPUCount := int64(d.Ncpu) + if freeCPUCount < reqCPUCount { totalCPUCount := h.GetInt64("CPUCount", 0) h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) h.SetCapacity(0) diff --git a/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go index feb9ca7d5d..e9696e0f5c 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go @@ -22,7 +22,7 @@ func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core d := u.SchedData() freeMemSize := h.GetInt64("FreeMemSize", 0) - reqMemSize := d.VMEMSize + reqMemSize := int64(d.Memory) if freeMemSize < reqMemSize { totalMemSize := h.GetInt64("MemSize", 0) h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) diff --git a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go index 62191c39ae..8aec50c6a5 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go @@ -7,9 +7,9 @@ import ( "yunion.io/x/pkg/utils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -35,7 +35,7 @@ func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, u.AppendSelectPlugin(p) d := u.SchedData() - if len(d.HostID) > 0 && len(d.Networks) == 0 { + if len(d.HostId) > 0 && len(d.Networks) == 0 { return false, nil } @@ -54,7 +54,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor counters := core.NewCounters() isMigrate := func() bool { - return len(schedData.HostID) > 0 + return len(schedData.HostId) > 0 } isRandomNetworkAvailable := func(private bool, exit bool, wire string) string { @@ -69,7 +69,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor if wire != "" && !utils.HasPrefix(wire, network.WireId) && !utils.HasPrefix(wire, network.GetWire().GetName()) { // re appendError(predicates.ErrWireIsNotMatch) } - if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.OwnerTenantID) { + if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.Project) { // TODO: support reservedNetworks reservedNetworks := 0 restPort := int64(network.GetPorts() - reservedNetworks) @@ -100,12 +100,12 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor h.SetCapacityCounter(counters) } - isNetworkAvaliable := func(network *api.Network) string { - if network.Idx == "" { + isNetworkAvaliable := func(network *computeapi.NetworkConfig) string { + if network.Network == "" { return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire) } for _, net := range candidate.Networks { - if (network.Idx == net.Id || network.Idx == net.Name) && (net.IsPublic || net.ProjectId == schedData.OwnerTenantID) && (net.GetPorts() > 0 || isMigrate()) { + if (network.Network == net.Id || network.Network == net.Name) && (net.IsPublic || net.ProjectId == schedData.Project) && (net.GetPorts() > 0 || isMigrate()) { h.SetCapacity(1) return "" } diff --git a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go index 2abf7694c4..8e170b6c3b 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go @@ -3,9 +3,9 @@ package baremetal import ( "fmt" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -21,22 +21,19 @@ func (p *StoragePredicate) Clone() core.FitPredicate { return &StoragePredicate{} } -func toBaremetalDisks(disks []*api.Disk) []*baremetal.Disk { +func toBaremetalDisks(disks []*computeapi.DiskConfig) []*baremetal.Disk { ret := make([]*baremetal.Disk, len(disks)) for i, disk := range disks { ret[i] = &baremetal.Disk{ - Backend: disk.Backend, - ImageID: disk.ImageID, - Fs: disk.Fs, - Os: disk.Os, - OSDistribution: disk.OSDistribution, - Format: disk.Format, - MountPoint: disk.MountPoint, - Driver: disk.Driver, - Cache: disk.Cache, - ImageDiskFormat: disk.ImageDiskFormat, - Size: disk.Size, - Storage: disk.Storage, + Backend: disk.Backend, + ImageID: disk.ImageId, + Fs: &disk.Fs, + Format: disk.Format, + MountPoint: &disk.Mountpoint, + Driver: &disk.Driver, + Cache: &disk.Cache, + Size: int64(disk.SizeMb), + Storage: &disk.Storage, } } return ret diff --git a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go index e6ba918a9e..ae8c26f893 100644 --- a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go +++ b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go @@ -6,17 +6,70 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/util/errors" + "yunion.io/x/pkg/utils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) -type DiskStoragesMap map[int][]*api.CandidateStorage +type DiskStoragesMap map[int][]*PredicatedStorage + +func (m DiskStoragesMap) getAllTags(isPrefer bool) []computeapi.SchedtagConfig { + ret := make([]computeapi.SchedtagConfig, 0) + for _, ss := range m { + for _, s := range ss { + var tags []computeapi.SchedtagConfig + if isPrefer { + tags = s.PreferTags + } else { + tags = s.AvoidTags + } + ret = append(ret, tags...) + } + } + return ret +} + +func (m DiskStoragesMap) GetPreferTags() []computeapi.SchedtagConfig { + return m.getAllTags(true) +} + +func (m DiskStoragesMap) GetAvoidTags() []computeapi.SchedtagConfig { + return m.getAllTags(false) +} type CandidateDiskStoragesMap map[string]DiskStoragesMap +type PredicatedStorage struct { + *api.CandidateStorage + PreferTags []computeapi.SchedtagConfig + AvoidTags []computeapi.SchedtagConfig +} + +func newPredicatedStorage(s *api.CandidateStorage, preferTags, avoidTags []computeapi.SchedtagConfig) *PredicatedStorage { + return &PredicatedStorage{ + CandidateStorage: s, + PreferTags: preferTags, + AvoidTags: avoidTags, + } +} + +func (s *PredicatedStorage) isNoTag() bool { + return len(s.PreferTags) == 0 && len(s.AvoidTags) == 0 +} + +func (s *PredicatedStorage) hasPreferTags() bool { + return len(s.PreferTags) != 0 +} + +func (s *PredicatedStorage) hasAvoidTags() bool { + return len(s.AvoidTags) != 0 +} + type DiskSchedtagPredicate struct { BasePredicate plugin.BasePlugin @@ -24,6 +77,8 @@ type DiskSchedtagPredicate struct { SchedtagPredicate *SchedtagPredicate CandidateDiskStoragesMap CandidateDiskStoragesMap + + Hypervisor string } func (p *DiskSchedtagPredicate) Name() string { @@ -36,9 +91,9 @@ func (p *DiskSchedtagPredicate) Clone() core.FitPredicate { } } -func (p *DiskSchedtagPredicate) getSchedtagDisks(disks []*api.Disk) ([]*api.Disk, []*api.Disk) { - noTagDisk := make([]*api.Disk, 0) - tagDisk := make([]*api.Disk, 0) +func (p *DiskSchedtagPredicate) getSchedtagDisks(disks []*computeapi.DiskConfig) ([]*computeapi.DiskConfig, []*computeapi.DiskConfig) { + noTagDisk := make([]*computeapi.DiskConfig, 0) + tagDisk := make([]*computeapi.DiskConfig, 0) for _, d := range disks { if len(d.Schedtags) != 0 { tagDisk = append(tagDisk, d) @@ -55,6 +110,8 @@ func (p *DiskSchedtagPredicate) PreExecute(u *core.Unit, cs []core.Candidater) ( return false, nil } + p.Hypervisor = u.SchedData().Hypervisor + // always select each storages to disks u.AppendSelectPlugin(p) @@ -63,11 +120,11 @@ func (p *DiskSchedtagPredicate) PreExecute(u *core.Unit, cs []core.Candidater) ( type schedtagStorageW struct { candidater *api.CandidateStorage - disk *api.Disk + disk *computeapi.DiskConfig } func (w schedtagStorageW) IndexKey() string { - return fmt.Sprintf("%d:%s", w.disk.Size, w.disk.Backend) + return fmt.Sprintf("%s:%s", w.candidater.GetName(), w.candidater.StorageType) } func (w schedtagStorageW) GetDynamicSchedDesc() *jsonutils.JSONDict { @@ -82,10 +139,10 @@ func (w schedtagStorageW) ResourceType() string { return models.StorageManager.KeywordPlural() } -func (p *DiskSchedtagPredicate) check(d *api.Disk, s *api.CandidateStorage) (bool, error) { +func (p *DiskSchedtagPredicate) check(d *computeapi.DiskConfig, s *api.CandidateStorage) (*PredicatedStorage, error) { allTags, err := GetAllSchedtags(models.StorageManager.KeywordPlural()) if err != nil { - return false, err + return nil, err } tagPredicate := NewSchedtagPredicate(d.Schedtags, allTags) if err := tagPredicate.Check( @@ -94,22 +151,24 @@ func (p *DiskSchedtagPredicate) check(d *api.Disk, s *api.CandidateStorage) (boo disk: d, }, ); err != nil { - return false, err + return nil, err } - return true, nil + avoidTags := tagPredicate.GetAvoidTags() + preferTags := tagPredicate.GetPreferTags() + return newPredicatedStorage(s, preferTags, avoidTags), nil } -func (p *DiskSchedtagPredicate) checkStorages(d *api.Disk, storages []*api.CandidateStorage) ([]*api.CandidateStorage, error) { +func (p *DiskSchedtagPredicate) checkStorages(d *computeapi.DiskConfig, storages []*api.CandidateStorage) ([]*PredicatedStorage, error) { errs := make([]error, 0) - ret := make([]*api.CandidateStorage, 0) + ret := make([]*PredicatedStorage, 0) for _, s := range storages { - _, err := p.check(d, s) + ps, err := p.check(d, s) if err != nil { // append err, storage not suit disk errs = append(errs, err) continue } - ret = append(ret, s) + ret = append(ret, ps) } if len(ret) == 0 { return nil, errors.NewAggregate(errs) @@ -120,7 +179,7 @@ func (p *DiskSchedtagPredicate) checkStorages(d *api.Disk, storages []*api.Candi func (p *DiskSchedtagPredicate) GetDiskStoragesMap(candidateId string) DiskStoragesMap { ret, ok := p.CandidateDiskStoragesMap[candidateId] if !ok { - ret = make(map[int][]*api.CandidateStorage) + ret = make(map[int][]*PredicatedStorage) p.CandidateDiskStoragesMap[candidateId] = ret } return ret @@ -129,40 +188,132 @@ func (p *DiskSchedtagPredicate) GetDiskStoragesMap(candidateId string) DiskStora func (p *DiskSchedtagPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(p, u, c) - //noTagDisks, tagDisks := p.getSchedtagDisks(u.SchedData().Disks) storages := c.Getter().Storages() ds := p.GetDiskStoragesMap(c.IndexKey()) disks := u.SchedData().Disks - for _, d := range disks { + for idx, d := range disks { matchedStorages, err := p.checkStorages(d, storages) if err != nil { h.Exclude(err.Error()) } - ds[d.Index] = matchedStorages + ds[idx] = matchedStorages } return h.GetResult() } +func (p *DiskSchedtagPredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { + storageTags := []models.SSchedtag{} + for _, s := range c.Getter().Storages() { + storageTags = append(storageTags, s.Schedtags...) + } + + ds := p.GetDiskStoragesMap(c.IndexKey()) + avoidTags := ds.GetAvoidTags() + preferTags := ds.GetPreferTags() + + avoidCountMap := GetSchedtagCount(avoidTags, storageTags, api.AggregateStrategyAvoid) + preferCountMap := GetSchedtagCount(preferTags, storageTags, api.AggregateStrategyPrefer) + + setScore := SetCandidateScoreBySchedtag + + setScore(u, c, preferCountMap, true) + setScore(u, c, avoidCountMap, false) +} + func (p *DiskSchedtagPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { res := u.GetAllocatedResource(c.IndexKey()) diskStorages := p.GetDiskStoragesMap(c.IndexKey()) - res.Disks = make([]*core.DiskAllocatedResource, len(diskStorages)) + res.Disks = make([]*schedapi.CandidateDisk, len(diskStorages)) disks := u.SchedData().Disks for idx, ds := range diskStorages { res.Disks[idx] = p.allocatedDiskResource(c, disks[idx], ds) } - log.Errorf("============OnSelectEnd %s called: %#v", c.Getter().Name(), jsonutils.Marshal(res.Disks).String()) } -func (p *DiskSchedtagPredicate) allocatedDiskResource(c core.Candidater, disk *api.Disk, storages []*api.CandidateStorage) *core.DiskAllocatedResource { +func (p *DiskSchedtagPredicate) allocatedDiskResource(c core.Candidater, disk *computeapi.DiskConfig, storages []*PredicatedStorage) *schedapi.CandidateDisk { storage := p.selectStorage(disk, storages) - return &core.DiskAllocatedResource{ + log.Debugf("Select storage %s:%s for disk: %s", storage.Id, storage.Name, disk.Index) + return &schedapi.CandidateDisk{ Index: disk.Index, StorageId: storage.Id, } } -func (p *DiskSchedtagPredicate) selectStorage(d *api.Disk, storages []*api.CandidateStorage) *api.CandidateStorage { - return storages[0] +func (p *DiskSchedtagPredicate) selectStorage(d *computeapi.DiskConfig, storages []*PredicatedStorage) *api.CandidateStorage { + preferStorages := []*api.CandidateStorage{} + noTagStorages := []*api.CandidateStorage{} + avoidStorages := []*api.CandidateStorage{} + for _, storage := range storages { + if p.isStorageFitDisk(storage, d) { + candi := storage.CandidateStorage + if storage.isNoTag() { + noTagStorages = append(noTagStorages, candi) + } else if storage.hasPreferTags() { + preferStorages = append(preferStorages, candi) + } else if storage.hasAvoidTags() { + avoidStorages = append(avoidStorages, candi) + } + } + } + sortStorages := []*api.CandidateStorage{} + sortStorages = append(sortStorages, preferStorages...) + sortStorages = append(sortStorages, noTagStorages...) + sortStorages = append(sortStorages, avoidStorages...) + return p.GetLeastUsedStorage(sortStorages, d.Backend) +} + +func (p *DiskSchedtagPredicate) GetLeastUsedStorage(storages []*api.CandidateStorage, backend string) *api.CandidateStorage { + var backends []string + if backend == computeapi.STORAGE_LOCAL { + backends = []string{computeapi.STORAGE_NAS, computeapi.STORAGE_LOCAL} + } else if len(backend) > 0 { + backends = []string{backend} + } else { + backends = []string{} + } + return p.getLeastUsedStorage(storages, backends) +} + +func (p *DiskSchedtagPredicate) getLeastUsedStorage(storages []*api.CandidateStorage, backends []string) *api.CandidateStorage { + var best *api.CandidateStorage + var bestCap int + for i := 0; i < len(storages); i++ { + s := storages[i] + if len(backends) > 0 { + in, _ := utils.InStringArray(s.StorageType, backends) + if !in { + continue + } + } + capa := s.GetFreeCapacity() + if best == nil || bestCap < capa { + bestCap = capa + best = s + } + } + return best +} + +func (p *DiskSchedtagPredicate) GetHypervisorDriver() models.IGuestDriver { + return models.GetDriver(p.Hypervisor) +} + +func (p *DiskSchedtagPredicate) isStorageFitDisk(storage *PredicatedStorage, d *computeapi.DiskConfig) bool { + if d.Storage != "" { + if storage.Id == d.Storage || storage.Name == d.Storage { + return true + } + return false + } + if storage.StorageType == d.Backend { + return true + } + + for _, stype := range p.GetHypervisorDriver().GetStorageTypes() { + if storage.StorageType == stype { + return true + } + } + return false } diff --git a/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go index f11296603c..6c544ebaa2 100644 --- a/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go @@ -26,7 +26,7 @@ func (f *CPUPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, err data := u.SchedData() - if data.VCPUCount <= 0 { + if data.Ncpu <= 0 { return false, nil } @@ -43,7 +43,7 @@ func (f *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.Pr useRsvd := h.UseReserved() freeCPUCount := hc.GetFreeCPUCount(useRsvd) - reqCPUCount := d.VCPUCount + reqCPUCount := int64(d.Ncpu) if freeCPUCount < reqCPUCount { totalCPUCount := hc.GetTotalCPUCount(useRsvd) h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) diff --git a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go index 9c9610c73c..02420ed12f 100644 --- a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go @@ -37,13 +37,13 @@ func (p *GroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, e for _, r := range d.GroupRelations { if r.Strategy == "exclude" { - p.ExcludeGroups = append(p.ExcludeGroups, r.GroupID) + p.ExcludeGroups = append(p.ExcludeGroups, r.GroupId) } else if r.Strategy == "require" { - p.RequireGroups = append(p.RequireGroups, r.GroupID) + p.RequireGroups = append(p.RequireGroups, r.GroupId) } else if r.Strategy == "avoid" { - p.AvoidGroups = append(p.AvoidGroups, r.GroupID) + p.AvoidGroups = append(p.AvoidGroups, r.GroupId) } else if r.Strategy == "prefer" { - p.PreferGroups = append(p.PreferGroups, r.GroupID) + p.PreferGroups = append(p.PreferGroups, r.GroupId) } } u.AppendSelectPlugin(p) diff --git a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go index 98d12fdaa2..7b1fcad6e9 100644 --- a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go @@ -41,16 +41,16 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool // check by specify device id for _, dev := range reqIsoDevs { - if len(dev.ID) == 0 { + if len(dev.Id) == 0 { continue } - if fDev := hc.GetIsolatedDevice(dev.ID); fDev != nil { + if fDev := hc.GetIsolatedDevice(dev.Id); fDev != nil { if len(fDev.GuestID) != 0 { - h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.ID, fDev.GuestID)) + h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.Id, fDev.GuestID)) return h.GetResult() } } else { - h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.ID)) + h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.Id)) return h.GetResult() } minCapacity = 1 @@ -72,8 +72,8 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool // check host device by type devTypeRequest := make(map[string]int, 0) for _, dev := range reqIsoDevs { - if len(dev.Type) != 0 { - devTypeRequest[dev.Type] += 1 + if len(dev.DevType) != 0 { + devTypeRequest[dev.DevType] += 1 } } for devType, reqCount := range devTypeRequest { diff --git a/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go index 4b505a8bb4..eb51cb41a9 100644 --- a/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go @@ -27,7 +27,7 @@ func (p *MemoryPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, data := u.SchedData() - if data.VMEMSize <= 0 { + if data.Memory <= 0 { return false, nil } @@ -44,7 +44,7 @@ func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core useRsvd := h.UseReserved() freeMemSize := hc.GetFreeMemSize(useRsvd) - reqMemSize := d.VMEMSize + reqMemSize := int64(d.Memory) if freeMemSize < reqMemSize { totalMemSize := hc.GetTotalMemSize(useRsvd) h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) diff --git a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go index 10c53736ce..ed50fe59c8 100644 --- a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go @@ -19,13 +19,13 @@ func (p *MigratePredicate) Clone() core.FitPredicate { } func (p *MigratePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { - return len(u.SchedData().HostID) > 0, nil + return len(u.SchedData().HostId) > 0, nil } func (p *MigratePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := predicates.NewPredicateHelper(p, u, c) - if u.SchedData().HostID == c.IndexKey() { + if u.SchedData().HostId == c.IndexKey() { h.Exclude(predicates.ErrHostIsSpecifiedForMigration) } diff --git a/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go index ec75e9a2de..69d8a41ddb 100644 --- a/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go @@ -30,7 +30,7 @@ func (p *NestPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.P d := u.SchedData() - if d.Meta["kvm"] == "enabled" { + if d.Metadata["kvm"] == "enabled" { if hc.Metadata["nest"] != "enabled" { h.Exclude(predicates.ErrNotSupportNest) } diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index cd42c648f1..32e3948a28 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -10,9 +10,9 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -35,7 +35,7 @@ func (p *NetworkPredicate) Clone() core.FitPredicate { func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { data := u.SchedData() - if len(data.HostID) > 0 && len(data.Networks) == 0 { + if len(data.HostId) > 0 && len(data.Networks) == 0 { return false, nil } @@ -53,7 +53,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor d := u.SchedData() isMigrate := func() bool { - return len(d.HostID) > 0 + return len(d.HostId) > 0 } // ServerType's value is 'guest', 'container' or ''(support all type) will return true. @@ -98,7 +98,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor appendError(predicates.ErrWireIsNotMatch) } - if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.OwnerTenantID)) { + if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.Project)) { appendError(predicates.ErrNotOwner) } @@ -131,9 +131,9 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor h.SetCapacityCounter(counters) } - isNetworkAvaliable := func(n *api.Network, counters *core.MinCounters, + isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters, networks []models.SNetwork) string { - if n.Idx == "" { + if n.Network == "" { counters0 := core.NewCounters() ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0) counters.Add(counters0) @@ -150,17 +150,17 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID)) continue }*/ - if !(n.Idx == net.GetId() || n.Idx == net.GetName()) { + if !(n.Network == net.GetId() || n.Network == net.GetName()) { errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id)) - } else if !(net.IsPublic || net.ProjectId == d.OwnerTenantID) { - errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.OwnerTenantID)) + } else if !(net.IsPublic || net.ProjectId == d.Project) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.Project)) } else if !(net.GetPorts() > 0 || isMigrate()) { errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id)) } else { // add resource reservedNetworks := 0 counter := counterOfNetwork(u, &net, reservedNetworks) - if counter.GetCount() < d.Count { + if counter.GetCount() < int64(d.Count) { errMsgs = append(errMsgs, fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count)) continue } diff --git a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go index 30a84ea995..0566a41854 100644 --- a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go @@ -44,7 +44,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor d := u.SchedData() isMigrate := func() bool { - return len(d.HostID) > 0 + return len(d.HostId) > 0 } isLocalhostBackend := func(backend string) bool { @@ -72,7 +72,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor ss := make([]string, 0, len(d.Disks)) for _, disk := range d.Disks { if disk.Backend == backend { - ss = append(ss, fmt.Sprintf("%v", disk.Size)) + ss = append(ss, fmt.Sprintf("%v", disk.SizeMb)) } } @@ -97,16 +97,16 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor storeRequest := make(map[string]int64, 0) for _, disk := range d.Disks { if isMigrate() && !isLocalhostBackend(disk.Backend) { - storeRequest[*disk.Storage] = 1 + storeRequest[disk.Storage] = 1 } else { if _, ok := sizeRequest[disk.Backend]; !ok { sizeRequest[disk.Backend] = map[string]int64{"max": -1, "total": 0} } max := sizeRequest[disk.Backend]["max"] - if max < disk.Size { - sizeRequest[disk.Backend]["max"] = disk.Size + if max < int64(disk.SizeMb) { + sizeRequest[disk.Backend]["max"] = int64(disk.SizeMb) } - sizeRequest[disk.Backend]["total"] += disk.Size + sizeRequest[disk.Backend]["total"] += int64(disk.SizeMb) } } diff --git a/pkg/scheduler/algorithm/predicates/schedtag_helper.go b/pkg/scheduler/algorithm/predicates/schedtag_helper.go index 868677205d..c394914073 100644 --- a/pkg/scheduler/algorithm/predicates/schedtag_helper.go +++ b/pkg/scheduler/algorithm/predicates/schedtag_helper.go @@ -6,16 +6,16 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/util/conditionparser" ) type ISchedtagPredicate interface { - GetExcludeTags() []api.Schedtag - GetRequireTags() []api.Schedtag - GetAvoidTags() []api.Schedtag - GetPreferTags() []api.Schedtag + GetExcludeTags() []computeapi.SchedtagConfig + GetRequireTags() []computeapi.SchedtagConfig + GetAvoidTags() []computeapi.SchedtagConfig + GetPreferTags() []computeapi.SchedtagConfig } type ISchedtagCandidate interface { @@ -28,14 +28,14 @@ type ISchedtagCandidate interface { } type SchedtagPredicate struct { - requireTags []api.Schedtag - execludeTags []api.Schedtag - preferTags []api.Schedtag - avoidTags []api.Schedtag + requireTags []computeapi.SchedtagConfig + execludeTags []computeapi.SchedtagConfig + preferTags []computeapi.SchedtagConfig + avoidTags []computeapi.SchedtagConfig checker *SchedtagChecker } -func NewSchedtagPredicate(reqTags []api.Schedtag, allTags []models.SSchedtag) *SchedtagPredicate { +func NewSchedtagPredicate(reqTags []*computeapi.SchedtagConfig, allTags []models.SSchedtag) *SchedtagPredicate { p := new(SchedtagPredicate) requireTags, execludeTags, preferTags, avoidTags := GetRequestSchedtags(reqTags, allTags) p.requireTags = requireTags @@ -46,19 +46,19 @@ func NewSchedtagPredicate(reqTags []api.Schedtag, allTags []models.SSchedtag) *S return p } -func (p *SchedtagPredicate) GetExcludeTags() []api.Schedtag { +func (p *SchedtagPredicate) GetExcludeTags() []computeapi.SchedtagConfig { return p.execludeTags } -func (p *SchedtagPredicate) GetRequireTags() []api.Schedtag { +func (p *SchedtagPredicate) GetRequireTags() []computeapi.SchedtagConfig { return p.requireTags } -func (p *SchedtagPredicate) GetAvoidTags() []api.Schedtag { +func (p *SchedtagPredicate) GetAvoidTags() []computeapi.SchedtagConfig { return p.avoidTags } -func (p *SchedtagPredicate) GetPreferTags() []api.Schedtag { +func (p *SchedtagPredicate) GetPreferTags() []computeapi.SchedtagConfig { return p.preferTags } @@ -66,12 +66,12 @@ func (p *SchedtagPredicate) Check(candidate ISchedtagCandidate) error { return p.checker.Check(p, candidate) } -func GetSchedtagCount(inTags []api.Schedtag, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { +func GetSchedtagCount(inTags []computeapi.SchedtagConfig, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { countMap = make(map[string]int) - in := func(objTag models.SSchedtag, inTags []api.Schedtag) bool { + in := func(objTag models.SSchedtag, inTags []computeapi.SchedtagConfig) bool { for _, tag := range inTags { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } @@ -94,31 +94,31 @@ func GetAllSchedtags(resType string) ([]models.SSchedtag, error) { return tags, nil } -func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (requireTags, execludeTags, preferTags, avoidTags []api.Schedtag) { - requireTags = make([]api.Schedtag, 0) - execludeTags = make([]api.Schedtag, 0) - preferTags = make([]api.Schedtag, 0) - avoidTags = make([]api.Schedtag, 0) +func GetRequestSchedtags(reqTags []*computeapi.SchedtagConfig, allTags []models.SSchedtag) (requireTags, execludeTags, preferTags, avoidTags []computeapi.SchedtagConfig) { + requireTags = make([]computeapi.SchedtagConfig, 0) + execludeTags = make([]computeapi.SchedtagConfig, 0) + preferTags = make([]computeapi.SchedtagConfig, 0) + avoidTags = make([]computeapi.SchedtagConfig, 0) appendedTagIds := make(map[string]int) - appendTagByStrategy := func(tag api.Schedtag) { + appendTagByStrategy := func(tag *computeapi.SchedtagConfig) { switch tag.Strategy { case models.STRATEGY_REQUIRE: - requireTags = append(requireTags, tag) + requireTags = append(requireTags, *tag) case models.STRATEGY_EXCLUDE: - execludeTags = append(execludeTags, tag) + execludeTags = append(execludeTags, *tag) case models.STRATEGY_PREFER: - preferTags = append(preferTags, tag) + preferTags = append(preferTags, *tag) case models.STRATEGY_AVOID: - avoidTags = append(avoidTags, tag) + avoidTags = append(avoidTags, *tag) } } for _, tag := range reqTags { appendTagByStrategy(tag) - appendedTagIds[tag.Idx] = 1 + appendedTagIds[tag.Id] = 1 } for _, tag := range allTags { @@ -126,7 +126,7 @@ func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (re _, idOk := appendedTagIds[tag.Id] if !(nameOk || idOk) { - apiTag := api.Schedtag{Idx: tag.Id, Strategy: tag.DefaultStrategy} + apiTag := &computeapi.SchedtagConfig{Id: tag.Id, Strategy: tag.DefaultStrategy} appendTagByStrategy(apiTag) } } @@ -136,11 +136,11 @@ func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (re type SchedtagChecker struct{} -type apiTags []api.Schedtag +type apiTags []computeapi.SchedtagConfig func (t apiTags) contains(objTag models.SSchedtag) bool { for _, tag := range t { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } @@ -149,25 +149,25 @@ func (t apiTags) contains(objTag models.SSchedtag) bool { type objTags []models.SSchedtag -func (t objTags) contains(atag api.Schedtag) bool { +func (t objTags) contains(atag computeapi.SchedtagConfig) bool { for _, tag := range t { - if tag.Id == atag.Idx || tag.Name == atag.Idx { + if tag.Id == atag.Id || tag.Name == atag.Id { return true } } return false } -func (c *SchedtagChecker) contains(tags []api.Schedtag, objTag models.SSchedtag) bool { +func (c *SchedtagChecker) contains(tags []computeapi.SchedtagConfig, objTag models.SSchedtag) bool { for _, tag := range tags { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } return false } -func (c *SchedtagChecker) HasIntersection(tags []api.Schedtag, objTags []models.SSchedtag) (bool, *models.SSchedtag) { +func (c *SchedtagChecker) HasIntersection(tags []computeapi.SchedtagConfig, objTags []models.SSchedtag) (bool, *models.SSchedtag) { var atags apiTags = tags for _, objTag := range objTags { if atags.contains(objTag) { @@ -177,7 +177,7 @@ func (c *SchedtagChecker) HasIntersection(tags []api.Schedtag, objTags []models. return false, nil } -func (c *SchedtagChecker) Contains(objectTags []models.SSchedtag, tags []api.Schedtag) (bool, *api.Schedtag) { +func (c *SchedtagChecker) Contains(objectTags []models.SSchedtag, tags []computeapi.SchedtagConfig) (bool, *computeapi.SchedtagConfig) { var otags objTags = objectTags for _, tag := range tags { if !otags.contains(tag) { @@ -245,17 +245,21 @@ func (c *SchedtagChecker) Check(p ISchedtagPredicate, candidate ISchedtagCandida if err != nil { return err } + execludeTags := p.GetExcludeTags() + requireTags := p.GetRequireTags() + + log.V(10).Debugf("[SchedtagChecker] check candidate: %s requireTags: %#v, execludeTags: %#v, candidateTags: %#v", candidate.IndexKey(), requireTags, execludeTags, candidateTags) + if len(execludeTags) > 0 { if ok, tag := c.HasIntersection(execludeTags, candidateTags); ok { return fmt.Errorf("Execlude by schedtag: '%s:%s'", tag.Name, tag.Id) } } - requireTags := p.GetRequireTags() if len(requireTags) > 0 { if ok, tag := c.Contains(candidateTags, requireTags); !ok { - return fmt.Errorf("Need schedtag: '%s'", tag.Idx) + return fmt.Errorf("Need schedtag: '%s'", tag.Id) } } diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go index 46698dd13d..b1b2e7da59 100644 --- a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go @@ -25,7 +25,7 @@ func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostP return core.HostPriority{}, err } - ownerTenantID := u.SchedData().OwnerTenantID + ownerTenantID := u.SchedData().Project if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { h.SetFrontRawScore(-1 * int(count)) } diff --git a/pkg/scheduler/api/history.go b/pkg/scheduler/api/history.go index 7f79a252ee..16823ccb0b 100644 --- a/pkg/scheduler/api/history.go +++ b/pkg/scheduler/api/history.go @@ -67,7 +67,7 @@ type HistoryDetailArgs struct { type HistoryTask struct { Type string `json:"type"` Status string `json:"status"` - Data *SchedData `json:"data"` + Data *SchedInfo `json:"data"` Time string `json:"time"` Consuming string `json:"consuming"` //Result []SchedResultItem `json:"result"` diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index eca23da386..9d3056555c 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -1,321 +1,144 @@ package api import ( - "encoding/json" - "fmt" - "strconv" - "strings" - - simplejson "github.com/bitly/go-simplejson" + "net/http" + //"yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/compute/baremetal" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/compute/models" o "yunion.io/x/onecloud/pkg/scheduler/options" ) -type Meta map[string]string +type SchedInfo struct { + *api.ScheduleInput -type Disk struct { - Backend string `json:"backend"` - ImageID string `json:"image_id"` - Fs *string `json:"fs"` - Os string `json:"os"` - OSDistribution string `json:"os_distribution"` - OsVersion string `json:"os_version"` - Format string `json:"format"` - MountPoint *string `json:"mountpoint"` - Driver *string `json:"driver"` - Cache *string `json:"cache"` - ImageDiskFormat string `json:"image_disk_format"` - Size int64 `json:"size"` - Storage *string `json:"storage"` - Schedtags []Schedtag `json:"schedtags"` - Index int `json:"index"` + Tag string `json:"tag"` + Type string `json:"type"` + IsContainer bool `json:"is_container"` + Candidates []string `json:"candidates"` + + IgnoreFilters map[string]bool `json:"ignore_filters"` + IsSuggestion bool `json:"suggestion"` + ShowSuggestionDetails bool `json:"suggestion_details"` + Raw string } -type Network struct { - Idx string `json:"idx"` - TenantId string `json:"tenant_id"` - Private bool `json:"private"` - Ports int64 `json:"ports"` - Exit bool `json:"exit"` - Wire string `json:"wire"` - Mac string `json:"mac"` - Address string `json:"address"` - Address6 string `json:"address6"` - Driver string `json:"driver"` - BwLimit int64 `json:"bw_limit"` - Vip bool `json:"vip"` - Reserved bool `json:"reserved"` -} - -type IsolatedDevice struct { - ID string `json:"id"` - Type string `json:"dev_type"` - Model string `json:"model"` - Vendor string `json:"vendor"` -} - -type ForGuest struct { - ID string `json:"id"` - Name string `json:"name"` -} - -type Schedtag struct { - Idx string `json:"idx"` - Strategy string `json:"strategy"` -} - -type GroupRelation struct { - GroupID string `json:"group_id"` - Strategy string `json:"strategy"` - Scope string `json:"scope"` -} - -type SchedData struct { - Tag string `json:"tag"` - Type string `json:"type"` - IsContainer bool `json:"is_container"` - Count int64 `json:"count"` - RegionID string `json:"region_id"` - ZoneID string `json:"zone_id"` - HostID string `json:"host_id"` - Candidates []string `json:"candidates"` - OwnerTenantID string `json:"owner_tenant_id"` - OwnerUserID string `json:"owner_user_id"` - VMEMSize int64 `json:"vmem_size"` - VCPUCount int64 `json:"vcpu_count"` - Disks []*Disk `json:"disks"` - Name string `json:"name"` - Networks []*Network `json:"networks"` - IsolatedDevices []*IsolatedDevice `json:"isolated_devices"` - Schedtags []Schedtag `json:"aggregate_stategy"` - Meta Meta `json:"__meta__"` - ForGuests []*ForGuest `json:"for_guests"` - GuestStatus string `json:"guest_status"` - Hypervisor string `json:"hypervisor"` - ResourceType string `json:"resource_type"` - - // VM - Groups []string `json:"group"` - GroupRelations []GroupRelation `json:"group_relations"` - - // baremental - BaremetalDiskConfigs []*baremetal.BaremetalDiskConfig `json:"baremetal_disk_config"` - - // vm backup schedule - // Schedule server with backup server - Backup bool `json:"backup"` - // Backup master host - PreferHostId string `json:"prefer_host_id"` - // Backup host should be scheduled - BackupHostID string `json:"backup_host_id"` -} - -func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, error) { - data := new(SchedData) - - if tag, ok := sjson.CheckGet("tag"); ok { - if str, err := tag.String(); err == nil { - data.Tag = str - } - } - - if zoneID, ok := sjson.CheckGet("prefer_zone_id"); ok { - if str, err := zoneID.String(); err == nil { - data.ZoneID = str - } - } else if zoneID, ok := sjson.CheckGet("prefer_zone"); ok { - if str, err := zoneID.String(); err == nil { - data.ZoneID = str - } - } - - if regionID, ok := sjson.CheckGet("prefer_region_id"); ok { - if str, err := regionID.String(); err == nil { - data.RegionID = str - } - } - - candidates := make([]string, 0) - - if backupObj, ok := sjson.CheckGet("backup"); ok { - if backup, err := backupObj.Bool(); err == nil && backup { - data.Backup = true - } - } - - if hostID, ok := sjson.CheckGet("prefer_host_id"); ok { - if str, err := hostID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } else { - data.PreferHostId = str - } - } - } else if hostID, ok := sjson.CheckGet("prefer_host"); ok { - if str, err := hostID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } else { - data.PreferHostId = str - } - } - } - - if baremetalID, ok := sjson.CheckGet("prefer_baremetal_id"); ok { - if str, err := baremetalID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } - } - } else if baremetalID, ok := sjson.CheckGet("prefer_baremetal"); ok { - if str, err := baremetalID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } - } - } - - if backupHostID, ok := sjson.CheckGet("prefer_backup_host_id"); ok { - if backHost, err := backupHostID.String(); err == nil { - data.BackupHostID = backHost - } - } else if backupHostID, ok := sjson.CheckGet("prefer_backup_host"); ok { - if backHost, err := backupHostID.String(); err == nil { - data.BackupHostID = backHost - } - } - - if resourceType, ok := sjson.CheckGet("resource_type"); ok { - data.ResourceType = resourceType.MustString() - } else { - data.ResourceType = models.HostResourceTypeShared - } - - data.Candidates = candidates - - err := data.reviseSchedType(sjson) +func FetchSchedInfo(req *http.Request) (*SchedInfo, error) { + body, err := appsrv.FetchJSON(req) if err != nil { return nil, err } - data.reviseSchedHypervisor(sjson) - - data.VMEMSize = sjson.Get("vmem_size").MustInt64() - data.OwnerTenantID = sjson.Get("owner_tenant_id").MustString() - data.OwnerUserID = sjson.Get("owner_user_id").MustString() - data.VCPUCount = sjson.Get("vcpu_count").MustInt64() - data.Name = sjson.Get("name").MustString() - data.Count = count - data.GuestStatus = sjson.Get("guest_status").MustString() - data.Meta = map[string]string{} - data.HostID = sjson.Get("host_id").MustString() - - for key, value := range sjson.Get("__meta__").MustMap() { - data.Meta[key] = fmt.Sprintf("%v", value) - } - - if err := data.fillNetworksInfo(sjson); err != nil { + input, err := cmdline.FetchScheduleInputByJSON(body) + if err != nil { return nil, err } - if err := data.fillIsolatedDeviceInfo(sjson); err != nil { - return nil, err - } - - if err := data.fillDisksInfo(sjson, byTest); err != nil { - return nil, err - } - - data.fillForGuests(sjson) - - if err := data.fillSchedtags(sjson, byTest); err != nil { - return nil, err - } - - if err := data.fillGroupRelations(sjson); err != nil { - return nil, err - } - - if err := data.fillBaremetalDiskConfig(sjson); err != nil { - return nil, err - } - - if err := data.fillGroups(sjson); err != nil { - return nil, err - } - - data.reviseSchedData() + data := NewSchedInfo(input) return data, nil } -func (s *SchedData) reviseSchedHypervisor(sjson *simplejson.Json) { - var finalHypervisor string - switch hypervisor := sjson.Get("hypervisor").MustString(); hypervisor { - case "", SchedTypeKvm: - finalHypervisor = HostHypervisorForKvm - default: - finalHypervisor = hypervisor +func NewSchedInfo(input *api.ScheduleInput) *SchedInfo { + data := new(SchedInfo) + data.ScheduleInput = input + + if data.Hypervisor == "" || data.Hypervisor == SchedTypeKvm { + data.Hypervisor = HostHypervisorForKvm } - s.Hypervisor = finalHypervisor + + candidates := make([]string, 0) + if !data.Backup && data.PreferHost != "" { + candidates = append(candidates, data.PreferHost) + } + + //if !data.Backup && data.PreferBaremetal != "" { + //candidates = append(candidates, data.PreferBaremetal) + //} + + if data.ResourceType == "" { + data.ResourceType = models.HostResourceTypeShared + } + + if len(data.BaremetalDiskConfigs) == 0 { + defaultConfs := []*computeapi.BaremetalDiskConfig{&computeapi.BaremetalDefaultDiskConfig} + data.BaremetalDiskConfigs = defaultConfs + log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) + } + + data.Candidates = candidates + + data.reviseData() + + return data } -func (s *SchedData) reviseSchedType(sjson *simplejson.Json) error { - var bodyType string - if reqType, ok := sjson.CheckGet("type"); ok { - bodyType = reqType.MustString() - } else { - bodyType = HostTypeHost +func (data *SchedInfo) reviseData() { + for _, d := range data.Disks { + if len(d.Backend) == 0 { + d.Backend = computeapi.STORAGE_LOCAL + } } - switch bodyType { - case "", HostTypeHost, SchedTypeGuest, SchedTypeEsxi, SchedTypeKvm, SchedTypeHyperV: - s.Type = SchedTypeGuest + input := data.ScheduleInput - case SchedTypeContainer: - s.Type = SchedTypeGuest - s.IsContainer = true - - case HostTypeBaremetal: - s.Type = SchedTypeBaremetal - - default: - return fmt.Errorf("Sched current type=%s not support", bodyType) + if input.Count <= 0 { + data.Count = 1 } - return nil + + ignoreFilters := make(map[string]bool, len(input.IgnoreFilters)) + for _, filter := range input.IgnoreFilters { + ignoreFilters[filter] = true + } + data.IgnoreFilters = ignoreFilters + + if data.SuggestionLimit == 0 { + data.SuggestionLimit = int64(o.GetOptions().SchedulerTestLimit) + } + + data.Raw = input.JSON(input).String() } -func (d *SchedData) SkipDirtyMarkHost() bool { +func (d *SchedInfo) SkipDirtyMarkHost() bool { isSharePublicCloudProvider := d.IsPublicCloudProvider() && (d.ResourceType == "" || d.ResourceType == models.HostResourceTypeShared) skipByHypervisor := isSharePublicCloudProvider || d.IsContainer || d.Hypervisor == SchedTypeContainer skipByBackup := d.Backup return skipByHypervisor || skipByBackup } -func (d *SchedData) IsPublicCloudProvider() bool { +func (d *SchedInfo) GetCandidateHostTypes() []string { + switch d.Hypervisor { + case SchedTypeContainer: + return []string{HostTypeKubelet, HostHypervisorForKvm} + default: + return []string{d.Hypervisor} + } +} + +func (d *SchedInfo) IsPublicCloudProvider() bool { return PublicCloudProviders.Has(d.Hypervisor) } -func (d *SchedData) getDiskSize(backend string) int64 { +func (d *SchedInfo) getDiskSize(backend string) int64 { total := int64(0) for _, disk := range d.Disks { if disk.Backend == backend { - total += disk.Size + total += int64(disk.SizeMb) } } return total } -func (d *SchedData) AllDiskBackendSize() map[string]int64 { +func (d *SchedInfo) AllDiskBackendSize() map[string]int64 { backendSizeMap := make(map[string]int64, len(d.Disks)) for _, disk := range d.Disks { - newSize := disk.Size + newSize := int64(disk.SizeMb) if size, ok := backendSizeMap[disk.Backend]; ok { newSize += size } @@ -326,552 +149,6 @@ func (d *SchedData) AllDiskBackendSize() map[string]int64 { return backendSizeMap } -func parseSize(s string, defaultUnit string, base int) (int64, error) { - if utils.IsMatch("^\\d+$", s) { - s = s + defaultUnit - } - unit := s[len(s)-1:] - size, err := strconv.ParseFloat(s[0:len(s)-1], 64) - if err != nil { - return 0, err - } - - if unit == "g" || unit == "G" { - size *= float64(base * base * base) - } else if unit == "m" || unit == "M" { - size *= float64(base * base) - } else if unit == "k" || unit == "K" { - size *= float64(base) - } - - return int64(size), nil -} - -func macUnpackHex(mac string) (string, error) { - d := strings.Split(mac, ":") - if len(d) < 6 { - d = strings.Split(mac, "-") - } - if len(d) == 6 { - return strings.Join(d, ""), nil - } - - return "", fmt.Errorf("Mac unpack hex: format error: %s", mac) -} - -func newNetworkFromDesc(desc string) (*Network, error) { - network := new(Network) - network.Driver = "virtio" - network.Idx = "" - network.Wire = "" - network.Exit = false - network.Private = false - network.Mac = "" - network.Address = "" - network.Address6 = "" - - for _, it := range strings.Split(desc, ":") { - if utils.IsMatchIP4(it) { - network.Address = it - } else if utils.IsMatchIP6(it) { - network.Address6 = it - } else if utils.IsMatchCompactMacAddr(it) { - mac, err := macUnpackHex(it) - if err != nil { - return nil, err - } - network.Mac = mac - } else if strings.HasPrefix(it, "wire=") { - network.Wire = it[len("wire="):] - } else if it == "[random_exit]" { - network.Exit = true - } else if it == "[random]" { - network.Exit = false - } else if it == "[private]" { - network.Private = true - } else if it == "[reserved]" { - network.Reserved = true - } else if it == "virtio" || it == "e1000" || it == "sriov" { - network.Driver = it - } else if utils.IsMatchSize(it) { - limit, err := parseSize(it, "M", 1000) - if err != nil { - return nil, err - } - network.BwLimit = limit / 1000 / 1000 - } else if it == "[vip]" { - network.Vip = true - } else { - network.Idx = it - } - } - - return network, nil -} - -func (d *SchedData) fillNetworksInfo(sjson *simplejson.Json) error { - networks := []*Network{} - for index := 0; ; index++ { - netIndex := fmt.Sprintf("net.%d", index) - s, ok := sjson.CheckGet(netIndex) - if !ok { - break - } - net, err := newNetworkFromDesc(s.MustString()) - if err != nil || net.Idx == "" { - net = new(Network) - net.Idx = s.Get("network").MustString() - net.Wire = s.Get("wire").MustString() - net.Driver = s.Get("driver").MustString() - net.Exit = s.Get("exit").MustBool() - net.BwLimit = s.Get("bw_limit").MustInt64() - net.Private = s.Get("private").MustBool() - net.Reserved = s.Get("reserved").MustBool() - net.Vip = s.Get("vip").MustBool() - } - networks = append(networks, net) - } - d.Networks = networks - return nil -} - -func newIsolatedDeviceFromDesc(desc string) (dev *IsolatedDevice, err error) { - dev = new(IsolatedDevice) - for _, it := range strings.Split(desc, ":") { - if utils.IsMatchUUID(it) { - dev.ID = it - } else if ValidPassthroughTypes.Has(it) { - dev.Type = it - } else if strings.HasPrefix(it, "vendor=") { - vendor := it[len("vendor="):] - vid, ok := IsolatedVendorIDMap[strings.ToUpper(vendor)] - if !ok { - vid = vendor - } - dev.Vendor = vid - } else { - dev.Model = it - } - } - if len(dev.ID) == 0 && len(dev.Model) == 0 { - dev = nil - err = fmt.Errorf("Invalid isolated device description: %s", desc) - return - } - return -} - -func (d *SchedData) fillIsolatedDeviceInfo(sjson *simplejson.Json) error { - devs := []*IsolatedDevice{} - for idx := 0; ; idx++ { - devIdx := fmt.Sprintf("isolated_device.%d", idx) - s, ok := sjson.CheckGet(devIdx) - if !ok { - break - } - dev, err := newIsolatedDeviceFromDesc(s.MustString()) - if err != nil { - dev = new(IsolatedDevice) - dev.ID = s.Get("id").MustString() - dev.Model = s.Get("model").MustString() - dev.Vendor = s.Get("vendor").MustString() - dev.Type = s.Get("dev_type").MustString() - if len(dev.ID) == 0 && len(dev.Model) == 0 { - return fmt.Errorf("Invalid isolated device description") - } - } - devs = append(devs, dev) - } - d.IsolatedDevices = devs - return nil -} - -func newDiskFromCmdline(dstr string) (*Disk, error) { - matchFs := func(fs string) bool { - switch fs { - case "swap", "ext2", "ext3", "ext4", "xfs", "ntfs", "fat", "hfsplus": - return true - default: - return false - } - } - - matchFormat := func(f string) bool { - switch f { - case "qcow2", "raw", "docker", "iso", "vmdk": - return true - default: - return false - } - } - - matchDriver := func(d string) bool { - switch d { - case "virtio", "ide", "scsi", "sata", "pvscsi": - return true - default: - return false - } - } - - matchCache := func(c string) bool { - switch c { - case "writeback", "none", "writethrough": - return true - default: - return false - } - } - - matchBackend := func(b string) bool { - switch b { - case "local", "baremetal", "docker": - return true - default: - return false - } - } - disk := &Disk{Backend: "local"} - for _, nd := range strings.Split(dstr, ":") { - d := nd - if utils.IsMatchSize(d) { - size, err := parseSize(d, "G", 1024) - if err != nil { - return nil, err - } - disk.Size = size / 1024 / 1024 - } - if matchFs(d) { - disk.Fs = &d - } - if matchFormat(d) { - disk.Format = d - } - if matchDriver(d) { - disk.Driver = &d - } - if matchCache(d) { - disk.Cache = &d - } - if d == "autoextend" { - disk.Size = -1 - } - if matchBackend(d) { - disk.Backend = d - } - if strings.HasPrefix(d, "/") { - disk.MountPoint = &d - } - } - return disk, nil -} - -func newDiskFromSimpleJson(sjson *simplejson.Json, byTest bool) (*Disk, error) { - disk := new(Disk) - if byTest { - return newDiskFromCmdline(sjson.MustString()) - } - disk.Backend = sjson.Get("backend").MustString() - disk.ImageID = sjson.Get("image_id").MustString() - if fs, ok := sjson.CheckGet("fs"); ok { - if str, err := fs.String(); err == nil { - disk.Fs = &str - } - } - disk.Os = sjson.Get("os").MustString() - disk.OSDistribution = sjson.Get("os_distribution").MustString() - disk.OsVersion = sjson.Get("os_version").MustString() - - disk.Format = sjson.Get("format").MustString() - if mountPoint, ok := sjson.CheckGet("mountpoint"); ok { - if str, err := mountPoint.String(); err == nil { - disk.MountPoint = &str - } - } - if driver, ok := sjson.CheckGet("driver"); ok { - if str, err := driver.String(); err == nil { - disk.Driver = &str - } - } - if cache, ok := sjson.CheckGet("cache"); ok { - if str, err := cache.String(); err == nil { - disk.Cache = &str - } - } - disk.ImageDiskFormat = sjson.Get("image_disk_format").MustString() - disk.Size = sjson.Get("size").MustInt64() - - if storage, ok := sjson.CheckGet("storage"); ok { - if str, err := storage.String(); err == nil { - disk.Storage = &str - } - } - return disk, nil -} - -func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*baremetal.BaremetalDiskConfig, error) { - baremetalDiskConfig := new(baremetal.BaremetalDiskConfig) - baremetalDiskConfig.Count = sjson.Get("count").MustInt64() - baremetalDiskConfig.Conf = sjson.Get("conf").MustString() - - rangeArray := make([]int64, 0) - ranges := sjson.Get("range").MustArray() - for _, size := range ranges { - sizeNum, ok := size.(json.Number) - if !ok { - return nil, fmt.Errorf("Invalid range size, not json.Number: %v", size) - } - rangeSize, err := sizeNum.Int64() - if err != nil { - return nil, err - } - rangeArray = append(rangeArray, rangeSize) - } - baremetalDiskConfig.Range = rangeArray - - baremetalDiskConfig.Splits = sjson.Get("splits").MustString() - baremetalDiskConfig.Type = sjson.Get("type").MustString() - ada := sjson.Get("adapter").MustInt() - baremetalDiskConfig.Adapter = &ada - if val, ok := sjson.CheckGet("strip"); ok { - strip := val.MustInt64() - baremetalDiskConfig.Strip = &strip - } - if val, ok := sjson.CheckGet("cachedbadbbu"); ok { - cachedbadbbu := val.MustBool() - baremetalDiskConfig.Cachedbadbbu = &cachedbadbbu - } - - return baremetalDiskConfig, nil -} - -func (d *SchedData) fillDisksInfo(sjson *simplejson.Json, byTest bool) error { - disks := make([]*Disk, 0) - index := 0 - for { - diskIndex := fmt.Sprintf("disk.%d", index) - d, ok := sjson.CheckGet(diskIndex) - if !ok { - break - } - disk, err := newDiskFromSimpleJson(d, byTest) - if err != nil { - return err - } - disk.Index = index - index++ - disks = append(disks, disk) - } - if index == 0 && d.Hypervisor != SchedTypeContainer { - return fmt.Errorf("No disk info found in json") - } - d.Disks = disks - return nil -} - -func (d *SchedData) fillBaremetalDiskConfig(sjson *simplejson.Json) error { - baremetalDiskConfigs := []*baremetal.BaremetalDiskConfig{} - if d.Hypervisor != HostTypeBaremetal { - return nil - } - config, ok := sjson.CheckGet("baremetal_disk_config") - if !ok { - defaultConfs := []*baremetal.BaremetalDiskConfig{&baremetal.BaremetalDefaultDiskConfig} - d.BaremetalDiskConfigs = defaultConfs - log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) - return nil - } - for index := range config.MustArray() { - baremetalDiskConfig, err := newBaremetalDiskConfigFromSimpleJson(config.GetIndex(index)) - if err != nil { - return err - } - baremetalDiskConfigs = append(baremetalDiskConfigs, baremetalDiskConfig) - } - - d.BaremetalDiskConfigs = baremetalDiskConfigs - return nil -} - -func (d *SchedData) fillForGuests(sjson *simplejson.Json) { - newGuest := func(sjson *simplejson.Json) *ForGuest { - gst := new(ForGuest) - gst.ID = sjson.Get("id").MustString() - gst.Name = sjson.Get("name").MustString() - return gst - } - for i := range sjson.Get("for_guests").MustArray() { - gst := sjson.Get("for_guests").GetIndex(i) - d.ForGuests = append(d.ForGuests, newGuest(gst)) - } -} - -func NewSchedTagFromCmdline(str string) (agg Schedtag, err error) { - rs := strings.Split(str, ":") - if len(rs) == 1 || rs[1] == "" { - err = fmt.Errorf("SchedTag %q no strategy.", str) - return - } - name, strategy := rs[0], rs[1] - - err = SchedtagStrategyCheck(strategy) - if err != nil { - return - } - - agg = Schedtag{name, strategy} - return -} - -func (d *SchedData) fillSchedtags(sjson *simplejson.Json, byTest bool) error { - d.Schedtags = []Schedtag{} - - if !byTest { - if aggNode, ok := sjson.CheckGet("aggregate_strategy"); ok { - for name, strategy := range aggNode.MustMap() { - d.Schedtags = append(d.Schedtags, Schedtag{ - Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), - }) - } - } - } else { - index := 0 - for { - aggIndex := fmt.Sprintf("aggregate.%d", index) - a, ok := sjson.CheckGet(aggIndex) - if !ok { - break - } - agg, err := NewSchedTagFromCmdline(a.MustString()) - if err != nil { - return err - } - d.Schedtags = append(d.Schedtags, agg) - index++ - } - } - - return nil -} - -func (d *SchedData) fillGroupRelations(sjson *simplejson.Json) error { - d.GroupRelations = []GroupRelation{} - - if aggNode, ok := sjson.CheckGet("group_relations"); ok { - for index := range aggNode.MustArray() { - it := aggNode.GetIndex(index) - - r := GroupRelation{} - - if v, ok := it.CheckGet("id"); ok { - r.GroupID = v.MustString() - } else { - return fmt.Errorf("id missing in group relation") - } - - if v, ok := it.CheckGet("scope"); ok { - r.Scope = v.MustString() - } else { - r.Scope = "host" - } - - if v, ok := it.CheckGet("strategy"); ok { - r.Strategy = v.MustString() - } else { - r.Strategy = "" - } - - d.GroupRelations = append(d.GroupRelations, r) - } - } - - return nil -} - -func (d *SchedData) fillGroups(sjson *simplejson.Json) error { - d.Groups = []string{} - if groupsNode, ok := sjson.CheckGet("groups"); ok { - for index := range groupsNode.MustArray() { - it := groupsNode.GetIndex(index) - d.Groups = append(d.Groups, it.MustString()) - } - } - - return nil -} - -// TODO -func (d *SchedData) reviseSchedData() { - /*if strategy, ok := data.Meta["aggregate_strategy"]; ok { - d.Aggregates = append(d.Aggregates, Aggregate{ - Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), - }) - }*/ -} - -type SchedInfo struct { - Data *SchedData `json:"scheduler"` - IgnoreFilters map[string]bool `json:"ignore_filters"` - SessionID string `json:"session_id"` - IsSuggestion bool `json:"suggestion"` - ShowSuggestionDetails bool `json:"suggestion_details"` - SuggestionLimit int64 `json:"suggestion_limit"` - SuggestionAll bool `json:"suggestion_all"` - Raw string `json:"raw"` - BestEffort bool `json:"best_effort"` -} - -func NewSchedInfo(sjson *simplejson.Json, byTest bool) (*SchedInfo, error) { - info := new(SchedInfo) - - schedData, ok := sjson.CheckGet("scheduler") - if !ok { - return nil, fmt.Errorf("Not found 'scheduler' from request body") - - } - - count := sjson.Get("count").MustInt64(1) - - if data, err := NewSchedData(schedData, count, byTest); err == nil { - info.Data = data - } else { - return nil, err - } - - info.SessionID = sjson.Get("session_id").MustString() - - ignoreFiltersString := sjson.Get("ignore_filters").MustString() - filters := strings.Split(ignoreFiltersString, ",") - ignoreFilters := make(map[string]bool, len(filters)) - for _, filter := range filters { - ignoreFilters[filter] = true - - } - info.IgnoreFilters = ignoreFilters - - info.SuggestionLimit = sjson.Get("suggestion_limit").MustInt64() - if info.SuggestionLimit == 0 { - info.SuggestionLimit = int64(o.GetOptions().SchedulerTestLimit) - } - info.SuggestionAll = sjson.Get("suggestion_all").MustBool() - info.ShowSuggestionDetails = sjson.Get("suggestion_details").MustBool() - info.BestEffort = sjson.Get("best_effort").MustBool() - - if bytes, err := sjson.EncodePretty(); err == nil { - info.Raw = string(bytes) - - } - return info, nil -} - -func (i *SchedInfo) String() (string, error) { - bytes, err := json.Marshal(i) - if err != nil { - return "", err - } - return string(bytes), nil -} - type SchedResultItem interface{} type SchedResult struct { diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go index 51dfc2b831..454f2b362c 100644 --- a/pkg/scheduler/cache/candidate/baremetals.go +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -189,7 +189,6 @@ func (bb *BaremetalBuilder) Type() string { } func (bb *BaremetalBuilder) AllIDs() ([]string, error) { - log.Errorf("=======get baremetal host ids") return models.AllBaremetalIDs() } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index fc7fc7aab2..b55a85aa3e 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -172,12 +172,14 @@ func (b *BaseHostDesc) fillNetworks(hostID string) error { return nil } -func (b *BaseHostDesc) fillStorages(storages []*computemodels.SStorage) error { +func (b *BaseHostDesc) fillStorages(host *computemodels.SHost) error { ss := make([]*api.CandidateStorage, 0) - for _, s := range storages { + //log.Errorf("====host %s append storages: %#v", b.Name, storages) + for _, s := range host.GetHoststorages() { + storage := s.GetStorage() ss = append(ss, &api.CandidateStorage{ - SStorage: s, - Schedtags: s.GetSchedtags(), + SStorage: storage, + Schedtags: storage.GetSchedtags(), }) } b.Storages = ss diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 32b9399530..0cad7c9ce1 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -182,8 +182,8 @@ type HostBuilder struct { guestDict map[string]interface{} guestIDs []string - hostStorages []computemodels.SHoststorage - hostStoragesDict map[string][]*computemodels.SStorage + hostStorages []computemodels.SHoststorage + //hostStoragesDict map[string][]*computemodels.SStorage storages []interface{} storageStatesSizeDict map[string]map[string]interface{} @@ -540,7 +540,7 @@ func (b *HostBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGr b.setMetadataInfo(ids, errMessageChannel) }, func() { - b.setStorages(ids, errMessageChannel) + //b.setStorages(ids, errMessageChannel) b.setDiskStats(errMessageChannel) }, func() { b.setMetadataInfo(ids, errMessageChannel) }, @@ -605,32 +605,32 @@ func (b *HostBuilder) setSchedtags(ids []string, errMessageChannel chan error) { b.schedtags = tags } -func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { - q := computemodels.HoststorageManager.Query().In("host_id", ids) - hostStorages := make([]computemodels.SHoststorage, 0) - err := computedb.FetchModelObjects(computemodels.HoststorageManager, q, &hostStorages) - if err != nil { - errMessageChannel <- err - return - } +//func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { +//q := computemodels.HoststorageManager.Query().In("host_id", ids) +//hostStorages := make([]computemodels.SHoststorage, 0) +//err := computedb.FetchModelObjects(computemodels.HoststorageManager, q, &hostStorages) +//if err != nil { +//errMessageChannel <- err +//return +//} - hostStoragesDict := make(map[string][]*computemodels.SStorage) +////hostStoragesDict := make(map[string][]*computemodels.SStorage) - for _, s := range hostStorages { - if ss, ok := hostStoragesDict[s.HostId]; !ok { - storage := s.GetStorage() - ss = make([]*computemodels.SStorage, 0) - ss = append(ss, storage) - hostStoragesDict[s.HostId] = ss - } else { - ss = append(ss, s.GetStorage()) - } - } +//for _, s := range hostStorages { +//if ss, ok := hostStoragesDict[s.HostId]; !ok { +//storage := s.GetStorage() +//ss = make([]*computemodels.SStorage, 0) +//ss = append(ss, storage) +//hostStoragesDict[s.HostId] = ss +//} else { +//ss = append(ss, s.GetStorage()) +//} +//} - b.hostStorages = hostStorages - b.hostStoragesDict = hostStoragesDict - return -} +//b.hostStorages = hostStorages +//b.hostStoragesDict = hostStoragesDict +//return +//} func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { guests, err := FetchGuestByHostIDs(ids) @@ -898,7 +898,6 @@ func (b *HostBuilder) AllIDs() ([]string, error) { } ret = append(ret, id) } - log.Errorf("==============================\n ============================get hostsIDs: %v", ret) return ret, nil } @@ -1126,7 +1125,7 @@ func (b *HostBuilder) guestAppTags(guest computemodels.SGuest) []string { } func (b *HostBuilder) fillStorages(desc *HostDesc, host *computemodels.SHost) error { - return desc.fillStorages(b.hostStoragesDict[host.GetId()]) + return desc.fillStorages(host) } func (b *HostBuilder) fillSchedtags(desc *HostDesc, host *computemodels.SHost) error { diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index 053d0e4852..de22bb09ef 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -394,11 +394,11 @@ func (u *Unit) Info() string { } func (u *Unit) SessionID() string { - return u.SchedInfo.SessionID + return u.SchedInfo.SessionId } -func (u *Unit) SchedData() *api.SchedData { - return u.SchedInfo.Data +func (u *Unit) SchedData() *api.SchedInfo { + return u.SchedInfo } func (u *Unit) IsPublicCloudProvider() bool { diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 0dc59410c3..2ee0941464 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -15,6 +15,7 @@ import ( utiltrace "yunion.io/x/pkg/util/trace" "yunion.io/x/pkg/util/workqueue" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" o "yunion.io/x/onecloud/pkg/scheduler/options" ) @@ -99,12 +100,12 @@ func (g *GenericScheduler) Schedule(unit *Unit, candidates []Candidater) (*Sched // new trace follow all steps trace := utiltrace.New(fmt.Sprintf("SessionID: %s, schedule info: %s", - schedInfo.SessionID, unit.Info())) + schedInfo.SessionId, unit.Info())) defer trace.LogIfLong(100 * time.Millisecond) if len(candidates) == 0 { return nil, &NoResourceError{ - sessionID: schedInfo.SessionID, + sessionID: schedInfo.SessionId, info: unit.Info(), } } @@ -242,6 +243,14 @@ type SchedResultItem struct { *AllocatedResource } +func (item *SchedResultItem) ToCandidateResource() *schedapi.CandidateResource { + return &schedapi.CandidateResource{ + HostId: item.ID, + Name: item.Name, + Disks: item.Disks, + } +} + func GetCapacities(u *Unit, id string) (res map[string]int64) { res = make(map[string]int64) capacities := u.GetCapacities(id) diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index ab420bcaef..f267d52683 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -3,6 +3,7 @@ package core import ( "yunion.io/x/jsonutils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core/score" @@ -129,17 +130,12 @@ type Priority interface { ScoreIntervals() score.Intervals } -type DiskAllocatedResource struct { - Index int `json:"index"` - StorageId string `json:"storage_id"` -} - type AllocatedResource struct { - Disks []*DiskAllocatedResource `json:"disks"` + Disks []*schedapi.CandidateDisk `json:"disks"` } func NewAllocatedResource() *AllocatedResource { return &AllocatedResource{ - Disks: make([]*DiskAllocatedResource, 0), + Disks: make([]*schedapi.CandidateDisk, 0), } } diff --git a/pkg/scheduler/data_manager/candidate_manager.go b/pkg/scheduler/data_manager/candidate_manager.go index 36dd18637d..3071b2fc76 100644 --- a/pkg/scheduler/data_manager/candidate_manager.go +++ b/pkg/scheduler/data_manager/candidate_manager.go @@ -5,6 +5,7 @@ import ( "time" "yunion.io/x/pkg/util/ttlpool" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/scheduler/cache" candidatecache "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" @@ -15,9 +16,10 @@ import ( type CandidateGetArgs struct { // ResType is candidate host_type - ResType string - RegionID string - ZoneID string + ResType string + RegionID string + ZoneID string + HostTypes []string } type DataManager struct { @@ -222,6 +224,13 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida return true } + matchHostTypes := func(c core.Candidater, hostTypes []string) bool { + if len(hostTypes) == 0 { + return true + } + return utils.IsInStringArray(c.Getter().HostType(), hostTypes) + } + for _, c := range candidates { r := c.(core.Candidater) @@ -237,6 +246,10 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida continue } + if !matchHostTypes(r, args.HostTypes) { + continue + } + result = append(result, r) } diff --git a/pkg/scheduler/handler/backup_helper.go b/pkg/scheduler/handler/backup_helper.go index da5012df92..ef90c016da 100644 --- a/pkg/scheduler/handler/backup_helper.go +++ b/pkg/scheduler/handler/backup_helper.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/scheduler/api" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/scheduler/core" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" ) @@ -18,24 +18,27 @@ func transToBackupSchedResult(result *core.SchedResultItemList, preferMasterHost } apiResults := newBackupSchedResult(result, preferMasterHost, preferBackupHost, count) - return regionResponse(apiResults) + return apiResults } -func newBackupSchedResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string, count int64) []api.SchedResultItem { - apiResults := make([]api.SchedResultItem, 0) +func newBackupSchedResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string, count int64) *schedapi.ScheduleOutput { + ret := new(schedapi.ScheduleOutput) + apiResults := make([]*schedapi.CandidateResource, 0) for i := 0; i < int(count); i++ { log.V(10).Debugf("Select backup host from result: %s", result) target, err := getSchedBackupResult(result, preferMasterHost, preferBackupHost) if err != nil { - apiResults = append(apiResults, api.SchedErrItem{Error: err.Error()}) + er := &schedapi.CandidateResource{Error: err.Error()} + apiResults = append(apiResults, er) continue } - apiResults = append(apiResults, api.SchedSuccItem{Candidate: target}) + apiResults = append(apiResults, target) } - return apiResults + ret.Candidates = apiResults + return ret } -func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string) (*api.SchedBackupResultItem, error) { +func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string) (*schedapi.CandidateResource, error) { masterHost := selectMasterHost(result.Data, preferMasterHost, preferBackupHost) if masterHost == nil { return nil, fmt.Errorf("Can't find master host") @@ -49,10 +52,8 @@ func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, pr markHostUsed(backupHost) sort.Sort(sort.Reverse(result)) - ret := &api.SchedBackupResultItem{ - MasterID: masterHost.ID, - SlaveID: backupHost.ID, - } + ret := masterHost.ToCandidateResource() + ret.BackupCandidate = backupHost.ToCandidateResource() return ret, nil } diff --git a/pkg/scheduler/handler/forecast_helper.go b/pkg/scheduler/handler/forecast_helper.go index 18556bb1b1..42ceee3ee7 100644 --- a/pkg/scheduler/handler/forecast_helper.go +++ b/pkg/scheduler/handler/forecast_helper.go @@ -9,7 +9,7 @@ import ( func transToSchedForecastResult(result *core.SchedResultItemList) interface{} { unit := result.Unit - reqCount := unit.SchedData().Count + reqCount := int64(unit.SchedData().Count) var readyCount int64 filters := make([]*api.ForecastFilter, 0) diff --git a/pkg/scheduler/handler/handler.go b/pkg/scheduler/handler/handler.go index 4c6e8b6394..83e14be5be 100644 --- a/pkg/scheduler/handler/handler.go +++ b/pkg/scheduler/handler/handler.go @@ -11,12 +11,12 @@ import ( "yunion.io/x/log" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/db/models" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" - - computemodels "yunion.io/x/onecloud/pkg/compute/models" ) // InstallHandler is an interface that registes route and @@ -93,12 +93,7 @@ func doSchedulerTest(c *gin.Context) { return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, true) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -128,12 +123,7 @@ func doSchedulerForecast(c *gin.Context) { return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, true) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -268,12 +258,7 @@ func doSyncSchedule(c *gin.Context) { c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, false) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -285,10 +270,10 @@ func doSyncSchedule(c *gin.Context) { return } - count := schedInfo.Data.Count + count := int64(schedInfo.Count) var resp interface{} - if schedInfo.Data.Backup { - resp = transToBackupSchedResult(result, schedInfo.Data.HostID, schedInfo.Data.BackupHostID, count) + if schedInfo.Backup { + resp = transToBackupSchedResult(result, schedInfo.HostId, schedInfo.PreferBackupHost, count) } else { resp = transToRegionSchedResult(result.Data, count) } @@ -296,21 +281,15 @@ func doSyncSchedule(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func transToRegionSchedResult(result []*core.SchedResultItem, count int64) interface{} { - apiResults := make([]api.SchedResultItem, 0) +func transToRegionSchedResult(result []*core.SchedResultItem, count int64) *schedapi.ScheduleOutput { + apiResults := make([]*schedapi.CandidateResource, 0) succCount := 0 for _, nr := range result { for { if nr.Count <= 0 { break } - tr := api.SchedSuccItem{ - Candidate: api.SchedNormalResultItem{ - ID: nr.ID, - Name: nr.Name, - Data: nr.Data, - }, - } + tr := nr.ToCandidateResource() apiResults = append(apiResults, tr) nr.Count-- succCount++ @@ -321,12 +300,14 @@ func transToRegionSchedResult(result []*core.SchedResultItem, count int64) inter if int64(succCount) >= count { break } - er := api.SchedErrItem{Error: "Out of resource"} + er := &schedapi.CandidateResource{Error: "Out of resource"} apiResults = append(apiResults, er) succCount++ } - return regionResponse(apiResults) + return &schedapi.ScheduleOutput{ + Candidates: apiResults, + } } func regionResponse(v interface{}) interface{} { diff --git a/pkg/scheduler/manager/manager.go b/pkg/scheduler/manager/manager.go index 0b43409cf3..4ba896812a 100644 --- a/pkg/scheduler/manager/manager.go +++ b/pkg/scheduler/manager/manager.go @@ -113,8 +113,8 @@ func NewSessionID() string { // Schedule process the request data that is scheduled for dispatch and complements // the session information. func Schedule(info *api.SchedInfo) (*core.SchedResultItemList, error) { - if len(info.SessionID) == 0 { - info.SessionID = NewSessionID() + if len(info.SessionId) == 0 { + info.SessionId = NewSessionID() } return schedManager.schedule(info) } @@ -364,18 +364,19 @@ func newHistoryItem(historyItem *HistoryItem) *api.HistoryItem { forGuests := []string{} countDict := make(map[string]int64) - data := schedInfo.Data - tenants = append(tenants, data.OwnerTenantID) + data := schedInfo + tenants = append(tenants, data.Project) for _, forGuest := range data.ForGuests { - forGuests = append(forGuests, fmt.Sprintf("%v(%v)", forGuest.ID, forGuest.Name)) + //forGuests = append(forGuests, fmt.Sprintf("%v(%v)", forGuest.ID, forGuest.Name)) + forGuests = append(forGuests, fmt.Sprintf("%v", forGuest)) } guestType := data.Hypervisor if c, ok := countDict[guestType]; !ok { - countDict[guestType] = data.Count + countDict[guestType] = int64(data.Count) } else { - countDict[guestType] = c + data.Count + countDict[guestType] = c + int64(data.Count) } counts := []string{} @@ -412,7 +413,7 @@ func GetHistoryDetail(historyDetailArgs *api.HistoryDetailArgs) (*api.HistoryDet schedInfo := task.SchedInfo historyTasks := []api.HistoryTask{} - data := schedInfo.Data + data := schedInfo taskExecutor := task.GetTaskExecutor(data.Tag) historyTask := api.HistoryTask{ Type: data.Hypervisor, diff --git a/pkg/scheduler/manager/scheduler.go b/pkg/scheduler/manager/scheduler.go index 35ac8903a6..a344e11f28 100644 --- a/pkg/scheduler/manager/scheduler.go +++ b/pkg/scheduler/manager/scheduler.go @@ -14,7 +14,7 @@ type CandidatesProvider interface { CandidateManager() *data_manager.CandidateManager } -func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) ([]core.Candidater, error) { +func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedInfo) ([]core.Candidater, error) { var hosts []core.Candidater var err error @@ -23,8 +23,10 @@ func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) hosts, err = candidateManager.GetCandidatesByIds(provider.CandidateType(), schedData.Candidates) } else { args := data_manager.CandidateGetArgs{ - ResType: provider.CandidateType(), - ZoneID: schedData.ZoneID, + ResType: provider.CandidateType(), + ZoneID: schedData.PreferZone, + RegionID: schedData.PreferRegion, + HostTypes: schedData.GetCandidateHostTypes(), } hosts, err = candidateManager.GetCandidates(args) } @@ -87,7 +89,7 @@ func (b *BaremetalCandidatesProvider) Candidates() ([]core.Candidater, error) { } type Scheduler interface { - SchedData() *api.SchedData + SchedData() *api.SchedInfo CandidateManager() *data_manager.CandidateManager // Schedule process @@ -123,8 +125,8 @@ func (s *BaseScheduler) CandidateManager() *data_manager.CandidateManager { return s.schedManager.CandidateManager } -func (s *BaseScheduler) SchedData() *api.SchedData { - return s.schedInfo.Data +func (s *BaseScheduler) SchedData() *api.SchedInfo { + return s.schedInfo } func (s *BaseScheduler) Unit() *core.Unit { diff --git a/pkg/scheduler/manager/task_queue.go b/pkg/scheduler/manager/task_queue.go index 6fa0e167d0..b341291b22 100644 --- a/pkg/scheduler/manager/task_queue.go +++ b/pkg/scheduler/manager/task_queue.go @@ -222,7 +222,7 @@ func (tm *TaskManager) AddTask(schedulerManager *SchedulerManager, schedInfo *ap task := NewTask(schedulerManager, schedInfo) // Split into multiple scheduling tasks by host specification type. - if schedInfo.Data.Hypervisor == api.SchedTypeBaremetal { + if schedInfo.Hypervisor == api.SchedTypeBaremetal { scheduler, err = newBaremetalScheduler(schedulerManager, schedInfo) } else { scheduler, err = newGuestScheduler(schedulerManager, schedInfo) @@ -282,7 +282,7 @@ func (t *Task) GetTaskExecutor(tag string) *TaskExecutor { } func (t *Task) GetSessionID() string { - return t.SchedInfo.SessionID + return t.SchedInfo.SessionId } func (t *Task) GetStatus() string { @@ -338,7 +338,7 @@ func (t *Task) onError() { taskExecutor.Kill() } - log.Errorf("Remove Session on error: %v\n", t.SchedInfo.SessionID) + log.Errorf("Remove Session on error: %v\n", t.SchedInfo.SessionId) //t.manager.ReservedPoolManager.RemoveSession(t.SchedInfo.SessionID) close(t.waitCh) diff --git a/pkg/scheduler/options/doc.go b/pkg/scheduler/options/doc.go new file mode 100644 index 0000000000..583155b3fc --- /dev/null +++ b/pkg/scheduler/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/scheduler/options" diff --git a/pkg/scheduler/options/options.go b/pkg/scheduler/options/options.go index e8b335a274..ef57948781 100644 --- a/pkg/scheduler/options/options.go +++ b/pkg/scheduler/options/options.go @@ -3,7 +3,7 @@ package options import ( "os" - "yunion.io/x/onecloud/pkg/cloudcommon" + common_optoins "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/compute/options" ) @@ -91,6 +91,6 @@ func GetOptions() *SchedulerOptions { } func Init() { - cloudcommon.ParseOptions(&opt, os.Args, "region.conf", "scheduler") + common_optoins.ParseOptions(&opt, os.Args, "region.conf", "scheduler") options.Options = opt.ComputeOptions } diff --git a/pkg/scheduler/service/doc.go b/pkg/scheduler/service/doc.go new file mode 100644 index 0000000000..9893e063d1 --- /dev/null +++ b/pkg/scheduler/service/doc.go @@ -0,0 +1 @@ +package service // import "yunion.io/x/onecloud/pkg/scheduler/service" diff --git a/pkg/scheduler/service/service.go b/pkg/scheduler/service/service.go index 6e744e0ae6..75d06c2bc2 100644 --- a/pkg/scheduler/service/service.go +++ b/pkg/scheduler/service/service.go @@ -13,6 +13,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/db" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/db/models" @@ -57,7 +58,7 @@ func StartService() error { } commonOpts := &opts.CommonOptions - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") startSched() }) From 076ad985cc2a0ac15188545c029de964dfbba8a1 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 20 Mar 2019 12:02:49 +0800 Subject: [PATCH 26/26] move consts to apis --- pkg/apis/compute/host_const.go | 64 +++++++++++++ .../compute/loadbalancer_const.go} | 2 +- pkg/baremetal/pxe/dhcp.go | 8 +- pkg/cloudcommon/cmdline/parser.go | 2 +- pkg/compute/consts/consts.go | 1 - pkg/compute/consts/doc.go | 1 - pkg/compute/consts/hosttypes.go | 18 ---- pkg/compute/consts/networks.go | 9 -- pkg/compute/consts/storages.go | 25 ----- pkg/compute/models/guests.go | 1 + pkg/compute/models/hosts.go | 93 ++++++++++--------- pkg/compute/models/loadbalancer_common.go | 4 +- pkg/compute/models/loadbalanceracls.go | 10 +- pkg/compute/models/loadbalanceragents.go | 4 +- .../models/loadbalancerbackendgroups.go | 16 ++-- pkg/compute/models/loadbalancerbackends.go | 14 +-- .../models/loadbalancercertificates.go | 20 ++-- .../models/loadbalancerlistenerrules.go | 12 +-- pkg/compute/models/loadbalancerlisteners.go | 86 ++++++++--------- pkg/compute/models/loadbalancers.go | 32 +++---- pkg/compute/regiondrivers/aliyun.go | 28 +++--- pkg/compute/regiondrivers/kvm.go | 18 ++-- pkg/compute/regiondrivers/managedvirtual.go | 8 +- .../tasks/loadbalancer_acl_create_task.go | 8 +- .../tasks/loadbalancer_acl_delete_task.go | 6 +- .../tasks/loadbalancer_acl_sync_task.go | 8 +- .../tasks/loadbalancer_backend_create_task.go | 8 +- .../tasks/loadbalancer_backend_delete_task.go | 4 +- .../loadbalancer_backendgroup_create_task.go | 8 +- .../loadbalancer_backendgroup_delete_task.go | 6 +- .../loadbalancer_certificate_create_task.go | 8 +- .../loadbalancer_certificate_delete_task.go | 6 +- pkg/compute/tasks/loadbalancer_create_task.go | 12 +-- pkg/compute/tasks/loadbalancer_delete_task.go | 6 +- .../loadbalancer_listener_create_task.go | 12 +-- .../loadbalancer_listener_delete_task.go | 6 +- .../loadbalancer_listener_rule_create_task.go | 8 +- .../loadbalancer_listener_rule_delete_task.go | 6 +- .../tasks/loadbalancer_listener_start_task.go | 8 +- .../tasks/loadbalancer_listener_stop_task.go | 8 +- .../tasks/loadbalancer_listener_sync_task.go | 8 +- .../loadbalancer_listener_syncstatus_task.go | 6 +- pkg/compute/tasks/loadbalancer_start_task.go | 8 +- pkg/compute/tasks/loadbalancer_stop_task.go | 8 +- .../tasks/loadbalancer_syncstatus_task.go | 6 +- pkg/hostman/guestman/qemu-kvm.go | 6 +- pkg/hostman/hostinfo/hostinfo.go | 16 ++-- pkg/hostman/storageman/core.go | 12 +-- pkg/hostman/storageman/disk_local.go | 4 +- pkg/hostman/storageman/disk_nfs.go | 4 +- pkg/hostman/storageman/disk_rbd.go | 4 +- .../storageman/imagecachemanager_rbd.go | 4 +- pkg/hostman/storageman/storage_local.go | 4 +- pkg/hostman/storageman/storage_nfs.go | 8 +- pkg/hostman/storageman/storage_rbd.go | 8 +- pkg/lbagent/api.go | 10 +- pkg/lbagent/hastate.go | 6 +- pkg/lbagent/models/reflect.go | 4 +- .../options/loadbalancerbackendgroups.go | 6 +- pkg/util/aliyun/loadbalancer.go | 22 ++--- pkg/util/aliyun/loadbalancerbackend.go | 6 +- pkg/util/aliyun/loadbalancerbackendgroup.go | 6 +- pkg/util/aliyun/loadbalancerdefaultbackend.go | 6 +- .../aliyun/loadbalancerdefaultbackendgroup.go | 6 +- pkg/util/aliyun/loadbalancerhttplistener.go | 10 +- pkg/util/aliyun/loadbalancerhttpslistener.go | 10 +- .../aliyun/loadbalancermasterslavebackend.go | 4 +- .../loadbalancermasterslavebackendgroup.go | 6 +- pkg/util/aliyun/loadbalancertcplistener.go | 16 ++-- pkg/util/aliyun/loadbalancerudplistener.go | 8 +- pkg/util/aliyun/region.go | 4 +- pkg/util/qcloud/loadbalancer.go | 20 ++-- pkg/util/qcloud/loadbalancer_acl.go | 4 +- pkg/util/qcloud/loadbalancer_backend.go | 6 +- pkg/util/qcloud/loadbalancer_backendgroup.go | 8 +- pkg/util/qcloud/loadbalancer_listener.go | 78 ++++++++-------- pkg/util/qcloud/loadbalancer_listenerrule.go | 4 +- pkg/util/qemuimg/qemuimg.go | 4 +- 78 files changed, 487 insertions(+), 477 deletions(-) create mode 100644 pkg/apis/compute/host_const.go rename pkg/{compute/consts/loadbalancer.go => apis/compute/loadbalancer_const.go} (99%) delete mode 100644 pkg/compute/consts/consts.go delete mode 100644 pkg/compute/consts/doc.go delete mode 100644 pkg/compute/consts/hosttypes.go delete mode 100644 pkg/compute/consts/networks.go delete mode 100644 pkg/compute/consts/storages.go diff --git a/pkg/apis/compute/host_const.go b/pkg/apis/compute/host_const.go new file mode 100644 index 0000000000..de21bee994 --- /dev/null +++ b/pkg/apis/compute/host_const.go @@ -0,0 +1,64 @@ +package compute + +const ( + HOST_TYPE_BAREMETAL = "baremetal" + HOST_TYPE_HYPERVISOR = "hypervisor" // KVM + HOST_TYPE_KVM = "kvm" + HOST_TYPE_ESXI = "esxi" // # VMWare vSphere ESXi + HOST_TYPE_KUBELET = "kubelet" // # Kubernetes Kubelet + HOST_TYPE_HYPERV = "hyperv" // # Microsoft Hyper-V + HOST_TYPE_XEN = "xen" // # XenServer + + HOST_TYPE_ALIYUN = "aliyun" + HOST_TYPE_AWS = "aws" + HOST_TYPE_QCLOUD = "qcloud" + HOST_TYPE_AZURE = "azure" + HOST_TYPE_HUAWEI = "huawei" + HOST_TYPE_OPENSTACK = "openstack" + + HOST_TYPE_DEFAULT = HOST_TYPE_HYPERVISOR + + // # possible status + HOST_ONLINE = "online" + HOST_ENABLED = "online" + HOST_OFFLINE = "offline" + HOST_DISABLED = "offline" + + NIC_TYPE_IPMI = "ipmi" + NIC_TYPE_ADMIN = "admin" + // #NIC_TYPE_NORMAL = 'normal' + + BAREMETAL_INIT = "init" + BAREMETAL_PREPARE = "prepare" + BAREMETAL_PREPARE_FAIL = "prepare_fail" + BAREMETAL_READY = "ready" + BAREMETAL_RUNNING = "running" + BAREMETAL_MAINTAINING = "maintaining" + BAREMETAL_START_MAINTAIN = "start_maintain" + BAREMETAL_DELETING = "deleting" + BAREMETAL_DELETE = "delete" + BAREMETAL_DELETE_FAIL = "delete_fail" + BAREMETAL_UNKNOWN = "unknown" + BAREMETAL_SYNCING_STATUS = "syncing_status" + BAREMETAL_SYNC = "sync" + BAREMETAL_SYNC_FAIL = "sync_fail" + BAREMETAL_START_CONVERT = "start_convert" + BAREMETAL_CONVERTING = "converting" + BAREMETAL_START_FAIL = "start_fail" + BAREMETAL_STOP_FAIL = "stop_fail" + + HOST_STATUS_RUNNING = BAREMETAL_RUNNING + HOST_STATUS_READY = BAREMETAL_READY + HOST_STATUS_UNKNOWN = BAREMETAL_UNKNOWN +) + +const ( + HostResourceTypeShared = "shared" + HostResourceTypeDefault = HostResourceTypeShared + HostResourceTypePrepaidRecycle = "prepaid" + HostResourceTypeDedicated = "dedicated" +) + +var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN, HOST_TYPE_AZURE, HOST_TYPE_AWS, HOST_TYPE_QCLOUD, HOST_TYPE_HUAWEI, HOST_TYPE_OPENSTACK} + +var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN} diff --git a/pkg/compute/consts/loadbalancer.go b/pkg/apis/compute/loadbalancer_const.go similarity index 99% rename from pkg/compute/consts/loadbalancer.go rename to pkg/apis/compute/loadbalancer_const.go index 56ec8848b5..da2507b483 100644 --- a/pkg/compute/consts/loadbalancer.go +++ b/pkg/apis/compute/loadbalancer_const.go @@ -1,4 +1,4 @@ -package consts +package compute import ( "yunion.io/x/onecloud/pkg/util/choices" diff --git a/pkg/baremetal/pxe/dhcp.go b/pkg/baremetal/pxe/dhcp.go index bca1951992..858de5359a 100644 --- a/pkg/baremetal/pxe/dhcp.go +++ b/pkg/baremetal/pxe/dhcp.go @@ -9,9 +9,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" o "yunion.io/x/onecloud/pkg/baremetal/options" "yunion.io/x/onecloud/pkg/cloudcommon/types" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/dhcp" ) @@ -165,7 +165,7 @@ func (h *DHCPHandler) fetchConfig() (*dhcp.ResponseConfig, error) { h.baremetalInstance = bmInstance ipmiNic := h.baremetalInstance.GetIPMINic(h.ClientMac) if ipmiNic != nil && ipmiNic.Mac == h.ClientMac.String() { - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, consts.NETWORK_TYPE_IPMI) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_IPMI, api.NETWORK_TYPE_IPMI) if err != nil { return nil, err } @@ -215,7 +215,7 @@ func (h *DHCPHandler) findNetworkConf(filterUseIp bool) (*types.SNetworkConfig, func (h *DHCPHandler) findBaremetalsOfAnyMac(isBaremetal bool) (*modules.ListResult, error) { session := h.baremetalManager.GetClientSession() params := jsonutils.NewDict() - params.Add(jsonutils.NewString(consts.HOST_TYPE_BAREMETAL), "host_type") + params.Add(jsonutils.NewString(api.HOST_TYPE_BAREMETAL), "host_type") params.Add(jsonutils.NewString(h.ClientMac.String()), "any_mac") if isBaremetal { params.Add(jsonutils.JSONTrue, "is_baremetal") @@ -292,7 +292,7 @@ func (h *DHCPHandler) doInitBaremetalAdminNetif(desc jsonutils.JSONObject) error if err != nil { return err } - err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, consts.NETWORK_TYPE_PXE) + err = h.baremetalInstance.InitAdminNetif(h.ClientMac, h.netConfig, types.NIC_TYPE_ADMIN, api.NETWORK_TYPE_PXE) return err } diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index 23cd42e3f6..81a458a1e2 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -308,7 +308,7 @@ func _parseRange(str string) (ret []int64, err error) { return } -// return bytes +// return KB func parseStrip(stripStr string, defaultSize string) int64 { size, _ := utils.GetSize(stripStr, defaultSize, 1024) return size / 1024 diff --git a/pkg/compute/consts/consts.go b/pkg/compute/consts/consts.go deleted file mode 100644 index d709a2be51..0000000000 --- a/pkg/compute/consts/consts.go +++ /dev/null @@ -1 +0,0 @@ -package consts diff --git a/pkg/compute/consts/doc.go b/pkg/compute/consts/doc.go deleted file mode 100644 index 08003c88c8..0000000000 --- a/pkg/compute/consts/doc.go +++ /dev/null @@ -1 +0,0 @@ -package consts // import "yunion.io/x/onecloud/pkg/compute/consts" diff --git a/pkg/compute/consts/hosttypes.go b/pkg/compute/consts/hosttypes.go deleted file mode 100644 index 8aa2ff88ae..0000000000 --- a/pkg/compute/consts/hosttypes.go +++ /dev/null @@ -1,18 +0,0 @@ -package consts - -const ( - HOST_TYPE_BAREMETAL = "baremetal" - HOST_TYPE_HYPERVISOR = "hypervisor" // KVM - HOST_TYPE_KVM = "kvm" - HOST_TYPE_ESXI = "esxi" // # VMWare vSphere ESXi - HOST_TYPE_KUBELET = "kubelet" // # Kubernetes Kubelet - HOST_TYPE_HYPERV = "hyperv" // # Microsoft Hyper-V - HOST_TYPE_XEN = "xen" // # XenServer - - HOST_TYPE_ALIYUN = "aliyun" - HOST_TYPE_AWS = "aws" - HOST_TYPE_QCLOUD = "qcloud" - HOST_TYPE_AZURE = "azure" - HOST_TYPE_HUAWEI = "huawei" - HOST_TYPE_OPENSTACK = "openstack" -) diff --git a/pkg/compute/consts/networks.go b/pkg/compute/consts/networks.go deleted file mode 100644 index f51058c06a..0000000000 --- a/pkg/compute/consts/networks.go +++ /dev/null @@ -1,9 +0,0 @@ -package consts - -const ( - NETWORK_TYPE_GUEST = "guest" - NETWORK_TYPE_BAREMETAL = "baremetal" - NETWORK_TYPE_CONTAINER = "container" - NETWORK_TYPE_PXE = "pxe" - NETWORK_TYPE_IPMI = "ipmi" -) diff --git a/pkg/compute/consts/storages.go b/pkg/compute/consts/storages.go deleted file mode 100644 index b3a158be4a..0000000000 --- a/pkg/compute/consts/storages.go +++ /dev/null @@ -1,25 +0,0 @@ -package consts - -// TODO: move models/storages.go storage types to this file -const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - STORAGE_NFS = "nfs" - - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" -) - -var Local = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS} - -const ( - STORAGE_ENABLED = "enabled" - // STORAGE_DISABLED = "disabled" - STORAGE_OFFLINE = "offline" - STORAGE_ONLINE = "online" -) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index fd50ef79ce..4fae389600 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -781,6 +781,7 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To } func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + // TODO: 定义 api.ServerCreateInput 的 Unmarshal 函数，直接通过 data.Unmarshal(input) 解析参数 input, err := cmdline.FetchServerCreateInputByJSON(data) if err != nil { return nil, err diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 5f88d90464..53648be750 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -37,67 +38,67 @@ import ( ) const ( - HOST_TYPE_BAREMETAL = "baremetal" - HOST_TYPE_HYPERVISOR = "hypervisor" // KVM - HOST_TYPE_KVM = "kvm" - HOST_TYPE_ESXI = "esxi" // # VMWare vSphere ESXi - HOST_TYPE_KUBELET = "kubelet" // # Kubernetes Kubelet - HOST_TYPE_HYPERV = "hyperv" // # Microsoft Hyper-V - HOST_TYPE_XEN = "xen" // # XenServer + HOST_TYPE_BAREMETAL = api.HOST_TYPE_BAREMETAL + HOST_TYPE_HYPERVISOR = api.HOST_TYPE_HYPERVISOR // KVM + HOST_TYPE_KVM = api.HOST_TYPE_KVM + HOST_TYPE_ESXI = api.HOST_TYPE_ESXI // # VMWare vSphere ESXi + HOST_TYPE_KUBELET = api.HOST_TYPE_KUBELET // # Kubernetes Kubelet + HOST_TYPE_HYPERV = api.HOST_TYPE_HYPERV // # Microsoft Hyper-V + HOST_TYPE_XEN = api.HOST_TYPE_XEN // # XenServer - HOST_TYPE_ALIYUN = "aliyun" - HOST_TYPE_AWS = "aws" - HOST_TYPE_QCLOUD = "qcloud" - HOST_TYPE_AZURE = "azure" - HOST_TYPE_HUAWEI = "huawei" - HOST_TYPE_OPENSTACK = "openstack" + HOST_TYPE_ALIYUN = api.HOST_TYPE_ALIYUN + HOST_TYPE_AWS = api.HOST_TYPE_AWS + HOST_TYPE_QCLOUD = api.HOST_TYPE_QCLOUD + HOST_TYPE_AZURE = api.HOST_TYPE_AZURE + HOST_TYPE_HUAWEI = api.HOST_TYPE_HUAWEI + HOST_TYPE_OPENSTACK = api.HOST_TYPE_OPENSTACK HOST_TYPE_DEFAULT = HOST_TYPE_HYPERVISOR // # possible status - HOST_ONLINE = "online" - HOST_ENABLED = "online" - HOST_OFFLINE = "offline" - HOST_DISABLED = "offline" + HOST_ONLINE = api.HOST_ONLINE + HOST_ENABLED = api.HOST_ENABLED + HOST_OFFLINE = api.HOST_OFFLINE + HOST_DISABLED = api.HOST_DISABLED - NIC_TYPE_IPMI = "ipmi" - NIC_TYPE_ADMIN = "admin" + NIC_TYPE_IPMI = api.NIC_TYPE_IPMI + NIC_TYPE_ADMIN = api.NIC_TYPE_ADMIN // #NIC_TYPE_NORMAL = 'normal' - BAREMETAL_INIT = "init" - BAREMETAL_PREPARE = "prepare" - BAREMETAL_PREPARE_FAIL = "prepare_fail" - BAREMETAL_READY = "ready" - BAREMETAL_RUNNING = "running" - BAREMETAL_MAINTAINING = "maintaining" - BAREMETAL_START_MAINTAIN = "start_maintain" - BAREMETAL_DELETING = "deleting" - BAREMETAL_DELETE = "delete" - BAREMETAL_DELETE_FAIL = "delete_fail" - BAREMETAL_UNKNOWN = "unknown" - BAREMETAL_SYNCING_STATUS = "syncing_status" - BAREMETAL_SYNC = "sync" - BAREMETAL_SYNC_FAIL = "sync_fail" - BAREMETAL_START_CONVERT = "start_convert" - BAREMETAL_CONVERTING = "converting" - BAREMETAL_START_FAIL = "start_fail" - BAREMETAL_STOP_FAIL = "stop_fail" + BAREMETAL_INIT = api.BAREMETAL_INIT + BAREMETAL_PREPARE = api.BAREMETAL_PREPARE + BAREMETAL_PREPARE_FAIL = api.BAREMETAL_PREPARE_FAIL + BAREMETAL_READY = api.BAREMETAL_READY + BAREMETAL_RUNNING = api.BAREMETAL_RUNNING + BAREMETAL_MAINTAINING = api.BAREMETAL_MAINTAINING + BAREMETAL_START_MAINTAIN = api.BAREMETAL_START_MAINTAIN + BAREMETAL_DELETING = api.BAREMETAL_DELETING + BAREMETAL_DELETE = api.BAREMETAL_DELETE + BAREMETAL_DELETE_FAIL = api.BAREMETAL_DELETE_FAIL + BAREMETAL_UNKNOWN = api.BAREMETAL_UNKNOWN + BAREMETAL_SYNCING_STATUS = api.BAREMETAL_SYNCING_STATUS + BAREMETAL_SYNC = api.BAREMETAL_SYNC + BAREMETAL_SYNC_FAIL = api.BAREMETAL_SYNC_FAIL + BAREMETAL_START_CONVERT = api.BAREMETAL_START_CONVERT + BAREMETAL_CONVERTING = api.BAREMETAL_CONVERTING + BAREMETAL_START_FAIL = api.BAREMETAL_START_FAIL + BAREMETAL_STOP_FAIL = api.BAREMETAL_STOP_FAIL - HOST_STATUS_RUNNING = BAREMETAL_RUNNING - HOST_STATUS_READY = BAREMETAL_READY - HOST_STATUS_UNKNOWN = BAREMETAL_UNKNOWN + HOST_STATUS_RUNNING = api.BAREMETAL_RUNNING + HOST_STATUS_READY = api.BAREMETAL_READY + HOST_STATUS_UNKNOWN = api.BAREMETAL_UNKNOWN ) const ( - HostResourceTypeShared = "shared" - HostResourceTypeDefault = HostResourceTypeShared - HostResourceTypePrepaidRecycle = "prepaid" - HostResourceTypeDedicated = "dedicated" + HostResourceTypeShared = api.HostResourceTypeShared + HostResourceTypeDefault = api.HostResourceTypeShared + HostResourceTypePrepaidRecycle = api.HostResourceTypePrepaidRecycle + HostResourceTypeDedicated = api.HostResourceTypeDedicated ) -var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN, HOST_TYPE_AZURE, HOST_TYPE_AWS, HOST_TYPE_QCLOUD, HOST_TYPE_HUAWEI, HOST_TYPE_OPENSTACK} +var HOST_TYPES = api.HOST_TYPES -var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN} +var NIC_TYPES = api.NIC_TYPES type SHostManager struct { db.SEnabledStatusStandaloneResourceBaseManager diff --git a/pkg/compute/models/loadbalancer_common.go b/pkg/compute/models/loadbalancer_common.go index 5e6ff57abb..8fde28387e 100644 --- a/pkg/compute/models/loadbalancer_common.go +++ b/pkg/compute/models/loadbalancer_common.go @@ -6,8 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -37,7 +37,7 @@ func (lls SLoadbalancerLogSkipper) skipLog(ctx context.Context, userCred mcclien if !ok { return false } - if val, _ := data.GetString(consts.LBAGENT_QUERY_ORIG_KEY); val != consts.LBAGENT_QUERY_ORIG_VAL { + if val, _ := data.GetString(api.LBAGENT_QUERY_ORIG_KEY); val != api.LBAGENT_QUERY_ORIG_VAL { return false } return true diff --git a/pkg/compute/models/loadbalanceracls.go b/pkg/compute/models/loadbalanceracls.go index bb4139acdf..cf04be0439 100644 --- a/pkg/compute/models/loadbalanceracls.go +++ b/pkg/compute/models/loadbalanceracls.go @@ -14,12 +14,12 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -156,7 +156,7 @@ func (lbacl *SLoadbalancerAcl) ValidateUpdateData(ctx context.Context, userCred func (lbacl *SLoadbalancerAcl) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { lbacl.SSharableVirtualResourceBase.PostUpdate(ctx, userCred, query, data) - lbacl.SetStatus(userCred, consts.LB_SYNC_CONF, "") + lbacl.SetStatus(userCred, api.LB_SYNC_CONF, "") lbacl.StartLoadBalancerAclSyncTask(ctx, userCred, "") } @@ -172,7 +172,7 @@ func (lbacl *SLoadbalancerAcl) StartLoadBalancerAclSyncTask(ctx context.Context, func (lbacl *SLoadbalancerAcl) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { lbacl.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - lbacl.SetStatus(userCred, consts.LB_CREATING, "") + lbacl.SetStatus(userCred, api.LB_CREATING, "") if err := lbacl.StartLoadBalancerAclCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalanceracl error: %v", err) } @@ -294,7 +294,7 @@ func (lbacl *SLoadbalancerAcl) PerformPurge(ctx context.Context, userCred mcclie } func (lbacl *SLoadbalancerAcl) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lbacl.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lbacl.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lbacl.StartLoadBalancerAclDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -381,7 +381,7 @@ func (self *SLoadbalancerAcl) syncRemoveCloudLoadbalanceAcl(ctx context.Context, err := self.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = self.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = self.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = self.Delete(ctx, userCred) } diff --git a/pkg/compute/models/loadbalanceragents.go b/pkg/compute/models/loadbalanceragents.go index ea95bc8612..d92f096dcd 100644 --- a/pkg/compute/models/loadbalanceragents.go +++ b/pkg/compute/models/loadbalanceragents.go @@ -12,9 +12,9 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/gotypes" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -426,7 +426,7 @@ func (lbagent *SLoadbalancerAgent) AllowPerformHb(ctx context.Context, userCred func (lbagent *SLoadbalancerAgent) PerformHb(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { ipV := validators.NewIPv4AddrValidator("ip") - haStateV := validators.NewStringChoicesValidator("ha_state", consts.LB_HA_STATES) + haStateV := validators.NewStringChoicesValidator("ha_state", api.LB_HA_STATES) { keyV := map[string]validators.IValidator{ "ip": ipV, diff --git a/pkg/compute/models/loadbalancerbackendgroups.go b/pkg/compute/models/loadbalancerbackendgroups.go index 4cb282f255..718a35a35e 100644 --- a/pkg/compute/models/loadbalancerbackendgroups.go +++ b/pkg/compute/models/loadbalancerbackendgroups.go @@ -10,12 +10,12 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -90,7 +90,7 @@ func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Cont } for i := 0; i < len(backends); i++ { if len(backends[i].BackendType) == 0 { - backends[i].BackendType = consts.LB_BACKEND_GUEST + backends[i].BackendType = api.LB_BACKEND_GUEST } if backends[i].Weight < 0 || backends[i].Weight > 256 { return nil, httperrors.NewInputParameterError("weight %s not support, only support range 0 ~ 256") @@ -103,7 +103,7 @@ func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Cont } switch backends[i].BackendType { - case consts.LB_BACKEND_GUEST: + case api.LB_BACKEND_GUEST: _guest, err := GuestManager.FetchByIdOrName(userCred, backends[i].ID) if err != nil { if err == sql.ErrNoRows { @@ -127,7 +127,7 @@ func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Cont return nil, err } backends[i].Address = address - case consts.LB_BACKEND_HOST: + case api.LB_BACKEND_HOST: if !db.IsAdminAllowCreate(userCred, man) { return nil, httperrors.NewForbiddenError("only sysadmin can specify host as backend") } @@ -263,7 +263,7 @@ func (lbbg *SLoadbalancerBackendGroup) PostCreate(ctx context.Context, userCred if backends != nil { params.Add(backends, "backends") } - lbbg.SetStatus(userCred, consts.LB_CREATING, "") + lbbg.SetStatus(userCred, api.LB_CREATING, "") if err := lbbg.StartLoadBalancerBackendGroupCreateTask(ctx, userCred, params, ""); err != nil { log.Errorf("Failed to create loadbalancer backendgroup error: %v", err) } @@ -300,7 +300,7 @@ func (lbbg *SLoadbalancerBackendGroup) PerformPurge(ctx context.Context, userCre } func (lbbg *SLoadbalancerBackendGroup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lbbg.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lbbg.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lbbg.StartLoadBalancerBackendGroupDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -403,7 +403,7 @@ func (lbbg *SLoadbalancerBackendGroup) syncRemoveCloudLoadbalancerBackendgroup(c err := lbbg.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lbbg.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lbbg.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lbbg.Delete(ctx, userCred) } @@ -489,7 +489,7 @@ func (man *SLoadbalancerBackendGroupManager) initBackendGroupType() error { } for i := 0; i < len(backendgroups); i++ { _, err := db.Update(&backendgroups[i], func() error { - backendgroups[i].Type = consts.LB_BACKENDGROUP_TYPE_NORMAL + backendgroups[i].Type = api.LB_BACKENDGROUP_TYPE_NORMAL return nil }) if err != nil { diff --git a/pkg/compute/models/loadbalancerbackends.go b/pkg/compute/models/loadbalancerbackends.go index 1b405242fe..e12a81eef1 100644 --- a/pkg/compute/models/loadbalancerbackends.go +++ b/pkg/compute/models/loadbalancerbackends.go @@ -9,12 +9,12 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -78,7 +78,7 @@ func (man *SLoadbalancerBackendManager) ListItemFilter(ctx context.Context, q *s func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) - backendTypeV := validators.NewStringChoicesValidator("backend_type", consts.LB_BACKEND_TYPES) + backendTypeV := validators.NewStringChoicesValidator("backend_type", api.LB_BACKEND_TYPES) keyV := map[string]validators.IValidator{ "backend_group": backendGroupV, "backend_type": backendTypeV, @@ -98,7 +98,7 @@ func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, var baseName string var backendV *validators.ValidatorModelIdOrName switch backendType { - case consts.LB_BACKEND_GUEST: + case api.LB_BACKEND_GUEST: backendV = validators.NewModelIdOrNameValidator("backend", "server", ownerProjId) err := backendV.Validate(data) if err != nil { @@ -106,7 +106,7 @@ func (man *SLoadbalancerBackendManager) ValidateCreateData(ctx context.Context, } guest := backendV.Model.(*SGuest) baseName = guest.Name - case consts.LB_BACKEND_HOST: + case api.LB_BACKEND_HOST: if !db.IsAdminAllowCreate(userCred, man) { return nil, fmt.Errorf("only sysadmin can specify host as backend") } @@ -208,7 +208,7 @@ func (lbb *SLoadbalancerBackend) ValidateUpdateData(ctx context.Context, userCre func (lbb *SLoadbalancerBackend) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { lbb.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - lbb.SetStatus(userCred, consts.LB_CREATING, "") + lbb.SetStatus(userCred, api.LB_CREATING, "") if err := lbb.StartLoadBalancerBackendCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalancer backend error: %v", err) } @@ -238,7 +238,7 @@ func (lbb *SLoadbalancerBackend) PerformPurge(ctx context.Context, userCred mccl } func (lbb *SLoadbalancerBackend) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lbb.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lbb.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lbb.StartLoadBalancerBackendDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -357,7 +357,7 @@ func (lbb *SLoadbalancerBackend) syncRemoveCloudLoadbalancerBackend(ctx context. err := lbb.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lbb.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lbb.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lbb.MarkPendingDelete(userCred) } diff --git a/pkg/compute/models/loadbalancercertificates.go b/pkg/compute/models/loadbalancercertificates.go index 760c764742..166497ab12 100644 --- a/pkg/compute/models/loadbalancercertificates.go +++ b/pkg/compute/models/loadbalancercertificates.go @@ -15,12 +15,12 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -93,15 +93,15 @@ func (man *SLoadbalancerCertificateManager) validateCertKey(ctx context.Context, // x509.PublicKeyAlgorithm.String() is only available since go1.10 switch cert.PublicKeyAlgorithm { case x509.RSA: - certPubKeyAlgo = consts.LB_TLS_CERT_PUBKEY_ALGO_RSA + certPubKeyAlgo = api.LB_TLS_CERT_PUBKEY_ALGO_RSA case x509.ECDSA: - certPubKeyAlgo = consts.LB_TLS_CERT_PUBKEY_ALGO_ECDSA + certPubKeyAlgo = api.LB_TLS_CERT_PUBKEY_ALGO_ECDSA default: certPubKeyAlgo = fmt.Sprintf("algo %#v", cert.PublicKeyAlgorithm) } - if !consts.LB_TLS_CERT_PUBKEY_ALGOS.Has(certPubKeyAlgo) { + if !api.LB_TLS_CERT_PUBKEY_ALGOS.Has(certPubKeyAlgo) { return nil, httperrors.NewInputParameterError("invalid cert pubkey algorithm: %s, want %s", - certPubKeyAlgo, consts.LB_TLS_CERT_PUBKEY_ALGOS.String()) + certPubKeyAlgo, api.LB_TLS_CERT_PUBKEY_ALGOS.String()) } } err := pkeyV.MatchCertificate(cert) @@ -120,7 +120,7 @@ func (man *SLoadbalancerCertificateManager) validateCertKey(ctx context.Context, data.Set("public_key_algorithm", jsonutils.NewString(certPubKeyAlgo)) data.Set("public_key_bit_len", jsonutils.NewInt(int64(certV.PublicKeyBitLen()))) data.Set("signature_algorithm", jsonutils.NewString(cert.SignatureAlgorithm.String())) - data.Set("fingerprint", jsonutils.NewString(consts.LB_TLS_CERT_FINGERPRINT_ALGO_SHA256+":"+certV.FingerprintSha256String())) + data.Set("fingerprint", jsonutils.NewString(api.LB_TLS_CERT_FINGERPRINT_ALGO_SHA256+":"+certV.FingerprintSha256String())) return data, nil } @@ -166,7 +166,7 @@ func (man *SLoadbalancerCertificateManager) InitializeData() error { continue } d := sha256.Sum256(c.Raw) - fp = consts.LB_TLS_CERT_FINGERPRINT_ALGO_SHA256 + ":" + hex.EncodeToString(d[:]) + fp = api.LB_TLS_CERT_FINGERPRINT_ALGO_SHA256 + ":" + hex.EncodeToString(d[:]) } _, err := db.Update(lbcert, func() error { lbcert.Fingerprint = fp @@ -207,7 +207,7 @@ func (lbcert *SLoadbalancerCertificate) ValidateUpdateData(ctx context.Context, func (lbcert *SLoadbalancerCertificate) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { lbcert.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - lbcert.SetStatus(userCred, consts.LB_CREATING, "") + lbcert.SetStatus(userCred, api.LB_CREATING, "") if err := lbcert.StartLoadBalancerCertificateCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalancercertificate error: %v", err) } @@ -278,7 +278,7 @@ func (lbcert *SLoadbalancerCertificate) PerformPurge(ctx context.Context, userCr } func (lbcert *SLoadbalancerCertificate) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lbcert.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lbcert.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lbcert.StartLoadBalancerCertificateDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -388,7 +388,7 @@ func (lbcert *SLoadbalancerCertificate) syncRemoveCloudLoadbalancerCertificate(c err := lbcert.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lbcert.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lbcert.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lbcert.Delete(ctx, userCred) } diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go index 241811c329..fc2189bcfc 100644 --- a/pkg/compute/models/loadbalancerlistenerrules.go +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -9,12 +9,12 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -97,7 +97,7 @@ func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Cont domainV := validators.NewDomainNameValidator("domain") pathV := validators.NewURLPathValidator("path") keyV := map[string]validators.IValidator{ - "status": validators.NewStringChoicesValidator("status", consts.LB_STATUS_SPEC).Default(consts.LB_STATUS_ENABLED), + "status": validators.NewStringChoicesValidator("status", api.LB_STATUS_SPEC).Default(api.LB_STATUS_ENABLED), "listener": listenerV, "backend_group": backendGroupV, @@ -116,7 +116,7 @@ func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Cont data.Set("cloudregion_id", jsonutils.NewString(listener.CloudregionId)) data.Set("manager_id", jsonutils.NewString(listener.ManagerId)) listenerType := listener.ListenerType - if listenerType != consts.LB_LISTENER_TYPE_HTTP && listenerType != consts.LB_LISTENER_TYPE_HTTPS { + if listenerType != api.LB_LISTENER_TYPE_HTTP && listenerType != api.LB_LISTENER_TYPE_HTTPS { return nil, fmt.Errorf("listener type must be http/https, got %s", listenerType) } { @@ -151,7 +151,7 @@ func (man *SLoadbalancerListenerRuleManager) ValidateCreateData(ctx context.Cont func (lbr *SLoadbalancerListenerRule) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { lbr.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - lbr.SetStatus(userCred, consts.LB_CREATING, "") + lbr.SetStatus(userCred, api.LB_CREATING, "") if err := lbr.StartLoadBalancerListenerRuleCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalancer listener rule error: %v", err) } @@ -177,7 +177,7 @@ func (lbr *SLoadbalancerListenerRule) PerformPurge(ctx context.Context, userCred } func (lbr *SLoadbalancerListenerRule) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lbr.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lbr.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lbr.StartLoadBalancerListenerRuleDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -392,7 +392,7 @@ func (lbr *SLoadbalancerListenerRule) syncRemoveCloudLoadbalancerListenerRule(ct err := lbr.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lbr.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lbr.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lbr.Delete(ctx, userCred) } diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index b6b89ac72a..547af9392d 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -11,12 +11,12 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -124,10 +124,10 @@ func (man *SLoadbalancerListenerManager) checkListenerUniqueness(ctx context.Con Equals("loadbalancer_id", lb.Id). Equals("listener_port", listenerPort) switch listenerType { - case consts.LB_LISTENER_TYPE_TCP, consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS: - q = q.NotEquals("listener_type", consts.LB_LISTENER_TYPE_UDP) - case consts.LB_LISTENER_TYPE_UDP: - q = q.Equals("listener_type", consts.LB_LISTENER_TYPE_UDP) + case api.LB_LISTENER_TYPE_TCP, api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS: + q = q.NotEquals("listener_type", api.LB_LISTENER_TYPE_UDP) + case api.LB_LISTENER_TYPE_UDP: + q = q.Equals("listener_type", api.LB_LISTENER_TYPE_UDP) default: return fmt.Errorf("unexpected listener type: %s", listenerType) } @@ -168,25 +168,25 @@ func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q * func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", ownerProjId) - listenerTypeV := validators.NewStringChoicesValidator("listener_type", consts.LB_LISTENER_TYPES) + listenerTypeV := validators.NewStringChoicesValidator("listener_type", api.LB_LISTENER_TYPES) listenerPortV := validators.NewPortValidator("listener_port") backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) - aclStatusV := validators.NewStringChoicesValidator("acl_status", consts.LB_BOOL_VALUES) - aclTypeV := validators.NewStringChoicesValidator("acl_type", consts.LB_ACL_TYPES) + aclStatusV := validators.NewStringChoicesValidator("acl_status", api.LB_BOOL_VALUES) + aclTypeV := validators.NewStringChoicesValidator("acl_type", api.LB_ACL_TYPES) aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId) keyV := map[string]validators.IValidator{ - "status": validators.NewStringChoicesValidator("status", consts.LB_STATUS_SPEC).Default(consts.LB_STATUS_ENABLED), + "status": validators.NewStringChoicesValidator("status", api.LB_STATUS_SPEC).Default(api.LB_STATUS_ENABLED), "loadbalancer": lbV, "listener_type": listenerTypeV, "listener_port": listenerPortV, "backend_group": backendGroupV.Optional(true), - "acl_status": aclStatusV.Default(consts.LB_BOOL_OFF), + "acl_status": aclStatusV.Default(api.LB_BOOL_OFF), "acl_type": aclTypeV.Optional(true), "acl": aclV.Optional(true), - "scheduler": validators.NewStringChoicesValidator("scheduler", consts.LB_SCHEDULER_TYPES), + "scheduler": validators.NewStringChoicesValidator("scheduler", api.LB_SCHEDULER_TYPES), "bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000).Optional(true), "client_request_timeout": validators.NewRangeValidator("client_request_timeout", 0, 600).Default(10), @@ -194,8 +194,8 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, "backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180).Default(5), "backend_idle_timeout": validators.NewRangeValidator("backend_idle_timeout", 0, 600).Default(90), - "sticky_session": validators.NewStringChoicesValidator("sticky_session", consts.LB_BOOL_VALUES).Default(consts.LB_BOOL_OFF), - "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", consts.LB_STICKY_SESSION_TYPES).Default(consts.LB_STICKY_SESSION_TYPE_INSERT), + "sticky_session": validators.NewStringChoicesValidator("sticky_session", api.LB_BOOL_VALUES).Default(api.LB_BOOL_OFF), + "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", api.LB_STICKY_SESSION_TYPES).Default(api.LB_STICKY_SESSION_TYPE_INSERT), "sticky_session_cookie": validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)).Optional(true), "sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout").Optional(true), @@ -237,9 +237,9 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, } } { - if listenerType == consts.LB_LISTENER_TYPE_HTTPS { + if listenerType == api.LB_LISTENER_TYPE_HTTPS { certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId) - tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", consts.LB_TLS_CIPHER_POLICIES).Default(consts.LB_TLS_CIPHER_POLICY_1_2) + tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", api.LB_TLS_CIPHER_POLICIES).Default(api.LB_TLS_CIPHER_POLICY_1_2) httpsV := map[string]validators.IValidator{ "certificate": certV, "tls_cipher_policy": tlsCipherPolicyV, @@ -260,12 +260,12 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, // health check default depends on input parameters checkTypeV := man.checkTypeV(listenerType) keyVHealth := map[string]validators.IValidator{ - "health_check": validators.NewStringChoicesValidator("health_check", consts.LB_BOOL_VALUES).Default(consts.LB_BOOL_ON), + "health_check": validators.NewStringChoicesValidator("health_check", api.LB_BOOL_VALUES).Default(api.LB_BOOL_ON), "health_check_type": checkTypeV, "health_check_domain": validators.NewDomainNameValidator("health_check_domain").AllowEmpty(true).Default(""), "health_check_path": validators.NewURLPathValidator("health_check_path").Default(""), - "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", consts.LB_HEALTH_CHECK_HTTP_CODES).Sep(",").Default(consts.LB_HEALTH_CHECK_HTTP_CODE_DEFAULT), + "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", api.LB_HEALTH_CHECK_HTTP_CODES).Sep(",").Default(api.LB_HEALTH_CHECK_HTTP_CODE_DEFAULT), "health_check_rise": validators.NewRangeValidator("health_check_rise", 1, 1000).Default(3), "health_check_fall": validators.NewRangeValidator("health_check_fall", 1, 1000).Default(3), @@ -293,19 +293,19 @@ func (man *SLoadbalancerListenerManager) ValidateCreateData(ctx context.Context, func (man *SLoadbalancerListenerManager) checkTypeV(listenerType string) validators.IValidator { switch listenerType { - case consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS: - return validators.NewStringChoicesValidator("health_check_type", consts.LB_HEALTH_CHECK_TYPES_TCP).Default(consts.LB_HEALTH_CHECK_HTTP) - case consts.LB_LISTENER_TYPE_TCP: - return validators.NewStringChoicesValidator("health_check_type", consts.LB_HEALTH_CHECK_TYPES_TCP).Default(consts.LB_HEALTH_CHECK_TCP) - case consts.LB_LISTENER_TYPE_UDP: - return validators.NewStringChoicesValidator("health_check_type", consts.LB_HEALTH_CHECK_TYPES_UDP).Default(consts.LB_HEALTH_CHECK_UDP) + case api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS: + return validators.NewStringChoicesValidator("health_check_type", api.LB_HEALTH_CHECK_TYPES_TCP).Default(api.LB_HEALTH_CHECK_HTTP) + case api.LB_LISTENER_TYPE_TCP: + return validators.NewStringChoicesValidator("health_check_type", api.LB_HEALTH_CHECK_TYPES_TCP).Default(api.LB_HEALTH_CHECK_TCP) + case api.LB_LISTENER_TYPE_UDP: + return validators.NewStringChoicesValidator("health_check_type", api.LB_HEALTH_CHECK_TYPES_UDP).Default(api.LB_HEALTH_CHECK_UDP) } // should it happen, panic then return nil } func (man *SLoadbalancerListenerManager) validateAcl(aclStatusV *validators.ValidatorStringChoices, aclTypeV *validators.ValidatorStringChoices, aclV *validators.ValidatorModelIdOrName, data *jsonutils.JSONDict) error { - if aclStatusV.Value == consts.LB_BOOL_ON { + if aclStatusV.Value == api.LB_BOOL_ON { if aclV.Model == nil { return httperrors.NewMissingParameterError("acl") } @@ -326,7 +326,7 @@ func (lblis *SLoadbalancerListener) PerformStatus(ctx context.Context, userCred if _, err := lblis.SVirtualResourceBase.PerformStatus(ctx, userCred, query, data); err != nil { return nil, err } - if lblis.Status == consts.LB_STATUS_ENABLED { + if lblis.Status == api.LB_STATUS_ENABLED { return nil, lblis.StartLoadBalancerListenerStartTask(ctx, userCred, "") } return nil, lblis.StartLoadBalancerListenerStopTask(ctx, userCred, "") @@ -356,7 +356,7 @@ func (lblis *SLoadbalancerListener) AllowPerformSyncstatus(ctx context.Context, func (lblis *SLoadbalancerListener) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { params := jsonutils.NewDict() - if utils.IsInStringArray(lblis.Status, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(lblis.Status, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { params.Add(jsonutils.NewString(lblis.Status), "origin_status") } return nil, lblis.StartLoadBalancerListenerSyncstatusTask(ctx, userCred, params, "") @@ -374,15 +374,15 @@ func (lblis *SLoadbalancerListener) StartLoadBalancerListenerSyncstatusTask(ctx func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { ownerProjId := lblis.GetOwnerProjectId() backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", ownerProjId) - aclStatusV := validators.NewStringChoicesValidator("acl_status", consts.LB_BOOL_VALUES) + aclStatusV := validators.NewStringChoicesValidator("acl_status", api.LB_BOOL_VALUES) aclStatusV.Default(lblis.AclStatus) - aclTypeV := validators.NewStringChoicesValidator("acl_type", consts.LB_ACL_TYPES) - if consts.LB_ACL_TYPES.Has(lblis.AclType) { + aclTypeV := validators.NewStringChoicesValidator("acl_type", api.LB_ACL_TYPES) + if api.LB_ACL_TYPES.Has(lblis.AclType) { aclTypeV.Default(lblis.AclType) } aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", ownerProjId) certV := validators.NewModelIdOrNameValidator("certificate", "loadbalancercertificate", ownerProjId) - tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", consts.LB_TLS_CIPHER_POLICIES).Default(consts.LB_TLS_CIPHER_POLICY_1_2) + tlsCipherPolicyV := validators.NewStringChoicesValidator("tls_cipher_policy", api.LB_TLS_CIPHER_POLICIES).Default(api.LB_TLS_CIPHER_POLICY_1_2) keyV := map[string]validators.IValidator{ "backend_group": backendGroupV, @@ -390,7 +390,7 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user "acl_type": aclTypeV, "acl": aclV, - "scheduler": validators.NewStringChoicesValidator("scheduler", consts.LB_SCHEDULER_TYPES), + "scheduler": validators.NewStringChoicesValidator("scheduler", api.LB_SCHEDULER_TYPES), "bandwidth": validators.NewRangeValidator("bandwidth", 0, 10000), "client_request_timeout": validators.NewRangeValidator("client_request_timeout", 0, 600), @@ -398,17 +398,17 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user "backend_connect_timeout": validators.NewRangeValidator("backend_connect_timeout", 0, 180), "backend_idle_timeout": validators.NewRangeValidator("backend_idle_timeout", 0, 600), - "sticky_session": validators.NewStringChoicesValidator("sticky_session", consts.LB_BOOL_VALUES), - "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", consts.LB_STICKY_SESSION_TYPES), + "sticky_session": validators.NewStringChoicesValidator("sticky_session", api.LB_BOOL_VALUES), + "sticky_session_type": validators.NewStringChoicesValidator("sticky_session_type", api.LB_STICKY_SESSION_TYPES), "sticky_session_cookie": validators.NewRegexpValidator("sticky_session_cookie", regexp.MustCompile(`\w+`)), "sticky_session_cookie_timeout": validators.NewNonNegativeValidator("sticky_session_cookie_timeout"), - "health_check": validators.NewStringChoicesValidator("health_check", consts.LB_BOOL_VALUES), + "health_check": validators.NewStringChoicesValidator("health_check", api.LB_BOOL_VALUES), "health_check_type": LoadbalancerListenerManager.checkTypeV(lblis.ListenerType), "health_check_domain": validators.NewDomainNameValidator("health_check_domain").AllowEmpty(true), "health_check_path": validators.NewURLPathValidator("health_check_path"), - "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", consts.LB_HEALTH_CHECK_HTTP_CODES).Sep(","), + "health_check_http_code": validators.NewStringMultiChoicesValidator("health_check_http_code", api.LB_HEALTH_CHECK_HTTP_CODES).Sep(","), "health_check_rise": validators.NewRangeValidator("health_check_rise", 1, 1000), "health_check_fall": validators.NewRangeValidator("health_check_fall", 1, 1000), @@ -459,10 +459,10 @@ func (lblis *SLoadbalancerListener) PostUpdate(ctx context.Context, userCred mcc func (lblis *SLoadbalancerListener) StartLoadBalancerListenerSyncTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { params := jsonutils.NewDict() - if utils.IsInStringArray(lblis.Status, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(lblis.Status, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { params.Add(jsonutils.NewString(lblis.Status), "origin_status") } - lblis.SetStatus(userCred, consts.LB_SYNC_CONF, "") + lblis.SetStatus(userCred, api.LB_SYNC_CONF, "") task, err := taskman.TaskManager.NewTask(ctx, "LoadbalancerListenerSyncTask", lblis, userCred, params, parentTaskId, "", nil) if err != nil { return err @@ -505,7 +505,7 @@ func (lblis *SLoadbalancerListener) GetExtraDetails(ctx context.Context, userCre func (lblis *SLoadbalancerListener) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { lblis.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - lblis.SetStatus(userCred, consts.LB_CREATING, "") + lblis.SetStatus(userCred, api.LB_CREATING, "") if err := lblis.StartLoadBalancerListenerCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalancer listener error: %v", err) } @@ -548,7 +548,7 @@ func (lblis *SLoadbalancerListener) StartLoadBalancerListenerDeleteTask(ctx cont } func (lblis *SLoadbalancerListener) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lblis.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lblis.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lblis.StartLoadBalancerListenerDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -603,7 +603,7 @@ func (lblis *SLoadbalancerListener) GetLoadbalancerListenerParams() (*cloudprovi listener.AccessControlListType = lblis.AclType listener.AccessControlListStatus = lblis.AclStatus } - if certificate := lblis.GetLoadbalancerCertificate(); certificate != nil && lblis.ListenerType == consts.LB_LISTENER_TYPE_HTTPS { + if certificate := lblis.GetLoadbalancerCertificate(); certificate != nil && lblis.ListenerType == api.LB_LISTENER_TYPE_HTTPS { listener.CertificateID = certificate.ExternalId } @@ -756,7 +756,7 @@ func (lblis *SLoadbalancerListener) constructFieldsFromCloudListener(userCred mc lblis.BackendServerPort = extListener.GetBackendServerPort() switch lblis.ListenerType { - case consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTPS: lblis.TLSCipherPolicy = extListener.GetTLSCipherPolicy() lblis.EnableHttp2 = extListener.HTTP2Enabled() if certificateId := extListener.GetCertificateId(); len(certificateId) > 0 { @@ -765,7 +765,7 @@ func (lblis *SLoadbalancerListener) constructFieldsFromCloudListener(userCred mc } } fallthrough - case consts.LB_LISTENER_TYPE_HTTP: + case api.LB_LISTENER_TYPE_HTTP: lblis.StickySession = extListener.GetStickySession() lblis.StickySessionType = extListener.GetStickySessionType() lblis.StickySessionCookie = extListener.GetStickySessionCookie() @@ -799,7 +799,7 @@ func (lblis *SLoadbalancerListener) syncRemoveCloudLoadbalancerListener(ctx cont err := lblis.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lblis.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lblis.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lblis.Delete(ctx, userCred) } diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 515060b59a..edfcdb0e7e 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -10,12 +10,12 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -96,20 +96,20 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre addressType, _ := data.GetString("address_type") zoneV := validators.NewModelIdOrNameValidator("zone", "zone", "") managerIdV := validators.NewModelIdOrNameValidator("manager_id", "cloudprovider", "") - if addressType == consts.LB_ADDR_TYPE_INTERNET { + if addressType == api.LB_ADDR_TYPE_INTERNET { networkV.Optional(true) } else { zoneV.Optional(true) managerIdV.Optional(true) } addressV := validators.NewIPv4AddrValidator("address") - addressTypeV := validators.NewStringChoicesValidator("address_type", consts.LB_ADDR_TYPES) + addressTypeV := validators.NewStringChoicesValidator("address_type", api.LB_ADDR_TYPES) { keyV := map[string]validators.IValidator{ - "status": validators.NewStringChoicesValidator("status", consts.LB_STATUS_SPEC).Default(consts.LB_STATUS_ENABLED), + "status": validators.NewStringChoicesValidator("status", api.LB_STATUS_SPEC).Default(api.LB_STATUS_ENABLED), "address": addressV.Optional(true), - "address_type": addressTypeV.Default(consts.LB_ADDR_TYPE_INTRANET), + "address_type": addressTypeV.Default(api.LB_ADDR_TYPE_INTRANET), "network": networkV, "zone": zoneV, "manager_id": managerIdV, @@ -121,7 +121,7 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre } } var region *SCloudregion - if addressTypeV.Value == consts.LB_ADDR_TYPE_INTRANET { + if addressTypeV.Value == api.LB_ADDR_TYPE_INTRANET { network := networkV.Model.(*SNetwork) if ipAddr := addressV.IP; ipAddr != nil { ipS := ipAddr.String() @@ -164,8 +164,8 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre } data.Set("cloudregion_id", jsonutils.NewString(region.GetId())) // TODO validate network is of classic type - data.Set("network_type", jsonutils.NewString(consts.LB_NETWORK_TYPE_CLASSIC)) - data.Set("address_type", jsonutils.NewString(consts.LB_ADDR_TYPE_INTRANET)) + data.Set("network_type", jsonutils.NewString(api.LB_NETWORK_TYPE_CLASSIC)) + data.Set("address_type", jsonutils.NewString(api.LB_ADDR_TYPE_INTRANET)) } else { zone := zoneV.Model.(*SZone) region = zone.GetRegion() @@ -177,8 +177,8 @@ func (man *SLoadbalancerManager) ValidateCreateData(ctx context.Context, userCre data.Set("address", jsonutils.NewString("")) data.Set("network_id", jsonutils.NewString("")) data.Set("cloudregion_id", jsonutils.NewString(region.GetId())) - data.Set("network_type", jsonutils.NewString(consts.LB_NETWORK_TYPE_VPC)) - data.Set("address_type", jsonutils.NewString(consts.LB_ADDR_TYPE_INTERNET)) + data.Set("network_type", jsonutils.NewString(api.LB_NETWORK_TYPE_VPC)) + data.Set("address_type", jsonutils.NewString(api.LB_ADDR_TYPE_INTERNET)) } if _, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil { return nil, err @@ -194,7 +194,7 @@ func (lb *SLoadbalancer) PerformStatus(ctx context.Context, userCred mcclient.To if _, err := lb.SVirtualResourceBase.PerformStatus(ctx, userCred, query, data); err != nil { return nil, err } - if lb.Status == consts.LB_STATUS_ENABLED { + if lb.Status == api.LB_STATUS_ENABLED { return nil, lb.StartLoadBalancerStartTask(ctx, userCred, "") } return nil, lb.StartLoadBalancerStopTask(ctx, userCred, "") @@ -229,7 +229,7 @@ func (lb *SLoadbalancer) PerformSyncstatus(ctx context.Context, userCred mcclien func (lb *SLoadbalancer) StartLoadBalancerSyncstatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(lb.Status), "origin_status") - lb.SetStatus(userCred, consts.LB_SYNC_STATUS, "") + lb.SetStatus(userCred, api.LB_SYNC_STATUS, "") task, err := taskman.TaskManager.NewTask(ctx, "LoadbalancerSyncstatusTask", lb, userCred, params, parentTaskId, "", nil) if err != nil { return err @@ -244,7 +244,7 @@ func (lb *SLoadbalancer) PostCreate(ctx context.Context, userCred mcclient.Token // NOTE this means lb.UpdateVersion will be 0, then 1 after creation // NOTE need ways to notify error - lb.SetStatus(userCred, consts.LB_CREATING, "") + lb.SetStatus(userCred, api.LB_CREATING, "") if err := lb.StartLoadBalancerCreateTask(ctx, userCred, ""); err != nil { log.Errorf("Failed to create loadbalancer error: %v", err) } @@ -326,7 +326,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(iRegion cloudprovider.IClou } params.ZoneID = iZone.GetId() } - if lb.AddressType == consts.LB_ADDR_TYPE_INTRANET { + if lb.AddressType == api.LB_ADDR_TYPE_INTRANET { vpc := lb.GetVpc() if vpc == nil { return nil, fmt.Errorf("failed to find vpc for lb %s", lb.Name) @@ -412,7 +412,7 @@ func (lb *SLoadbalancer) GetExtraDetails(ctx context.Context, userCred mcclient. } func (lb *SLoadbalancer) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - lb.SetStatus(userCred, consts.LB_STATUS_DELETING, "") + lb.SetStatus(userCred, api.LB_STATUS_DELETING, "") return lb.StartLoadBalancerDeleteTask(ctx, userCred, jsonutils.NewDict(), "") } @@ -571,7 +571,7 @@ func (lb *SLoadbalancer) syncRemoveCloudLoadbalancer(ctx context.Context, userCr err := lb.ValidateDeleteCondition(ctx) if err != nil { // cannot delete - err = lb.SetStatus(userCred, consts.LB_STATUS_UNKNOWN, "sync to delete") + err = lb.SetStatus(userCred, api.LB_STATUS_UNKNOWN, "sync to delete") } else { err = lb.Delete(ctx, userCred) } diff --git a/pkg/compute/regiondrivers/aliyun.go b/pkg/compute/regiondrivers/aliyun.go index c10043e3e0..84e277beb0 100644 --- a/pkg/compute/regiondrivers/aliyun.go +++ b/pkg/compute/regiondrivers/aliyun.go @@ -7,10 +7,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -46,14 +46,14 @@ func (self *SAliyunRegionDriver) ValidateUpdateLoadbalancerCertificateData(ctx c func (self *SAliyunRegionDriver) ValidateDeleteLoadbalancerBackendCondition(ctx context.Context, lbb *models.SLoadbalancerBackend) error { backendGroup := lbb.GetLoadbalancerBackendGroup() - if backendGroup.Type == consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE { + if backendGroup.Type == api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE { return httperrors.NewUnsupportOperationError("backend %s belong master slave backendgroup, not allow delete", lbb.Name) } return nil } func (self *SAliyunRegionDriver) ValidateDeleteLoadbalancerBackendGroupCondition(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup) error { - if lbbg.Type == consts.LB_BACKENDGROUP_TYPE_DEFAULT { + if lbbg.Type == api.LB_BACKENDGROUP_TYPE_DEFAULT { return httperrors.NewUnsupportOperationError("not allow to delete default backend group") } return nil @@ -62,9 +62,9 @@ func (self *SAliyunRegionDriver) ValidateDeleteLoadbalancerBackendGroupCondition func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerBackendGroupData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, lb *models.SLoadbalancer, backends []cloudprovider.SLoadbalancerBackend) (*jsonutils.JSONDict, error) { groupType, _ := data.GetString("type") switch groupType { - case "", consts.LB_BACKENDGROUP_TYPE_NORMAL: + case "", api.LB_BACKENDGROUP_TYPE_NORMAL: break - case consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: + case api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: if len(backends) != 2 { return nil, httperrors.NewInputParameterError("master slave backendgorup must contain two backend") } @@ -83,10 +83,10 @@ func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerBackendGroupData(ctx } func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerBackendData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, backendType string, lb *models.SLoadbalancer, backendGroup *models.SLoadbalancerBackendGroup, backend db.IModel) (*jsonutils.JSONDict, error) { - if backendType != consts.LB_BACKEND_GUEST { + if backendType != api.LB_BACKEND_GUEST { return nil, httperrors.NewUnsupportOperationError("internal error: unexpected backend type %s", backendType) } - if !utils.IsInStringArray(backendGroup.Type, []string{consts.LB_BACKENDGROUP_TYPE_DEFAULT, consts.LB_BACKENDGROUP_TYPE_NORMAL}) { + if !utils.IsInStringArray(backendGroup.Type, []string{api.LB_BACKENDGROUP_TYPE_DEFAULT, api.LB_BACKENDGROUP_TYPE_NORMAL}) { return nil, httperrors.NewUnsupportOperationError("backendgroup %s not support this operation", backendGroup.Name) } guest := backend.(*models.SGuest) @@ -120,7 +120,7 @@ func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerListenerRuleData(ctx if !ok { return nil, httperrors.NewMissingParameterError("backend_group") } - if backendgroup.Type != consts.LB_BACKENDGROUP_TYPE_NORMAL { + if backendgroup.Type != api.LB_BACKENDGROUP_TYPE_NORMAL { return nil, httperrors.NewInputParameterError("backend group type must be normal") } return data, nil @@ -132,7 +132,7 @@ func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerListenerData(ctx cont return nil, httperrors.NewMissingParameterError("backend_group") } listenerType, _ := data.GetString("listener_type") - if utils.IsInStringArray(listenerType, []string{consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS}) && !utils.IsInStringArray(backendgroup.Type, []string{consts.LB_BACKENDGROUP_TYPE_DEFAULT, consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE}) { + if utils.IsInStringArray(listenerType, []string{api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS}) && !utils.IsInStringArray(backendgroup.Type, []string{api.LB_BACKENDGROUP_TYPE_DEFAULT, api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE}) { return nil, httperrors.NewUnsupportOperationError("http or https listener only supportd default or master_slave backendgroup") } @@ -156,11 +156,11 @@ func (self *SAliyunRegionDriver) ValidateCreateLoadbalancerListenerData(ctx cont "health_check_timeout": validators.NewRangeValidator("health_check_timeout", 1, 300), "health_check_interval": validators.NewRangeValidator("health_check_interval", 1, 50), } - if !utils.IsInStringArray(listenerType, []string{consts.LB_LISTENER_TYPE_UDP, consts.LB_LISTENER_TYPE_TCP}) { + if !utils.IsInStringArray(listenerType, []string{api.LB_LISTENER_TYPE_UDP, api.LB_LISTENER_TYPE_TCP}) { keyV["client_idle_timeout"] = validators.NewRangeValidator("client_idle_timeout", 1, 60) } - if backendgroup.Type == consts.LB_BACKENDGROUP_TYPE_DEFAULT { + if backendgroup.Type == api.LB_BACKENDGROUP_TYPE_DEFAULT { keyV["backend_server_port"] = validators.NewPortValidator("backend_server_port") } @@ -194,11 +194,11 @@ func (self *SAliyunRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx cont backendgroup, ok := backendGroup.(*models.SLoadbalancerBackendGroup) if ok { - if utils.IsInStringArray(listenerType, []string{consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS}) && !utils.IsInStringArray(backendgroup.Type, []string{consts.LB_BACKENDGROUP_TYPE_DEFAULT, consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE}) { + if utils.IsInStringArray(listenerType, []string{api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS}) && !utils.IsInStringArray(backendgroup.Type, []string{api.LB_BACKENDGROUP_TYPE_DEFAULT, api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE}) { return nil, httperrors.NewUnsupportOperationError("http or https listener only supportd default or master_slave backendgroup") } - if backendgroup.Type == consts.LB_BACKENDGROUP_TYPE_DEFAULT { + if backendgroup.Type == api.LB_BACKENDGROUP_TYPE_DEFAULT { keyV["backend_server_port"] = validators.NewPortValidator("backend_server_port") } @@ -208,7 +208,7 @@ func (self *SAliyunRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx cont } } - if !utils.IsInStringArray(listenerType, []string{consts.LB_LISTENER_TYPE_UDP, consts.LB_LISTENER_TYPE_TCP}) { + if !utils.IsInStringArray(listenerType, []string{api.LB_LISTENER_TYPE_UDP, api.LB_LISTENER_TYPE_TCP}) { keyV["client_idle_timeout"] = validators.NewRangeValidator("client_idle_timeout", 1, 60) } diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index e2b95a6cfc..69371f32ba 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -8,10 +8,10 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -48,7 +48,7 @@ func (self *SKVMRegionDriver) ValidateUpdateLoadbalancerCertificateData(ctx cont func (self *SKVMRegionDriver) ValidateCreateLoadbalancerBackendGroupData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, lb *models.SLoadbalancer, backends []cloudprovider.SLoadbalancerBackend) (*jsonutils.JSONDict, error) { for _, backend := range backends { switch backend.BackendType { - case consts.LB_BACKEND_GUEST: + case api.LB_BACKEND_GUEST: if backend.ZoneId != lb.ZoneId { return nil, fmt.Errorf("zone of host %q (%s) != zone of loadbalancer %q (%s)", backend.HostName, backend.ZoneId, lb.Name, lb.ZoneId) @@ -60,7 +60,7 @@ func (self *SKVMRegionDriver) ValidateCreateLoadbalancerBackendGroupData(ctx con func (self *SKVMRegionDriver) ValidateCreateLoadbalancerBackendData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, backendType string, lb *models.SLoadbalancer, backendGroup *models.SLoadbalancerBackendGroup, backend db.IModel) (*jsonutils.JSONDict, error) { switch backendType { - case consts.LB_BACKEND_GUEST: + case api.LB_BACKEND_GUEST: guest := backend.(*models.SGuest) { // guest zone must match that of loadbalancer's @@ -106,7 +106,7 @@ func (self *SKVMRegionDriver) ValidateUpdateLoadbalancerListenerData(ctx context func (self *SKVMRegionDriver) RequestCreateLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, lb *models.SLoadbalancer, task taskman.ITask) error { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { _, err := db.Update(lb, func() error { - if lb.AddressType == consts.LB_ADDR_TYPE_INTRANET { + if lb.AddressType == api.LB_ADDR_TYPE_INTRANET { // TODO support use reserved ip address // TODO prefer ip address from server_type loadbalancer? req := &models.SLoadbalancerNetworkRequestData{ @@ -142,10 +142,10 @@ func (self *SKVMRegionDriver) RequestStopLoadbalancer(ctx context.Context, userC func (self *SKVMRegionDriver) RequestSyncstatusLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, lb *models.SLoadbalancer, task taskman.ITask) error { originStatus, _ := task.GetParams().GetString("origin_status") - if utils.IsInStringArray(originStatus, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(originStatus, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { lb.SetStatus(userCred, originStatus, "") } else { - lb.SetStatus(userCred, consts.LB_STATUS_ENABLED, "") + lb.SetStatus(userCred, api.LB_STATUS_ENABLED, "") } task.ScheduleRun(nil) return nil @@ -193,7 +193,7 @@ func (self *SKVMRegionDriver) RequestCreateLoadbalancerBackendGroup(ctx context. Address: backend.Address, Port: backend.Port, } - loadbalancerBackend.Status = consts.LB_STATUS_ENABLED + loadbalancerBackend.Status = api.LB_STATUS_ENABLED loadbalancerBackend.ProjectId = userCred.GetProjectId() loadbalancerBackend.Name = fmt.Sprintf("%s-%s-%s", lbbg.Name, backend.BackendType, backend.Name) if err := models.LoadbalancerBackendManager.TableSpec().Insert(&loadbalancerBackend); err != nil { @@ -250,10 +250,10 @@ func (self *SKVMRegionDriver) RequestStopLoadbalancerListener(ctx context.Contex func (self *SKVMRegionDriver) RequestSyncstatusLoadbalancerListener(ctx context.Context, userCred mcclient.TokenCredential, lblis *models.SLoadbalancerListener, task taskman.ITask) error { originStatus, _ := task.GetParams().GetString("origin_status") - if utils.IsInStringArray(originStatus, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(originStatus, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { lblis.SetStatus(userCred, originStatus, "") } else { - lblis.SetStatus(userCred, consts.LB_STATUS_ENABLED, "") + lblis.SetStatus(userCred, api.LB_STATUS_ENABLED, "") } task.ScheduleRun(nil) return nil diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index 4ad1d13d26..b7c00d3764 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -8,11 +8,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -55,7 +55,7 @@ func (self *SManagedVirtualizationRegionDriver) ValidateUpdateLoadbalancerCertif } func (self *SManagedVirtualizationRegionDriver) ValidateCreateLoadbalancerBackendData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, backendType string, lb *models.SLoadbalancer, backendGroup *models.SLoadbalancerBackendGroup, backend db.IModel) (*jsonutils.JSONDict, error) { - if backendType != consts.LB_BACKEND_GUEST { + if backendType != api.LB_BACKEND_GUEST { return nil, httperrors.NewUnsupportOperationError("internal error: unexpected backend type %s", backendType) } guest := backend.(*models.SGuest) @@ -186,7 +186,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncstatusLoadbalancer(ct return nil, err } status := iLoadbalancer.GetStatus() - if utils.IsInStringArray(status, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(status, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { return nil, lb.SetStatus(userCred, status, "") } return nil, fmt.Errorf("Unknown loadbalancer status %s", status) @@ -628,7 +628,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncstatusLoadbalancerLis return nil, err } status := iListener.GetStatus() - if utils.IsInStringArray(status, []string{consts.LB_STATUS_ENABLED, consts.LB_STATUS_DISABLED}) { + if utils.IsInStringArray(status, []string{api.LB_STATUS_ENABLED, api.LB_STATUS_DISABLED}) { return nil, lblis.SetStatus(userCred, status, "") } return nil, fmt.Errorf("Unknown loadbalancer listener status %s", status) diff --git a/pkg/compute/tasks/loadbalancer_acl_create_task.go b/pkg/compute/tasks/loadbalancer_acl_create_task.go index 801a483ca1..c299b590a5 100644 --- a/pkg/compute/tasks/loadbalancer_acl_create_task.go +++ b/pkg/compute/tasks/loadbalancer_acl_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerAclCreateTask) taskFail(ctx context.Context, lbacl *models.SLoadbalancerAcl, reason string) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lbacl.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lbacl, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerAclCreateTask) OnInit(ctx context.Context, obj db.IStand } func (self *LoadbalancerAclCreateTask) OnLoadbalancerAclCreateComplete(ctx context.Context, lbacl *models.SLoadbalancerAcl, data jsonutils.JSONObject) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbacl.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbacl, db.ACT_ALLOCATE, lbacl.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_acl_delete_task.go b/pkg/compute/tasks/loadbalancer_acl_delete_task.go index a57cda9645..e2663aae55 100644 --- a/pkg/compute/tasks/loadbalancer_acl_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_acl_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerAclDeleteTask) taskFail(ctx context.Context, lbacl *models.SLoadbalancerAcl, reason string) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lbacl.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lbacl, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_acl_sync_task.go b/pkg/compute/tasks/loadbalancer_acl_sync_task.go index 183feca846..140d19f0b5 100644 --- a/pkg/compute/tasks/loadbalancer_acl_sync_task.go +++ b/pkg/compute/tasks/loadbalancer_acl_sync_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerAclSyncTask) taskFail(ctx context.Context, lbacl *models.SLoadbalancerAcl, reason string) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_SYNC_CONF_FAILED, reason) + lbacl.SetStatus(self.GetUserCred(), api.LB_SYNC_CONF_FAILED, reason) db.OpsLog.LogEvent(lbacl, db.ACT_SYNC_CONF, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_SYNC_CONF, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, consts.LB_SYNC_CONF_FAILED, reason) + notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, api.LB_SYNC_CONF_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerAclSyncTask) OnInit(ctx context.Context, obj db.IStandal } func (self *LoadbalancerAclSyncTask) OnLoadbalancerAclSyncComplete(ctx context.Context, lbacl *models.SLoadbalancerAcl, data jsonutils.JSONObject) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbacl.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbacl, db.ACT_SYNC_CONF, lbacl.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_SYNC_CONF, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_backend_create_task.go b/pkg/compute/tasks/loadbalancer_backend_create_task.go index e96edc6a06..9a4aa6fcf5 100644 --- a/pkg/compute/tasks/loadbalancer_backend_create_task.go +++ b/pkg/compute/tasks/loadbalancer_backend_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerBackendCreateTask) taskFail(ctx context.Context, lbb *models.SLoadbalancerBackend, reason string) { - lbb.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lbb.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lbb, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbb, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbb.Id, lbb.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lbb.Id, lbb.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerBackendCreateTask) OnInit(ctx context.Context, obj db.IS } func (self *LoadbalancerBackendCreateTask) OnLoadbalancerBackendCreateComplete(ctx context.Context, lbb *models.SLoadbalancerBackend, data jsonutils.JSONObject) { - lbb.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbb.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbb, db.ACT_ALLOCATE, lbb.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbb, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_backend_delete_task.go b/pkg/compute/tasks/loadbalancer_backend_delete_task.go index 69ae7e9a77..150eba1157 100644 --- a/pkg/compute/tasks/loadbalancer_backend_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_backend_delete_task.go @@ -6,9 +6,9 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -21,7 +21,7 @@ func init() { } func (self *LoadbalancerBackendDeleteTask) taskFail(ctx context.Context, lbb *models.SLoadbalancerBackend, reason string) { - lbb.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lbb.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_backendgroup_create_task.go b/pkg/compute/tasks/loadbalancer_backendgroup_create_task.go index 96a8fc64ae..d887991e09 100644 --- a/pkg/compute/tasks/loadbalancer_backendgroup_create_task.go +++ b/pkg/compute/tasks/loadbalancer_backendgroup_create_task.go @@ -6,11 +6,11 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -24,10 +24,10 @@ func init() { } func (self *LoadbalancerLoadbalancerBackendGroupCreateTask) taskFail(ctx context.Context, lbacl *models.SLoadbalancerBackendGroup, reason string) { - lbacl.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lbacl.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lbacl, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbacl, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lbacl.Id, lbacl.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -47,7 +47,7 @@ func (self *LoadbalancerLoadbalancerBackendGroupCreateTask) OnInit(ctx context.C } func (self *LoadbalancerLoadbalancerBackendGroupCreateTask) OnLoadbalancerBackendGroupCreateComplete(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup, data jsonutils.JSONObject) { - lbbg.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbbg.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbbg, db.ACT_ALLOCATE, lbbg.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbbg, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_backendgroup_delete_task.go b/pkg/compute/tasks/loadbalancer_backendgroup_delete_task.go index 778096b7a8..49ac262158 100644 --- a/pkg/compute/tasks/loadbalancer_backendgroup_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_backendgroup_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerBackendGroupDeleteTask) taskFail(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup, reason string) { - lbbg.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lbbg.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lbbg, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbbg, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbbg.Id, lbbg.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lbbg.Id, lbbg.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_certificate_create_task.go b/pkg/compute/tasks/loadbalancer_certificate_create_task.go index 30625c447a..60cc762774 100644 --- a/pkg/compute/tasks/loadbalancer_certificate_create_task.go +++ b/pkg/compute/tasks/loadbalancer_certificate_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerCertificateCreateTask) taskFail(ctx context.Context, lbcert *models.SLoadbalancerCertificate, reason string) { - lbcert.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lbcert.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lbcert, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbcert, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbcert.Id, lbcert.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lbcert.Id, lbcert.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerCertificateCreateTask) OnInit(ctx context.Context, obj d } func (self *LoadbalancerCertificateCreateTask) OnLoadbalancerCertificateCreateComplete(ctx context.Context, lbcert *models.SLoadbalancerCertificate, data jsonutils.JSONObject) { - lbcert.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbcert.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbcert, db.ACT_ALLOCATE, lbcert.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbcert, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_certificate_delete_task.go b/pkg/compute/tasks/loadbalancer_certificate_delete_task.go index faabed16d0..d87d9f2866 100644 --- a/pkg/compute/tasks/loadbalancer_certificate_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_certificate_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerCertificateDeleteTask) taskFail(ctx context.Context, lbcert *models.SLoadbalancerCertificate, reason string) { - lbcert.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lbcert.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lbcert, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbcert, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbcert.Id, lbcert.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lbcert.Id, lbcert.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_create_task.go b/pkg/compute/tasks/loadbalancer_create_task.go index b2c9bde73c..cf5af82059 100644 --- a/pkg/compute/tasks/loadbalancer_create_task.go +++ b/pkg/compute/tasks/loadbalancer_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerCreateTask) taskFail(ctx context.Context, lb *models.SLoadbalancer, reason string) { - lb.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lb.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lb, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lb.Id, lb.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lb.Id, lb.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerCreateTask) OnInit(ctx context.Context, obj db.IStandalo } func (self *LoadbalancerCreateTask) OnLoadbalancerCreateComplete(ctx context.Context, lb *models.SLoadbalancer, data jsonutils.JSONObject) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lb, db.ACT_ALLOCATE, lb.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStage("OnLoadbalancerStartComplete", nil) @@ -56,11 +56,11 @@ func (self *LoadbalancerCreateTask) OnLoadbalancerCreateCompleteFailed(ctx conte } func (self *LoadbalancerCreateTask) OnLoadbalancerStartComplete(ctx context.Context, lb *models.SLoadbalancer, data jsonutils.JSONObject) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") self.SetStageComplete(ctx, nil) } func (self *LoadbalancerCreateTask) OnLoadbalancerStartCompleteFailed(ctx context.Context, lb *models.SLoadbalancer, reason jsonutils.JSONObject) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, reason.String()) + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, reason.String()) self.SetStageFailed(ctx, reason.String()) } diff --git a/pkg/compute/tasks/loadbalancer_delete_task.go b/pkg/compute/tasks/loadbalancer_delete_task.go index 1cf7b99ffb..57526bef93 100644 --- a/pkg/compute/tasks/loadbalancer_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerDeleteTask) taskFail(ctx context.Context, lb *models.SLoadbalancer, reason string) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lb, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lb.Id, lb.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lb.Id, lb.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_listener_create_task.go b/pkg/compute/tasks/loadbalancer_listener_create_task.go index 6a80efb37f..dd4d2105df 100644 --- a/pkg/compute/tasks/loadbalancer_listener_create_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerCreateTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lblis, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerListenerCreateTask) OnInit(ctx context.Context, obj db.I } func (self *LoadbalancerListenerCreateTask) OnLoadbalancerListenerCreateComplete(ctx context.Context, lblis *models.SLoadbalancerListener, data jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lblis, db.ACT_ALLOCATE, lblis.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStage("OnLoadbalancerListenerStartComplete", nil) @@ -56,11 +56,11 @@ func (self *LoadbalancerListenerCreateTask) OnLoadbalancerListenerCreateComplete } func (self *LoadbalancerListenerCreateTask) OnLoadbalancerListenerStartComplete(ctx context.Context, lblis *models.SLoadbalancerListener, data jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") self.SetStageComplete(ctx, nil) } func (self *LoadbalancerListenerCreateTask) OnLoadbalancerListenerStartCompleteFailed(ctx context.Context, lblis *models.SLoadbalancerListener, reason jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, reason.String()) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, reason.String()) self.SetStageFailed(ctx, reason.String()) } diff --git a/pkg/compute/tasks/loadbalancer_listener_delete_task.go b/pkg/compute/tasks/loadbalancer_listener_delete_task.go index a93d0d0cf0..3856778d02 100644 --- a/pkg/compute/tasks/loadbalancer_listener_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerDeleteTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lblis, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go b/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go index ea46eadadb..83925968cd 100644 --- a/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerRuleCreateTask) taskFail(ctx context.Context, lbr *models.SLoadbalancerListenerRule, reason string) { - lbr.SetStatus(self.GetUserCred(), consts.LB_CREATE_FAILED, reason) + lbr.SetStatus(self.GetUserCred(), api.LB_CREATE_FAILED, reason) db.OpsLog.LogEvent(lbr, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbr, logclient.ACT_CREATE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbr.Id, lbr.Name, consts.LB_CREATE_FAILED, reason) + notifyclient.NotifySystemError(lbr.Id, lbr.Name, api.LB_CREATE_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerListenerRuleCreateTask) OnInit(ctx context.Context, obj } func (self *LoadbalancerListenerRuleCreateTask) OnLoadbalancerListenerRuleCreateComplete(ctx context.Context, lbr *models.SLoadbalancerListenerRule, data jsonutils.JSONObject) { - lbr.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lbr.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lbr, db.ACT_ALLOCATE, lbr.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lbr, logclient.ACT_CREATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_listener_rule_delete_task.go b/pkg/compute/tasks/loadbalancer_listener_rule_delete_task.go index 21d8dfc0b1..bcef97fd5f 100644 --- a/pkg/compute/tasks/loadbalancer_listener_rule_delete_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_rule_delete_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerRuleDeleteTask) taskFail(ctx context.Context, lbr *models.SLoadbalancerListenerRule, reason string) { - lbr.SetStatus(self.GetUserCred(), consts.LB_STATUS_DELETE_FAILED, reason) + lbr.SetStatus(self.GetUserCred(), api.LB_STATUS_DELETE_FAILED, reason) db.OpsLog.LogEvent(lbr, db.ACT_DELOCATE_FAIL, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lbr, logclient.ACT_DELETE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lbr.Id, lbr.Name, consts.LB_STATUS_DELETE_FAILED, reason) + notifyclient.NotifySystemError(lbr.Id, lbr.Name, api.LB_STATUS_DELETE_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_listener_start_task.go b/pkg/compute/tasks/loadbalancer_listener_start_task.go index c775c90e48..dbb495d558 100644 --- a/pkg/compute/tasks/loadbalancer_listener_start_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_start_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerStartTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, reason) db.OpsLog.LogEvent(lblis, db.ACT_ENABLE, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_ENABLE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_STATUS_DISABLED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_STATUS_DISABLED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerListenerStartTask) OnInit(ctx context.Context, obj db.IS } func (self *LoadbalancerListenerStartTask) OnLoadbalancerListenerStartComplete(ctx context.Context, lblis *models.SLoadbalancerListener, data jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lblis, db.ACT_ENABLE, lblis.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_ENABLE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_listener_stop_task.go b/pkg/compute/tasks/loadbalancer_listener_stop_task.go index c2f6526124..03cde30787 100644 --- a/pkg/compute/tasks/loadbalancer_listener_stop_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_stop_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerStopTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, reason) db.OpsLog.LogEvent(lblis, db.ACT_DISABLE, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_DISABLE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_STATUS_ENABLED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_STATUS_ENABLED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerListenerStopTask) OnInit(ctx context.Context, obj db.ISt } func (self *LoadbalancerListenerStopTask) OnLoadbalancerListenerStopComplete(ctx context.Context, lblis *models.SLoadbalancerListener, data jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, "") + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, "") db.OpsLog.LogEvent(lblis, db.ACT_DISABLE, lblis.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_DISABLE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_listener_sync_task.go b/pkg/compute/tasks/loadbalancer_listener_sync_task.go index ac422a613c..b9a1c253b7 100644 --- a/pkg/compute/tasks/loadbalancer_listener_sync_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_sync_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerSyncTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_SYNC_CONF_FAILED, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_SYNC_CONF_FAILED, reason) db.OpsLog.LogEvent(lblis, db.ACT_SYNC_CONF, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_SYNC_CONF, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_SYNC_CONF_FAILED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_SYNC_CONF_FAILED, reason) self.SetStageFailed(ctx, reason) } @@ -59,6 +59,6 @@ func (self *LoadbalancerListenerSyncTask) OnLoadbalancerListenerSyncStatusComple } func (self *LoadbalancerListenerSyncTask) OnLoadbalancerListenerSyncStatusCompleteFailed(ctx context.Context, lblis *models.SLoadbalancerListener, reason jsonutils.JSONObject) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_UNKNOWN, reason.String()) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_UNKNOWN, reason.String()) self.SetStageFailed(ctx, reason.String()) } diff --git a/pkg/compute/tasks/loadbalancer_listener_syncstatus_task.go b/pkg/compute/tasks/loadbalancer_listener_syncstatus_task.go index cfc4577a23..f758410644 100644 --- a/pkg/compute/tasks/loadbalancer_listener_syncstatus_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_syncstatus_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerListenerSyncstatusTask) taskFail(ctx context.Context, lblis *models.SLoadbalancerListener, reason string) { - lblis.SetStatus(self.GetUserCred(), consts.LB_STATUS_UNKNOWN, reason) + lblis.SetStatus(self.GetUserCred(), api.LB_STATUS_UNKNOWN, reason) db.OpsLog.LogEvent(lblis, db.ACT_SYNC_STATUS, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lblis, logclient.ACT_SYNC_STATUS, reason, self.UserCred, false) - notifyclient.NotifySystemError(lblis.Id, lblis.Name, consts.LB_SYNC_CONF_FAILED, reason) + notifyclient.NotifySystemError(lblis.Id, lblis.Name, api.LB_SYNC_CONF_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/loadbalancer_start_task.go b/pkg/compute/tasks/loadbalancer_start_task.go index 3b34618277..ee868c23a6 100644 --- a/pkg/compute/tasks/loadbalancer_start_task.go +++ b/pkg/compute/tasks/loadbalancer_start_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerStartTask) taskFail(ctx context.Context, lb *models.SLoadbalancer, reason string) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, reason) + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, reason) db.OpsLog.LogEvent(lb, db.ACT_ENABLE, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_ENABLE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lb.Id, lb.Name, consts.LB_STATUS_DISABLED, reason) + notifyclient.NotifySystemError(lb.Id, lb.Name, api.LB_STATUS_DISABLED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerStartTask) OnInit(ctx context.Context, obj db.IStandalon } func (self *LoadbalancerStartTask) OnLoadbalancerStartComplete(ctx context.Context, lb *models.SLoadbalancer, data jsonutils.JSONObject) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, "") + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, "") db.OpsLog.LogEvent(lb, db.ACT_ENABLE, lb.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_ENABLE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_stop_task.go b/pkg/compute/tasks/loadbalancer_stop_task.go index e3a60c30ca..b0d4b5e740 100644 --- a/pkg/compute/tasks/loadbalancer_stop_task.go +++ b/pkg/compute/tasks/loadbalancer_stop_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerStopTask) taskFail(ctx context.Context, lb *models.SLoadbalancer, reason string) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_ENABLED, reason) + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_ENABLED, reason) db.OpsLog.LogEvent(lb, db.ACT_DISABLE, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_DISABLE, reason, self.UserCred, false) - notifyclient.NotifySystemError(lb.Id, lb.Name, consts.LB_STATUS_ENABLED, reason) + notifyclient.NotifySystemError(lb.Id, lb.Name, api.LB_STATUS_ENABLED, reason) self.SetStageFailed(ctx, reason) } @@ -44,7 +44,7 @@ func (self *LoadbalancerStopTask) OnInit(ctx context.Context, obj db.IStandalone } func (self *LoadbalancerStopTask) OnLoadbalancerStopComplete(ctx context.Context, lb *models.SLoadbalancer, data jsonutils.JSONObject) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_DISABLED, "") + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_DISABLED, "") db.OpsLog.LogEvent(lb, db.ACT_DISABLE, lb.GetShortDesc(ctx), self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_DISABLE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/loadbalancer_syncstatus_task.go b/pkg/compute/tasks/loadbalancer_syncstatus_task.go index c62f5f694b..aaabf16eee 100644 --- a/pkg/compute/tasks/loadbalancer_syncstatus_task.go +++ b/pkg/compute/tasks/loadbalancer_syncstatus_task.go @@ -6,10 +6,10 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -23,10 +23,10 @@ func init() { } func (self *LoadbalancerSyncstatusTask) taskFail(ctx context.Context, lb *models.SLoadbalancer, reason string) { - lb.SetStatus(self.GetUserCred(), consts.LB_STATUS_UNKNOWN, reason) + lb.SetStatus(self.GetUserCred(), api.LB_STATUS_UNKNOWN, reason) db.OpsLog.LogEvent(lb, db.ACT_SYNC_STATUS, reason, self.UserCred) logclient.AddActionLogWithStartable(self, lb, logclient.ACT_SYNC_STATUS, reason, self.UserCred, false) - notifyclient.NotifySystemError(lb.Id, lb.Name, consts.LB_SYNC_CONF_FAILED, reason) + notifyclient.NotifySystemError(lb.Id, lb.Name, api.LB_SYNC_CONF_FAILED, reason) self.SetStageFailed(ctx, reason) } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 782467f48c..788a941592 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -17,8 +17,8 @@ import ( "yunion.io/x/pkg/util/seclib" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -698,7 +698,7 @@ func (s *SKVMGuestInstance) delTmpDisks(ctx context.Context, migrated bool) erro if disk.Contains("path") { diskPath, _ := disk.GetString("path") d := storageman.GetManager().GetDiskByPath(diskPath) - if d != nil && d.GetType() == consts.STORAGE_LOCAL && migrated { + if d != nil && d.GetType() == api.STORAGE_LOCAL && migrated { if err := d.DeleteAllSnapshot(); err != nil { log.Errorln(err) return err @@ -1218,7 +1218,7 @@ func (s *SKVMGuestInstance) PrepareMigrate(liveMigrage bool) (*jsonutils.JSONDic if disk.Contains("path") { diskPath, _ := disk.GetString("path") d := storageman.GetManager().GetDiskByPath(diskPath) - if d.GetType() == consts.STORAGE_LOCAL { + if d.GetType() == api.STORAGE_LOCAL { back, err := d.PrepareMigrate(liveMigrage) if err != nil { return nil, err diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index c9e536cd1f..6f3142b1c8 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -15,10 +15,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - bare2 "yunion.io/x/onecloud/pkg/baremetal" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" - bare1 "yunion.io/x/onecloud/pkg/compute/baremetal" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -320,9 +318,9 @@ func (h *SHostInfo) checkSystemServices() error { } func (h *SHostInfo) detectiveStorageSystem() { - var stype = consts.DISK_TYPE_ROTATE + var stype = api.DISK_TYPE_ROTATE if options.HostOptions.DiskIsSsd { - stype = consts.DISK_TYPE_SSD + stype = api.DISK_TYPE_SSD } h.sysinfo.StorageType = stype } @@ -858,7 +856,7 @@ func (h *SHostInfo) updateHostRecord(hostId string) { content.Set("cpu_mhz", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Freq))) content.Set("cpu_cache", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Cache))) content.Set("mem_size", jsonutils.NewInt(int64(h.Mem.MemInfo.Total))) - content.Set("storage_driver", jsonutils.NewString(bare1.DISK_DRIVER_LINUX)) + content.Set("storage_driver", jsonutils.NewString(api.DISK_DRIVER_LINUX)) content.Set("storage_type", jsonutils.NewString(h.sysinfo.StorageType)) content.Set("storage_size", jsonutils.NewInt(int64(storageman.GetManager().GetTotalCapacity()))) @@ -1009,7 +1007,7 @@ func (h *SHostInfo) doUploadNicInfo(nic *SNIC) { if len(nic.Ip) > 0 { content.Set("ip_addr", jsonutils.NewString(nic.Ip)) if nic.Ip == h.GetMasterIp() { - content.Set("nic_type", jsonutils.NewString(bare2.NIC_TYPE_ADMIN)) + content.Set("nic_type", jsonutils.NewString(api.NIC_TYPE_ADMIN)) } } _, err := modules.Hosts.PerformAction(h.GetSession(), @@ -1128,7 +1126,7 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { log.Infof("Storage %s(%s) mountpoint %s", storageName, storagetype, mountPoint) - if !utils.IsInStringArray(storagetype, consts.Local) { + if !utils.IsInStringArray(storagetype, api.STORAGE_LOCAL_TYPES) { storage := storageManager.NewSharedStorageInstance(mountPoint, storagetype) if storage != nil { storage.SetStoragecacheId(storagecacheId) @@ -1145,7 +1143,7 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { storage.SetStorageInfo(storageId, storageName, storageConf) } else { // XXX hack: storage type baremetal is a converted host，reserve storage - if storagetype != consts.STORAGE_BAREMETAL { + if storagetype != api.STORAGE_BAREMETAL { detachStorages = append(detachStorages, hs) } } diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 87e4954d7d..42e57f4b61 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" @@ -73,9 +73,9 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) { } func (s *SStorageManager) Remove(storage IStorage) { - if storage.StorageType() == consts.STORAGE_NFS { + if storage.StorageType() == api.STORAGE_NFS { delete(s.NfsStorageImagecacheManagers, storage.GetStoragecacheId()) - } else if storage.StorageType() == consts.STORAGE_RBD { + } else if storage.StorageType() == api.STORAGE_RBD { delete(s.RbdStorageImagecacheManagers, storage.GetStoragecacheId()) } for index, iS := range s.Storages { @@ -253,9 +253,9 @@ func (s *SStorageManager) NewSharedStorageInstance(mountPoint, storageType strin } func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheId, imagecachePath string, storage IStorage) { - if storageType == consts.STORAGE_NFS { + if storageType == api.STORAGE_NFS { s.InitNfsStorageImagecache(storagecacheId, imagecachePath) - } else if storageType == consts.STORAGE_RBD { + } else if storageType == api.STORAGE_RBD { if rbdStorage := s.GetStoragecacheById(storagecacheId); rbdStorage == nil { // Done s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId) @@ -280,7 +280,7 @@ func (s *SStorageManager) AddRbdStorageImagecache(imagecachePath string, storage s.RbdStorageImagecacheManagers = map[string]IImageCacheManger{} } if _, ok := s.RbdStorageImagecacheManagers[storagecacheId]; !ok { - if imagecache := NewImageCacheManager(s, imagecachePath, storage, storagecacheId, consts.STORAGE_RBD); imagecache != nil { + if imagecache := NewImageCacheManager(s, imagecachePath, storage, storagecacheId, api.STORAGE_RBD); imagecache != nil { s.RbdStorageImagecacheManagers[storagecacheId] = imagecache return } diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index bc9d004922..0d87b03057 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -10,8 +10,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" @@ -36,7 +36,7 @@ func NewLocalDisk(storage IStorage, id string) *SLocalDisk { } func (d *SBaseDisk) GetType() string { - return consts.STORAGE_LOCAL + return api.STORAGE_LOCAL } func (d *SLocalDisk) getPath() string { diff --git a/pkg/hostman/storageman/disk_nfs.go b/pkg/hostman/storageman/disk_nfs.go index 9c7450f629..bafaf26e95 100644 --- a/pkg/hostman/storageman/disk_nfs.go +++ b/pkg/hostman/storageman/disk_nfs.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" ) type SNFSDisk struct { @@ -20,7 +20,7 @@ func NewNFSDisk(storage IStorage, id string) *SNFSDisk { } func (d *SNFSDisk) GetType() string { - return consts.STORAGE_NFS + return api.STORAGE_NFS } func (d *SNFSDisk) CreateFromTemplate(ctx context.Context, imageId, format string, size int64) (jsonutils.JSONObject, error) { diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index f96bcd0b8e..daafcd7cf5 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -12,8 +12,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) @@ -28,7 +28,7 @@ func NewRBDDisk(storage IStorage, id string) *SRBDDisk { } func (d *SRBDDisk) GetType() string { - return consts.STORAGE_RBD + return api.STORAGE_RBD } func (d *SRBDDisk) Probe() error { diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go index fc7eb73c40..9ab76c919c 100644 --- a/pkg/hostman/storageman/imagecachemanager_rbd.go +++ b/pkg/hostman/storageman/imagecachemanager_rbd.go @@ -10,7 +10,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/hostutils" ) @@ -49,7 +49,7 @@ func (factory *SRbdImageCacheManagerFactory) NewImageCacheManager(manager *SStor } func (factory *SRbdImageCacheManagerFactory) StorageType() string { - return consts.STORAGE_RBD + return api.STORAGE_RBD } func init() { diff --git a/pkg/hostman/storageman/storage_local.go b/pkg/hostman/storageman/storage_local.go index 59b0867608..dbc38c2a4d 100644 --- a/pkg/hostman/storageman/storage_local.go +++ b/pkg/hostman/storageman/storage_local.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/timeutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -56,7 +56,7 @@ func (s *SLocalStorage) GetFuseMountPath() string { } func (s *SLocalStorage) StorageType() string { - return consts.STORAGE_LOCAL + return api.STORAGE_LOCAL } func (s *SLocalStorage) GetSnapshotDir() string { diff --git a/pkg/hostman/storageman/storage_nfs.go b/pkg/hostman/storageman/storage_nfs.go index 6bc6511c60..89c6564522 100644 --- a/pkg/hostman/storageman/storage_nfs.go +++ b/pkg/hostman/storageman/storage_nfs.go @@ -8,7 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/fileutils2" @@ -27,7 +27,7 @@ func (factory *SNFSStorageFactory) NewStorage(manager *SStorageManager, mountPoi } func (factory *SNFSStorageFactory) StorageType() string { - return consts.STORAGE_NFS + return api.STORAGE_NFS } type SNFSStorage struct { @@ -44,7 +44,7 @@ func NewNFSStorage(manager *SStorageManager, path string) *SNFSStorage { } func (s *SNFSStorage) StorageType() string { - return consts.STORAGE_NFS + return api.STORAGE_NFS } func (s *SNFSStorage) CreateDisk(diskId string) IDisk { @@ -84,7 +84,7 @@ func (s *SNFSStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content := jsonutils.NewDict() content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb()))) content.Set("storage_type", jsonutils.NewString(s.StorageType())) - content.Set("status", jsonutils.NewString(consts.STORAGE_ONLINE)) + content.Set("status", jsonutils.NewString(api.STORAGE_ONLINE)) content.Set("zone", jsonutils.NewString(s.GetZone())) log.Infof("Sync storage info %s", s.StorageId) res, err := modules.Storages.Put( diff --git a/pkg/hostman/storageman/storage_rbd.go b/pkg/hostman/storageman/storage_rbd.go index c01dc85c1f..2c5703c167 100644 --- a/pkg/hostman/storageman/storage_rbd.go +++ b/pkg/hostman/storageman/storage_rbd.go @@ -16,7 +16,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" @@ -54,7 +54,7 @@ func (factory *SRbdStorageFactory) NewStorage(manager *SStorageManager, mountPoi } func (factory *SRbdStorageFactory) StorageType() string { - return consts.STORAGE_RBD + return api.STORAGE_RBD } func init() { @@ -62,7 +62,7 @@ func init() { } func (s *SRbdStorage) StorageType() string { - return consts.STORAGE_RBD + return api.STORAGE_RBD } func (s *SRbdStorage) GetSnapshotPathByIds(diskId, snapshotId string) string { @@ -355,7 +355,7 @@ func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content = map[string]interface{}{ "name": s.StorageName, "capacity": capacity, - "status": consts.STORAGE_ONLINE, + "status": api.STORAGE_ONLINE, "zone": s.GetZone(), } return modules.Storages.Put(hostutils.GetComputeSession(context.Background()), s.StorageId, jsonutils.Marshal(content)) diff --git a/pkg/lbagent/api.go b/pkg/lbagent/api.go index c65e20134a..a519fc61bf 100644 --- a/pkg/lbagent/api.go +++ b/pkg/lbagent/api.go @@ -11,7 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/version" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" agentmodels "yunion.io/x/onecloud/pkg/lbagent/models" agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" "yunion.io/x/onecloud/pkg/mcclient" @@ -37,7 +37,7 @@ func NewApiHelper(opts *Options) (*ApiHelper, error) { helper := &ApiHelper{ opts: opts, dataDirMan: agentutils.NewConfigDirManager(opts.apiDataStoreDir), - haState: consts.LB_HA_STATE_UNKNOWN, + haState: api.LB_HA_STATE_UNKNOWN, } return helper, nil } @@ -69,7 +69,7 @@ func (h *ApiHelper) Run(ctx context.Context) { } case state := <-h.haStateProvider.StateChannel(): switch state { - case consts.LB_HA_STATE_BACKUP: + case api.LB_HA_STATE_BACKUP: h.doStopDaemons(ctx) default: if state != h.haState { @@ -98,7 +98,7 @@ func (h *ApiHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess func (h *ApiHelper) agentPeekOnce(ctx context.Context) (*models.LoadbalancerAgent, error) { s := h.adminClientSession(ctx) params := jsonutils.NewDict() - params.Set(consts.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(consts.LBAGENT_QUERY_ORIG_VAL)) + params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL)) data, err := modules.LoadbalancerAgents.Get(s, h.opts.ApiLbagentId, params) if err != nil { err := fmt.Errorf("agent get error: %s", err) @@ -116,7 +116,7 @@ func (h *ApiHelper) agentPeekOnce(ctx context.Context) (*models.LoadbalancerAgen func (h *ApiHelper) agentPeekPeers(ctx context.Context, vri int) ([]*models.LoadbalancerAgent, error) { s := h.adminClientSession(ctx) params := jsonutils.NewDict() - params.Set(consts.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(consts.LBAGENT_QUERY_ORIG_VAL)) + params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL)) listResult, err := modules.LoadbalancerAgents.List(s, params) if err != nil { err := fmt.Errorf("agent listing error: %s", err) diff --git a/pkg/lbagent/hastate.go b/pkg/lbagent/hastate.go index b282e2d05c..501f6daac5 100644 --- a/pkg/lbagent/hastate.go +++ b/pkg/lbagent/hastate.go @@ -15,7 +15,7 @@ import ( "github.com/fsnotify/fsnotify" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" agentutils "yunion.io/x/onecloud/pkg/lbagent/utils" ) @@ -113,7 +113,7 @@ func (hsw *HaStateWatcher) Run(ctx context.Context) { func (hsw *HaStateWatcher) loadHaState() (err error) { defer func() { if err != nil { - hsw.CurrentState = consts.LB_HA_STATE_UNKNOWN + hsw.CurrentState = api.LB_HA_STATE_UNKNOWN } }() data, err := ioutil.ReadFile(hsw.HaStatePath) @@ -185,7 +185,7 @@ func NewHaStateWatcher(opts *Options) (hsw *HaStateWatcher, err error) { hsw = &HaStateWatcher{ HaStateScriptPath: haStateScriptPath, HaStatePath: haStatePath, - CurrentState: consts.LB_HA_STATE_UNKNOWN, + CurrentState: api.LB_HA_STATE_UNKNOWN, opts: opts, w: w, diff --git a/pkg/lbagent/models/reflect.go b/pkg/lbagent/models/reflect.go index 6060aacb66..45580633e9 100644 --- a/pkg/lbagent/models/reflect.go +++ b/pkg/lbagent/models/reflect.go @@ -10,7 +10,7 @@ import ( "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/models" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -106,7 +106,7 @@ func GetModels(opts *GetModelsOptions) error { if err != nil { return fmt.Errorf("%s: making list params: %s", manKeyPlural, err) } - params.Set(consts.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(consts.LBAGENT_QUERY_ORIG_VAL)) + params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL)) entriesJson := []jsonutils.JSONObject{} for { diff --git a/pkg/mcclient/options/loadbalancerbackendgroups.go b/pkg/mcclient/options/loadbalancerbackendgroups.go index c4e1219b62..e70895adfe 100644 --- a/pkg/mcclient/options/loadbalancerbackendgroups.go +++ b/pkg/mcclient/options/loadbalancerbackendgroups.go @@ -8,7 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" ) type LoadbalancerBackendGroupCreateOptions struct { @@ -55,8 +55,8 @@ func NewBackend(s string, index int) (*SBackend, error) { } backend.Port = port case "backend_type": - if utils.IsInStringArray(value[1], []string{consts.LB_BACKEND_GUEST, consts.LB_BACKEND_HOST}) { - return nil, fmt.Errorf("invalid backend type %s only support %s %s", value[1], consts.LB_BACKEND_GUEST, consts.LB_BACKEND_HOST) + if utils.IsInStringArray(value[1], []string{api.LB_BACKEND_GUEST, api.LB_BACKEND_HOST}) { + return nil, fmt.Errorf("invalid backend type %s only support %s %s", value[1], api.LB_BACKEND_GUEST, api.LB_BACKEND_HOST) } backend.BackendType = value[1] case "id": diff --git a/pkg/util/aliyun/loadbalancer.go b/pkg/util/aliyun/loadbalancer.go index 5187ba5fce..09f6300b1f 100644 --- a/pkg/util/aliyun/loadbalancer.go +++ b/pkg/util/aliyun/loadbalancer.go @@ -7,8 +7,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type ListenerProtocol string @@ -78,9 +78,9 @@ func (lb *SLoadbalancer) GetGlobalId() string { func (lb *SLoadbalancer) GetStatus() string { if lb.LoadBalancerStatus == "active" { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } - return consts.LB_STATUS_DISABLED + return api.LB_STATUS_DISABLED } func (lb *SLoadbalancer) GetMetadata() *jsonutils.JSONDict { @@ -195,14 +195,14 @@ func (lb *SLoadbalancer) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloud func (lb *SLoadbalancer) CreateILoadBalancerBackendGroup(group *cloudprovider.SLoadbalancerBackendGroup) (cloudprovider.ICloudLoadbalancerBackendGroup, error) { switch group.GroupType { - case consts.LB_BACKENDGROUP_TYPE_NORMAL: + case api.LB_BACKENDGROUP_TYPE_NORMAL: group, err := lb.region.CreateLoadbalancerBackendGroup(group.Name, lb.LoadBalancerId, group.Backends) if err != nil { return nil, err } group.lb = lb return group, nil - case consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: + case api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: group, err := lb.region.CreateLoadbalancerMasterSlaveBackendGroup(group.Name, lb.LoadBalancerId, group.Backends) if err != nil { return nil, err @@ -216,13 +216,13 @@ func (lb *SLoadbalancer) CreateILoadBalancerBackendGroup(group *cloudprovider.SL func (lb *SLoadbalancer) CreateILoadBalancerListener(listener *cloudprovider.SLoadbalancerListener) (cloudprovider.ICloudLoadbalancerListener, error) { switch listener.ListenerType { - case consts.LB_LISTENER_TYPE_TCP: + case api.LB_LISTENER_TYPE_TCP: return lb.region.CreateLoadbalancerTCPListener(lb, listener) - case consts.LB_LISTENER_TYPE_UDP: + case api.LB_LISTENER_TYPE_UDP: return lb.region.CreateLoadbalancerUDPListener(lb, listener) - case consts.LB_LISTENER_TYPE_HTTP: + case api.LB_LISTENER_TYPE_HTTP: return lb.region.CreateLoadbalancerHTTPListener(lb, listener) - case consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTPS: return lb.region.CreateLoadbalancerHTTPSListener(lb, listener) } return nil, fmt.Errorf("unsupport listener type %s", listener.ListenerType) @@ -238,9 +238,9 @@ func (lb *SLoadbalancer) GetLoadbalancerSpec() string { func (lb *SLoadbalancer) GetChargeType() string { switch lb.InternetChargeType { case "paybybandwidth": - return consts.LB_CHARGE_TYPE_BY_BANDWIDTH + return api.LB_CHARGE_TYPE_BY_BANDWIDTH case "paybytraffic": - return consts.LB_CHARGE_TYPE_BY_TRAFFIC + return api.LB_CHARGE_TYPE_BY_TRAFFIC } return "unknown" } diff --git a/pkg/util/aliyun/loadbalancerbackend.go b/pkg/util/aliyun/loadbalancerbackend.go index f62b73ff32..fd4c79507b 100644 --- a/pkg/util/aliyun/loadbalancerbackend.go +++ b/pkg/util/aliyun/loadbalancerbackend.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerBackend struct { @@ -63,11 +63,11 @@ func (backend *SLoadbalancerBackend) GetPort() int { } func (backend *SLoadbalancerBackend) GetBackendType() string { - return consts.LB_BACKEND_GUEST + return api.LB_BACKEND_GUEST } func (backend *SLoadbalancerBackend) GetBackendRole() string { - return consts.LB_BACKEND_ROLE_DEFAULT + return api.LB_BACKEND_ROLE_DEFAULT } func (backend *SLoadbalancerBackend) GetBackendId() string { diff --git a/pkg/util/aliyun/loadbalancerbackendgroup.go b/pkg/util/aliyun/loadbalancerbackendgroup.go index cdee94621e..af8edafc01 100644 --- a/pkg/util/aliyun/loadbalancerbackendgroup.go +++ b/pkg/util/aliyun/loadbalancerbackendgroup.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type Rule struct { @@ -55,7 +55,7 @@ func (backendgroup *SLoadbalancerBackendGroup) GetGlobalId() string { } func (backendgroup *SLoadbalancerBackendGroup) GetStatus() string { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } func (backendgroup *SLoadbalancerBackendGroup) IsDefault() bool { @@ -63,7 +63,7 @@ func (backendgroup *SLoadbalancerBackendGroup) IsDefault() bool { } func (backendgroup *SLoadbalancerBackendGroup) GetType() string { - return consts.LB_BACKENDGROUP_TYPE_NORMAL + return api.LB_BACKENDGROUP_TYPE_NORMAL } func (backendgroup *SLoadbalancerBackendGroup) GetMetadata() *jsonutils.JSONDict { diff --git a/pkg/util/aliyun/loadbalancerdefaultbackend.go b/pkg/util/aliyun/loadbalancerdefaultbackend.go index 485f8ae706..f89d23f593 100644 --- a/pkg/util/aliyun/loadbalancerdefaultbackend.go +++ b/pkg/util/aliyun/loadbalancerdefaultbackend.go @@ -5,7 +5,7 @@ import ( "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" ) type SLoadbalancerDefaultBackend struct { @@ -52,11 +52,11 @@ func (backend *SLoadbalancerDefaultBackend) GetPort() int { } func (backend *SLoadbalancerDefaultBackend) GetBackendType() string { - return consts.LB_BACKEND_GUEST + return api.LB_BACKEND_GUEST } func (backend *SLoadbalancerDefaultBackend) GetBackendRole() string { - return consts.LB_BACKEND_ROLE_DEFAULT + return api.LB_BACKEND_ROLE_DEFAULT } func (backend *SLoadbalancerDefaultBackend) GetBackendId() string { diff --git a/pkg/util/aliyun/loadbalancerdefaultbackendgroup.go b/pkg/util/aliyun/loadbalancerdefaultbackendgroup.go index 996d2e9782..f7a1b540eb 100644 --- a/pkg/util/aliyun/loadbalancerdefaultbackendgroup.go +++ b/pkg/util/aliyun/loadbalancerdefaultbackendgroup.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerDefaultBackendGroup struct { @@ -26,7 +26,7 @@ func (backendgroup *SLoadbalancerDefaultBackendGroup) GetGlobalId() string { } func (backendgroup *SLoadbalancerDefaultBackendGroup) GetStatus() string { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } func (backendgroup *SLoadbalancerDefaultBackendGroup) IsDefault() bool { @@ -34,7 +34,7 @@ func (backendgroup *SLoadbalancerDefaultBackendGroup) IsDefault() bool { } func (backendgroup *SLoadbalancerDefaultBackendGroup) GetType() string { - return consts.LB_BACKENDGROUP_TYPE_DEFAULT + return api.LB_BACKENDGROUP_TYPE_DEFAULT } func (backendgroup *SLoadbalancerDefaultBackendGroup) GetMetadata() *jsonutils.JSONDict { diff --git a/pkg/util/aliyun/loadbalancerhttplistener.go b/pkg/util/aliyun/loadbalancerhttplistener.go index 6e417e4f00..9a8da6ce67 100644 --- a/pkg/util/aliyun/loadbalancerhttplistener.go +++ b/pkg/util/aliyun/loadbalancerhttplistener.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerHTTPListener struct { @@ -79,11 +79,11 @@ func (listerner *SLoadbalancerHTTPListener) GetGlobalId() string { func (listerner *SLoadbalancerHTTPListener) GetStatus() string { switch listerner.Status { case "starting", "running": - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED case "configuring", "stopping", "stopped": - return consts.LB_STATUS_DISABLED + return api.LB_STATUS_DISABLED default: - return consts.LB_STATUS_UNKNOWN + return api.LB_STATUS_UNKNOWN } } @@ -287,7 +287,7 @@ func (listerner *SLoadbalancerHTTPListener) CreateILoadBalancerListenerRule(rule Url: rule.Path, RuleName: rule.Name, } - if len(rule.BackendGroupID) > 0 { //&& rule.BackendGroupType == consts.LB_BACKENDGROUP_TYPE_NORMAL { + if len(rule.BackendGroupID) > 0 { //&& rule.BackendGroupType == api.LB_BACKENDGROUP_TYPE_NORMAL { _rule.VServerGroupId = rule.BackendGroupID } listenerRule, err := listerner.lb.region.CreateLoadbalancerListenerRule(listerner.ListenerPort, listerner.lb.LoadBalancerId, _rule) diff --git a/pkg/util/aliyun/loadbalancerhttpslistener.go b/pkg/util/aliyun/loadbalancerhttpslistener.go index 135b4e7b77..2e0c2200c9 100644 --- a/pkg/util/aliyun/loadbalancerhttpslistener.go +++ b/pkg/util/aliyun/loadbalancerhttpslistener.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerHTTPSListener struct { @@ -74,11 +74,11 @@ func (listerner *SLoadbalancerHTTPSListener) GetGlobalId() string { func (listerner *SLoadbalancerHTTPSListener) GetStatus() string { switch listerner.Status { case "starting", "running": - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED case "configuring", "stopping", "stopped": - return consts.LB_STATUS_DISABLED + return api.LB_STATUS_DISABLED default: - return consts.LB_STATUS_UNKNOWN + return api.LB_STATUS_UNKNOWN } } @@ -308,7 +308,7 @@ func (listerner *SLoadbalancerHTTPSListener) CreateILoadBalancerListenerRule(rul Url: rule.Path, RuleName: rule.Name, } - if len(rule.BackendGroupID) > 0 { //&& rule.BackendGroupType == consts.LB_BACKENDGROUP_TYPE_NORMAL { + if len(rule.BackendGroupID) > 0 { //&& rule.BackendGroupType == api.LB_BACKENDGROUP_TYPE_NORMAL { _rule.VServerGroupId = rule.BackendGroupID } listenerRule, err := listerner.lb.region.CreateLoadbalancerListenerRule(listerner.ListenerPort, listerner.lb.LoadBalancerId, _rule) diff --git a/pkg/util/aliyun/loadbalancermasterslavebackend.go b/pkg/util/aliyun/loadbalancermasterslavebackend.go index fb30afe141..e0da25030a 100644 --- a/pkg/util/aliyun/loadbalancermasterslavebackend.go +++ b/pkg/util/aliyun/loadbalancermasterslavebackend.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" ) type SLoadbalancerMasterSlaveBackend struct { @@ -55,7 +55,7 @@ func (backend *SLoadbalancerMasterSlaveBackend) GetPort() int { } func (backend *SLoadbalancerMasterSlaveBackend) GetBackendType() string { - return consts.LB_BACKEND_GUEST + return api.LB_BACKEND_GUEST } func (backend *SLoadbalancerMasterSlaveBackend) GetBackendRole() string { diff --git a/pkg/util/aliyun/loadbalancermasterslavebackendgroup.go b/pkg/util/aliyun/loadbalancermasterslavebackendgroup.go index 5e9a560616..84bf6fd149 100644 --- a/pkg/util/aliyun/loadbalancermasterslavebackendgroup.go +++ b/pkg/util/aliyun/loadbalancermasterslavebackendgroup.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerMasterSlaveBackendGroup struct { @@ -29,7 +29,7 @@ func (backendgroup *SLoadbalancerMasterSlaveBackendGroup) GetGlobalId() string { } func (backendgroup *SLoadbalancerMasterSlaveBackendGroup) GetStatus() string { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } func (backendgroup *SLoadbalancerMasterSlaveBackendGroup) GetMetadata() *jsonutils.JSONDict { @@ -49,7 +49,7 @@ func (backendgroup *SLoadbalancerMasterSlaveBackendGroup) IsDefault() bool { } func (backendgroup *SLoadbalancerMasterSlaveBackendGroup) GetType() string { - return consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE + return api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE } func (region *SRegion) GetLoadbalancerMasterSlaveBackendgroups(loadbalancerId string) ([]SLoadbalancerMasterSlaveBackendGroup, error) { diff --git a/pkg/util/aliyun/loadbalancertcplistener.go b/pkg/util/aliyun/loadbalancertcplistener.go index 6c4ed55aa5..296af58b80 100644 --- a/pkg/util/aliyun/loadbalancertcplistener.go +++ b/pkg/util/aliyun/loadbalancertcplistener.go @@ -6,8 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerTCPListener struct { @@ -57,11 +57,11 @@ func (listerner *SLoadbalancerTCPListener) GetGlobalId() string { func (listerner *SLoadbalancerTCPListener) GetStatus() string { switch listerner.Status { case "starting", "running": - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED case "configuring", "stopping", "stopped": - return consts.LB_STATUS_DISABLED + return api.LB_STATUS_DISABLED default: - return consts.LB_STATUS_UNKNOWN + return api.LB_STATUS_UNKNOWN } } @@ -238,11 +238,11 @@ func (region *SRegion) constructBaseCreateListenerParams(lb *SLoadbalancer, list params["AclType"] = listener.AccessControlListType } switch listener.BackendGroupType { - case consts.LB_BACKENDGROUP_TYPE_NORMAL: + case api.LB_BACKENDGROUP_TYPE_NORMAL: params["VServerGroupId"] = listener.BackendGroupID - case consts.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: + case api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE: params["MasterSlaveServerGroupId"] = listener.BackendGroupID - case consts.LB_BACKENDGROUP_TYPE_DEFAULT: + case api.LB_BACKENDGROUP_TYPE_DEFAULT: params["BackendServerPort"] = fmt.Sprintf("%d", listener.BackendServerPort) } if len(listener.Name) > 0 { @@ -252,7 +252,7 @@ func (region *SRegion) constructBaseCreateListenerParams(lb *SLoadbalancer, list params["EstablishedTimeout"] = fmt.Sprintf("%d", listener.EstablishedTimeout) } - if utils.IsInStringArray(listener.ListenerType, []string{consts.LB_LISTENER_TYPE_TCP, consts.LB_LISTENER_TYPE_UDP}) { + if utils.IsInStringArray(listener.ListenerType, []string{api.LB_LISTENER_TYPE_TCP, api.LB_LISTENER_TYPE_UDP}) { if listener.HealthCheckTimeout >= 1 && listener.HealthCheckTimeout <= 300 { params["HealthCheckConnectTimeout"] = fmt.Sprintf("%d", listener.HealthCheckTimeout) } diff --git a/pkg/util/aliyun/loadbalancerudplistener.go b/pkg/util/aliyun/loadbalancerudplistener.go index 5ca3bf9015..a38788fb87 100644 --- a/pkg/util/aliyun/loadbalancerudplistener.go +++ b/pkg/util/aliyun/loadbalancerudplistener.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLoadbalancerUDPListener struct { @@ -56,11 +56,11 @@ func (listerner *SLoadbalancerUDPListener) GetGlobalId() string { func (listerner *SLoadbalancerUDPListener) GetStatus() string { switch listerner.Status { case "starting", "running": - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED case "configuring", "stopping", "stopped": - return consts.LB_STATUS_DISABLED + return api.LB_STATUS_DISABLED default: - return consts.LB_STATUS_UNKNOWN + return api.LB_STATUS_UNKNOWN } } diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index a558d1f763..b2575e723a 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -14,8 +14,8 @@ import ( "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -848,7 +848,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala if err != nil { return nil, err } - return iLoadbalancer, cloudprovider.WaitStatus(iLoadbalancer, consts.LB_STATUS_ENABLED, time.Second*5, time.Minute*5) + return iLoadbalancer, cloudprovider.WaitStatus(iLoadbalancer, api.LB_STATUS_ENABLED, time.Second*5, time.Minute*5) } func (region *SRegion) AddAccessControlListEntry(aclId string, entrys []cloudprovider.SLoadbalancerAccessControlListEntry) error { diff --git a/pkg/util/qcloud/loadbalancer.go b/pkg/util/qcloud/loadbalancer.go index 2bf9664005..d8b04b5af0 100755 --- a/pkg/util/qcloud/loadbalancer.go +++ b/pkg/util/qcloud/loadbalancer.go @@ -9,8 +9,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) const ( @@ -58,7 +58,7 @@ func (self *SLoadbalancer) GetLoadbalancerSpec() string { } func (self *SLoadbalancer) GetChargeType() string { - return consts.LB_CHARGE_TYPE_BY_HOUR + return api.LB_CHARGE_TYPE_BY_HOUR } // https://cloud.tencent.com/document/product/214/30689 @@ -125,7 +125,7 @@ func onecloudHealthCodeToQcloud(codes string) int { // 应用型负载均衡 https监听默认开启SNI。传统型不支持设置SNI func (self *SLoadbalancer) CreateILoadBalancerListener(listener *cloudprovider.SLoadbalancerListener) (cloudprovider.ICloudLoadbalancerListener, error) { sniSwitch := 0 - if listener.ListenerType == consts.LB_LISTENER_TYPE_HTTPS { + if listener.ListenerType == api.LB_LISTENER_TYPE_HTTPS { sniSwitch = 1 } @@ -202,11 +202,11 @@ func (self *SLoadbalancer) GetGlobalId() string { func (self *SLoadbalancer) GetStatus() string { switch self.Status { case 0: - return consts.LB_STATUS_INIT + return api.LB_STATUS_INIT case 1: - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED default: - return consts.LB_STATUS_UNKNOWN + return api.LB_STATUS_UNKNOWN } } @@ -241,16 +241,16 @@ func (self *SLoadbalancer) GetAddress() string { func (self *SLoadbalancer) GetAddressType() string { switch self.LoadBalancerType { case LB_ADDR_TYPE_INTERNAL: - return consts.LB_ADDR_TYPE_INTRANET + return api.LB_ADDR_TYPE_INTRANET case LB_ADDR_TYPE_OPEN: - return consts.LB_ADDR_TYPE_INTERNET + return api.LB_ADDR_TYPE_INTERNET default: return "" } } func (self *SLoadbalancer) GetNetworkType() string { - return consts.LB_NETWORK_TYPE_VPC + return api.LB_NETWORK_TYPE_VPC } func (self *SLoadbalancer) GetNetworkId() string { @@ -303,7 +303,7 @@ func (self *SLoadbalancer) GetILoadBalancerBackendGroups() ([]cloudprovider.IClo for i := range listeners { listener := listeners[i] t := listener.GetListenerType() - if t == consts.LB_LISTENER_TYPE_HTTP || t == consts.LB_LISTENER_TYPE_HTTPS { + if t == api.LB_LISTENER_TYPE_HTTP || t == api.LB_LISTENER_TYPE_HTTPS { rules := listener.Rules for i := range rules { rule := rules[i] diff --git a/pkg/util/qcloud/loadbalancer_acl.go b/pkg/util/qcloud/loadbalancer_acl.go index 2a7ea962db..d680ffd854 100755 --- a/pkg/util/qcloud/loadbalancer_acl.go +++ b/pkg/util/qcloud/loadbalancer_acl.go @@ -3,7 +3,7 @@ package qcloud import ( "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/compute/consts" + api "yunion.io/x/onecloud/pkg/apis/compute" ) // 腾讯云没有LB ACL @@ -22,7 +22,7 @@ func (self *SLBACL) GetGlobalId() string { } func (self *SLBACL) GetStatus() string { - return consts.LB_BOOL_OFF + return api.LB_BOOL_OFF } func (self *SLBACL) Refresh() error { diff --git a/pkg/util/qcloud/loadbalancer_backend.go b/pkg/util/qcloud/loadbalancer_backend.go index 9d37bb6e90..6eb3f4d80d 100755 --- a/pkg/util/qcloud/loadbalancer_backend.go +++ b/pkg/util/qcloud/loadbalancer_backend.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLBBackend struct { @@ -89,11 +89,11 @@ func (self *SLBBackend) GetPort() int { } func (self *SLBBackend) GetBackendType() string { - return consts.LB_BACKEND_GUEST + return api.LB_BACKEND_GUEST } func (self *SLBBackend) GetBackendRole() string { - return consts.LB_BACKEND_ROLE_DEFAULT + return api.LB_BACKEND_ROLE_DEFAULT } func (self *SLBBackend) GetBackendId() string { diff --git a/pkg/util/qcloud/loadbalancer_backendgroup.go b/pkg/util/qcloud/loadbalancer_backendgroup.go index 36218cdd5d..0c07d37430 100755 --- a/pkg/util/qcloud/loadbalancer_backendgroup.go +++ b/pkg/util/qcloud/loadbalancer_backendgroup.go @@ -8,8 +8,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLBBackendGroup struct { @@ -141,7 +141,7 @@ func backendGroupIdGen(lbid string, secondId string) string { func (self *SLBBackendGroup) GetId() string { t := self.listener.GetListenerType() - if t == consts.LB_LISTENER_TYPE_HTTP || t == consts.LB_LISTENER_TYPE_HTTPS { + if t == api.LB_LISTENER_TYPE_HTTP || t == api.LB_LISTENER_TYPE_HTTPS { // http https 后端服务器只与规则绑定 return backendGroupIdGen(self.lb.GetId(), self.rule.GetId()) } else if self.lb.Forward == LB_TYPE_APPLICATION { @@ -161,7 +161,7 @@ func (self *SLBBackendGroup) GetGlobalId() string { } func (self *SLBBackendGroup) GetStatus() string { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } func (self *SLBBackendGroup) Refresh() error { @@ -182,7 +182,7 @@ func (self *SLBBackendGroup) IsDefault() bool { } func (self *SLBBackendGroup) GetType() string { - return consts.LB_BACKENDGROUP_TYPE_NORMAL + return api.LB_BACKENDGROUP_TYPE_NORMAL } func (self *SLBBackendGroup) GetILoadbalancerBackends() ([]cloudprovider.ICloudLoadbalancerBackend, error) { diff --git a/pkg/util/qcloud/loadbalancer_listener.go b/pkg/util/qcloud/loadbalancer_listener.go index 06600be8c6..6170fccd69 100755 --- a/pkg/util/qcloud/loadbalancer_listener.go +++ b/pkg/util/qcloud/loadbalancer_listener.go @@ -8,16 +8,16 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) var HTTP_CODES = []string{ - consts.LB_HEALTH_CHECK_HTTP_CODE_1xx, - consts.LB_HEALTH_CHECK_HTTP_CODE_2xx, - consts.LB_HEALTH_CHECK_HTTP_CODE_3xx, - consts.LB_HEALTH_CHECK_HTTP_CODE_4xx, - consts.LB_HEALTH_CHECK_HTTP_CODE_5xx, + api.LB_HEALTH_CHECK_HTTP_CODE_1xx, + api.LB_HEALTH_CHECK_HTTP_CODE_2xx, + api.LB_HEALTH_CHECK_HTTP_CODE_3xx, + api.LB_HEALTH_CHECK_HTTP_CODE_4xx, + api.LB_HEALTH_CHECK_HTTP_CODE_5xx, } type certificate struct { @@ -213,7 +213,7 @@ func (self *SLBListener) GetGlobalId() string { // 腾讯云负载均衡没有启用禁用操作 func (self *SLBListener) GetStatus() string { - return consts.LB_STATUS_ENABLED + return api.LB_STATUS_ENABLED } func (self *SLBListener) Refresh() error { @@ -246,15 +246,15 @@ func (self *SLBListener) GetMetadata() *jsonutils.JSONDict { func (self *SLBListener) GetListenerType() string { switch self.Protocol { case "TCP": - return consts.LB_LISTENER_TYPE_TCP + return api.LB_LISTENER_TYPE_TCP case "UDP": - return consts.LB_LISTENER_TYPE_UDP + return api.LB_LISTENER_TYPE_UDP case "HTTP": - return consts.LB_LISTENER_TYPE_HTTP + return api.LB_LISTENER_TYPE_HTTP case "HTTPS": - return consts.LB_LISTENER_TYPE_HTTPS + return api.LB_LISTENER_TYPE_HTTPS case "TCP_SSL": - return consts.LB_LISTENER_TYPE_TCP + return api.LB_LISTENER_TYPE_TCP default: return "" } @@ -267,18 +267,18 @@ func (self *SLBListener) GetListenerPort() int { func (self *SLBListener) GetScheduler() string { switch strings.ToLower(self.Scheduler) { case "wrr": - return consts.LB_SCHEDULER_WRR + return api.LB_SCHEDULER_WRR case "ip_hash": - return consts.LB_SCHEDULER_SCH + return api.LB_SCHEDULER_SCH case "least_conn": - return consts.LB_SCHEDULER_WLC + return api.LB_SCHEDULER_WLC default: return "" } } func (self *SLBListener) GetAclStatus() string { - return consts.LB_BOOL_OFF + return api.LB_BOOL_OFF } func (self *SLBListener) GetAclType() string { @@ -291,17 +291,17 @@ func (self *SLBListener) GetAclId() string { func (self *SLBListener) GetHealthCheck() string { if self.HealthCheck.HealthSwitch == 0 { - return consts.LB_BOOL_OFF + return api.LB_BOOL_OFF } else { - return consts.LB_BOOL_ON + return api.LB_BOOL_ON } } func (self *SLBListener) GetHealthCheckType() string { if len(self.HealthCheck.HTTPCheckMethod) > 0 { - return consts.LB_HEALTH_CHECK_HTTP + return api.LB_HEALTH_CHECK_HTTP } else { - return consts.LB_HEALTH_CHECK_TCP + return api.LB_HEALTH_CHECK_TCP } } @@ -332,7 +332,7 @@ func (self *SLBListener) GetHealthCheckExp() string { func (self *SLBListener) GetBackendGroup() *SLBBackendGroup { t := self.GetListenerType() // http、https类型的监听不能直接绑定服务器 - if t == consts.LB_LISTENER_TYPE_HTTP || t == consts.LB_LISTENER_TYPE_HTTPS { + if t == api.LB_LISTENER_TYPE_HTTP || t == api.LB_LISTENER_TYPE_HTTPS { return nil } else { return &SLBBackendGroup{lb: self.lb, listener: self} @@ -381,21 +381,21 @@ func (self *SLBListener) GetILoadbalancerListenerRules() ([]cloudprovider.ICloud func (self *SLBListener) GetStickySession() string { if self.SessionExpireTime == 0 { - return consts.LB_BOOL_OFF + return api.LB_BOOL_OFF } else { - return consts.LB_BOOL_ON + return api.LB_BOOL_ON } } // 支持基于 cookie 插入的会话保持能力 https://cloud.tencent.com/document/product/214/6154 func (self *SLBListener) GetStickySessionType() string { - return consts.LB_STICKY_SESSION_TYPE_INSERT + return api.LB_STICKY_SESSION_TYPE_INSERT } // https://cloud.tencent.com/document/product/214/2736 // 经测试应用型负载均衡返回都是 tgw_l7_route。 func (self *SLBListener) GetStickySessionCookie() string { - if self.GetListenerType() == consts.LB_LISTENER_TYPE_HTTPS { + if self.GetListenerType() == api.LB_LISTENER_TYPE_HTTPS { return "tgw_l7_route" } @@ -414,7 +414,7 @@ https://cloud.tencent.com/document/product/214/6151 */ func (self *SLBListener) XForwardedForEnabled() bool { switch self.GetListenerType() { - case consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS: return true default: return false @@ -425,7 +425,7 @@ func (self *SLBListener) XForwardedForEnabled() bool { // 负载均衡开启Gzip配置及检测方法说明 https://cloud.tencent.com/document/product/214/5404 func (self *SLBListener) GzipEnabled() bool { switch self.GetListenerType() { - case consts.LB_LISTENER_TYPE_HTTP, consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTP, api.LB_LISTENER_TYPE_HTTPS: return true default: return false @@ -648,7 +648,7 @@ func (self *SRegion) UpdateLoadbalancerListener(t LB_TYPE, lbid string, listener func getHealthCheck(listener *cloudprovider.SLoadbalancerListener) *healthCheck { var hc *healthCheck - if listener.HealthCheck == consts.LB_BOOL_ON { + if listener.HealthCheck == api.LB_BOOL_ON { hc = &healthCheck{ HealthSwitch: 1, UnHealthNum: listener.HealthCheckFail, @@ -684,13 +684,13 @@ func getCertificate(listener *cloudprovider.SLoadbalancerListener) *certificate func getProtocol(listener *cloudprovider.SLoadbalancerListener) string { switch listener.ListenerType { - case consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTPS: return "HTTPS" - case consts.LB_LISTENER_TYPE_HTTP: + case api.LB_LISTENER_TYPE_HTTP: return "HTTP" - case consts.LB_LISTENER_TYPE_TCP: + case api.LB_LISTENER_TYPE_TCP: return "TCP" - case consts.LB_LISTENER_TYPE_UDP: + case api.LB_LISTENER_TYPE_UDP: return "UDP" case "tcp_ssl": return "TCP_SSL" @@ -701,13 +701,13 @@ func getProtocol(listener *cloudprovider.SLoadbalancerListener) string { func getClassicLBProtocol(listener *cloudprovider.SLoadbalancerListener) int { switch listener.ListenerType { - case consts.LB_LISTENER_TYPE_HTTP: + case api.LB_LISTENER_TYPE_HTTP: return 1 - case consts.LB_LISTENER_TYPE_HTTPS: + case api.LB_LISTENER_TYPE_HTTPS: return 4 - case consts.LB_LISTENER_TYPE_TCP: + case api.LB_LISTENER_TYPE_TCP: return 2 - case consts.LB_LISTENER_TYPE_UDP: + case api.LB_LISTENER_TYPE_UDP: return 3 default: return 0 // 非法值 @@ -717,11 +717,11 @@ func getClassicLBProtocol(listener *cloudprovider.SLoadbalancerListener) int { func getScheduler(listener *cloudprovider.SLoadbalancerListener) *string { var sch string switch listener.Scheduler { - case consts.LB_SCHEDULER_WRR: + case api.LB_SCHEDULER_WRR: sch = "WRR" - case consts.LB_SCHEDULER_WLC: + case api.LB_SCHEDULER_WLC: sch = "LEAST_CONN" - case consts.LB_SCHEDULER_SCH: + case api.LB_SCHEDULER_SCH: sch = "IP_HASH" default: return nil diff --git a/pkg/util/qcloud/loadbalancer_listenerrule.go b/pkg/util/qcloud/loadbalancer_listenerrule.go index 03cf94140b..8def46d729 100755 --- a/pkg/util/qcloud/loadbalancer_listenerrule.go +++ b/pkg/util/qcloud/loadbalancer_listenerrule.go @@ -6,8 +6,8 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" ) type SLBListenerRule struct { @@ -86,7 +86,7 @@ func (self *SLBListenerRule) GetProjectId() string { func (self *SLBListenerRule) GetBackendGroup() *SLBBackendGroup { t := self.listener.GetListenerType() - if t == consts.LB_LISTENER_TYPE_HTTP || t == consts.LB_LISTENER_TYPE_HTTPS { + if t == api.LB_LISTENER_TYPE_HTTP || t == api.LB_LISTENER_TYPE_HTTPS { return &SLBBackendGroup{ lb: self.listener.lb, listener: self.listener, diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 46d8eb49b2..d5a1c44411 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -13,8 +13,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/consts" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/qemutils" ) @@ -82,7 +82,7 @@ func (img *SQemuImage) parse() error { } else if strings.HasPrefix(img.Path, "sheepdog") { // sheepdog -> sheepdog[+tcp|+unix]://[host:port]/vdiname[?socket=path][#snapid|#tag] return cloudprovider.ErrNotImplemented - } else if strings.HasPrefix(img.Path, consts.STORAGE_RBD) { + } else if strings.HasPrefix(img.Path, api.STORAGE_RBD) { img.ActualSizeBytes = 0 } else { fileInfo, err := os.Stat(img.Path)