diff --git a/Gopkg.lock b/Gopkg.lock index 02e25daf62..0310a4d7a0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1703,7 +1703,7 @@ [[projects]] branch = "master" - digest = "1:04fac3de327ed7bf4490436dd3eb5fb10c7e80b919a109935d053f584fdb6566" + digest = "1:65cac3daa80d78369c59b0e5ca4d8db352bf412db23767213e04f8c19dc822e1" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1737,7 +1737,7 @@ "utils", ] pruneopts = "UT" - revision = "5d28ae811a2aaaed19efdc468573ef236b8327f9" + revision = "e9943d77232072e08a77520fdc775dd29a43462e" [[projects]] branch = "master" diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 9a6af79ebe..9fac46e66a 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -157,31 +157,15 @@ func init() { return nil }) - type DiskCreateOptions struct { - options.ScheduleOptions - NAME string `help:"Name of the disk"` - DISKDESC string `help:"Image size or size of virtual disk"` - Desc string `help:"Description" metavar:"Description"` - Storage string `help:"ID or name of storage where the disk is created"` - TaskNotify bool `help:"Setup task notify"` - } - R(&DiskCreateOptions{}, "disk-create", "Create a virtual disk", func(s *mcclient.ClientSession, args *DiskCreateOptions) error { - params, err := args.ScheduleOptions.Params() + R(&options.DiskCreateOptions{}, "disk-create", "Create a virtual disk", func(s *mcclient.ClientSession, args *options.DiskCreateOptions) error { + params, err := args.Params() if err != nil { return err } - params.Add(jsonutils.NewString(args.NAME), "name") - params.Add(jsonutils.NewString(args.DISKDESC), "disk") - if len(args.Desc) > 0 { - params.Add(jsonutils.NewString(args.Desc), "description") - } if args.TaskNotify { s.PrepareTask() } - if args.Storage != "" { - params.Add(jsonutils.NewString(args.Storage), "storage") - } - disk, err := modules.Disks.Create(s, params) + disk, err := modules.Disks.Create(s, params.JSON(params)) if err != nil { return err } diff --git a/cmd/climc/shell/schedtaghosts.go b/cmd/climc/shell/schedtaghosts.go index b68b5144e6..b2d636dfe1 100644 --- a/cmd/climc/shell/schedtaghosts.go +++ b/cmd/climc/shell/schedtaghosts.go @@ -3,8 +3,6 @@ package shell import ( "fmt" - "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" diff --git a/cmd/climc/shell/schedulers.go b/cmd/climc/shell/schedulers.go index 8c3c47b808..5bcea109b4 100644 --- a/cmd/climc/shell/schedulers.go +++ b/cmd/climc/shell/schedulers.go @@ -35,7 +35,7 @@ func init() { if err != nil { return err } - result, err := modules.SchedManager.DoForecast(s, params) + result, err := modules.SchedManager.DoForecast(s, params.JSON(params)) if err != nil { return err } diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 9ad2743bb5..0d1b2b9d10 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -8,7 +8,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -116,47 +115,31 @@ func init() { if err != nil { return err } - - if opts.GenerateName { - params.Add(jsonutils.NewString(opts.NAME), "generate_name") - } else { - params.Add(jsonutils.NewString(opts.NAME), "name") - } - if regutils.MatchSize(opts.MEMSPEC) { - params.Add(jsonutils.NewString(opts.MEMSPEC), "vmem_size") - } else { - params.Add(jsonutils.NewString(opts.MEMSPEC), "instance_type") - } - - if opts.NoAccountInit != nil && *opts.NoAccountInit { - params.Add(jsonutils.JSONFalse, "reset_password") - } - - if len(opts.UserDataFile) > 0 { - userdata, err := ioutil.ReadFile(opts.UserDataFile) - if err != nil { - return err - } - params.Add(jsonutils.NewString(string(userdata)), "user_data") - } - - count := options.IntV(opts.Count) + count := params.Count if options.BoolV(opts.DryRun) { - results, err := modules.SchedManager.DoScheduleListResult(s, params, count) + listFields := []string{"id", "name", "capacity", "count", "score", "capacity_details", "score_details"} + input, err := opts.ToScheduleInput() if err != nil { return err } - printList(results, []string{"id", "name", "rank", "capacity", "error"}) + result, err := modules.SchedManager.Test(s, input) + if err != nil { + return err + } + if err != nil { + return err + } + printList(modules.JSON2ListResult(result), listFields) } else { taskNotify := options.BoolV(opts.TaskNotify) if taskNotify { s.PrepareTask() } if count > 1 { - results := modules.Servers.BatchCreate(s, params, count) + results := modules.Servers.BatchCreate(s, params.JSON(params), count) printBatchResults(results, modules.Servers.GetColumns(s)) } else { - server, err := modules.Servers.Create(s, params) + server, err := modules.Servers.Create(s, params.JSON(params)) if err != nil { return err } @@ -384,7 +367,7 @@ func init() { if err != nil { return err } - srv, e := modules.Servers.PerformAction(s, opts.ID, "deploy", params) + srv, e := modules.Servers.PerformAction(s, opts.ID, "deploy", params.JSON(params)) if e != nil { return e } diff --git a/pkg/apis/api.go b/pkg/apis/api.go new file mode 100644 index 0000000000..2b6d540b7e --- /dev/null +++ b/pkg/apis/api.go @@ -0,0 +1,12 @@ +package apis + +import ( + "yunion.io/x/jsonutils" +) + +// Meta is embedded by every input or output params +type Meta struct{} + +func (m Meta) JSON(self interface{}) *jsonutils.JSONDict { + return jsonutils.Marshal(self).(*jsonutils.JSONDict) +} diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go new file mode 100644 index 0000000000..90a67806fc --- /dev/null +++ b/pkg/apis/compute/api.go @@ -0,0 +1,178 @@ +package compute + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" +) + +type SchedtagConfig struct { + apis.Meta + + Id string `json:"id"` + Strategy string `json:"strategy"` +} + +type NetworkConfig struct { + apis.Meta + + Index int `json:"index"` + Network string `json:"network"` + Wire string `json:"wire"` + Exit bool `json:"exit"` + Private bool `json:"private"` + Mac string `json:"mac"` + Address string `json:"address"` + Address6 string `json:"address6"` + Driver string `json:"driver"` + BwLimit int `json:"bw_limit"` + Vip bool `json:"vip"` + Reserved bool `json:"reserved"` + NetType string `json:"net_type"` + + RequireTeaming bool `json:"require_teaming"` + TryTeaming bool `json:"try_teaming"` + + StandbyPortCount int `json:"standby_port_count"` + StandbyAddrCount int `json:"standby_addr_count"` + + Project string `json:"project"` + Ifname string `json:"ifname"` +} + +type DiskConfig struct { + apis.Meta + + Index int `json:"index"` // -1 means not attach to any guests + ImageId string `json:"image_id"` + SnapshotId string `json:"snapshot_id"` + DiskType string `json:"disk_type"` + Schedtags []*SchedtagConfig `json:"schedtags"` + + SizeMb int `json:"size"` + Fs string `json:"fs"` + Format string `json:"format"` + Driver string `json:"driver"` + Cache string `json:"cache"` + Mountpoint string `json:"mountpoint"` + Backend string `json:"backend"` // stroageType + Medium string `json:"medium"` + ImageProperties map[string]string `json:"image_properties"` + + Storage string `json:"storage"` + DiskId string `json:"disk_id"` +} + +type IsolatedDeviceConfig struct { + Index int `json:"index"` + Id string `json:"id"` + DevType string `json:"dev_type"` + Model string `json:"model"` + Vendor string `json:"vendor"` +} + +type BaremetalDiskConfig struct { + //Index int `json:"index"` + // disk type + Type string `json:"type"` + // raid config + Conf string `json:"conf"` + Count int64 `json:"count"` + Range []int64 `json:"range"` + Splits string `json:"splits"` + Size []int64 `json:"size"` + Adapter *int `json:"adapter,omitempty"` + Driver string `json:"driver"` + Cachedbadbbu *bool `json:"cachedbadbbu,omitempty"` + Strip *int64 `json:"strip,omitempty"` + RA *bool `json:"ra,omitempty"` + WT *bool `json:"wt,omitempty"` + Direct *bool `json:"direct,omitempty"` +} + +type ServerConfigs struct { + // prefer options + PreferRegion string `json:"prefer_region_id"` + PreferZone string `json:"prefer_zone_id"` + PreferWire string `json:"prefer_wire_id"` + PreferHost string `json:"prefer_host_id"` + PreferBackupHost string `json:"prefer_backup_host"` + + Hypervisor string `json:"hypervisor"` + // ResourceType "shared|prepaid|dedicated"` + ResourceType string `json:"resource_type"` + Project string `json:"project"` + Backup bool `json:"backup"` + Count int `json:"count"` + + Disks []*DiskConfig `json:"disks"` + Networks []*NetworkConfig `json:"nets"` + Schedtags []*SchedtagConfig `json:"schedtags"` + IsolatedDevices []*IsolatedDeviceConfig `json:"isolated_devices"` + BaremetalDiskConfigs []*BaremetalDiskConfig `json:"baremetal_disk_configs"` + + // DEPRECATE + Suggestion bool `json:"suggestion"` +} + +type DeployConfig struct { + Action string `json:"action"` + Path string `json:"path"` + Content string `json:"content"` +} + +type ServerCreateInput struct { + apis.Meta + + *ServerConfigs + + Name string `json:"name"` + GenerateName string `json:"generate_name"` + VmemSize int `json:"vmem_size"` + VcpuCount int `json:"vcpu_count"` + InstanceType string `json:"instance_type"` + UserData string `json:"user_data"` + + Keypair string `json:"keypair"` + Password string `json:"password"` + Cdrom string `json:"cdrom"` + Vga string `json:"vga"` + Vdi string `json:"vdi"` + Bios string `json:"bios"` + Description string `json:"description"` + BootOrder string `json:"boot_order"` + ResetPassword *bool `json:"reset_password"` + DisableDelete bool `json:"disable_delete"` + ShutdownBehavior string `json:"shutdown_behavior"` + AutoStart bool `json:"auto_start"` + DeployConfigs []*DeployConfig `json:"deploy_configs"` + IsSystem bool `json:"is_system"` + Duration string `json:"duration"` + AutoPrepaidRecycle bool `json:"auto_prepaid_recycle,omitfalse"` + Secgroup string `json:"secgroup"` + EipBw int `json:"eip_bw,omitzero"` + EipChargeType string `json:"eip_charge_type,omitempty"` + Eip string `json:"eip,omitempty"` + + OsType string `json:"os_type"` + // Fill by server + OsProfile jsonutils.JSONObject `json:"__os_profile__"` + BillingType string `json:"billing_type"` + BillingCycle string `json:"billing_cycle"` + + // DEPRECATE or not used fields + Baremetal bool `json:"baremetal"` +} + +type ServerDeployInput struct { + apis.Meta + + Id string + + Keypair string `json:"keypair"` + DeleteKeypair bool `json:"__delete_keypair__"` + DeployConfigs []*DeployConfig `json:"deploy_configs"` + ResetPassword *bool `json:"reset_password"` + Password string `json:"password"` + AutoStart *bool `json:"auto_start"` +} diff --git a/pkg/apis/compute/baremetal_const.go b/pkg/apis/compute/baremetal_const.go new file mode 100644 index 0000000000..30a3d7f310 --- /dev/null +++ b/pkg/apis/compute/baremetal_const.go @@ -0,0 +1,54 @@ +package compute + +import ( + "yunion.io/x/pkg/util/sets" +) + +const ( + DISK_CONF_RAID0 = "raid0" + DISK_CONF_RAID1 = "raid1" + DISK_CONF_RAID5 = "raid5" + DISK_CONF_RAID10 = "raid10" + DISK_CONF_NONE = "none" + + DEFAULT_DISK_CONF = DISK_CONF_NONE + + DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + + DISK_DRIVER_MEGARAID = "MegaRaid" + DISK_DRIVER_LINUX = "Linux" + DISK_DRIVER_HPSARAID = "HPSARaid" + DISK_DRIVER_MPT2SAS = "Mpt2SAS" + DISK_DRIVER_MARVELRAID = "MarvelRaid" + DISK_DRIVER_PCIE = "PCIE" + + HDD_DISK_SPEC_TYPE = "HDD" + SSD_DISK_SPEC_TYPE = "SSD" +) + +var ( + BaremetalDefaultDiskConfig = BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_NONE, + Count: 0, + } + + DISK_CONFS = sets.NewString( + DISK_CONF_RAID0, + DISK_CONF_RAID1, + DISK_CONF_RAID5, + DISK_CONF_RAID10, + DISK_CONF_NONE, + ) + + DISK_DRIVERS_RAID = sets.NewString( + DISK_DRIVER_MEGARAID, + DISK_DRIVER_HPSARAID, + DISK_DRIVER_MPT2SAS, + DISK_DRIVER_MARVELRAID, + ) + + DISK_DRIVERS = sets.NewString( + DISK_DRIVER_LINUX, + DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) +) diff --git a/pkg/apis/compute/disk.go b/pkg/apis/compute/disk.go new file mode 100644 index 0000000000..34491ca2a5 --- /dev/null +++ b/pkg/apis/compute/disk.go @@ -0,0 +1,51 @@ +package compute + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type DiskCreateInput struct { + apis.Meta + + *DiskConfig + + // prefer options + PreferRegion string `json:"prefer_region_id"` + PreferZone string `json:"prefer_zone_id"` + PreferWire string `json:"prefer_wire_id"` + PreferHost string `json:"prefer_host_id"` + + Name string `json:"name"` + Description string `json:"description"` + Hypervisor string `json:"hypervisor"` + Project string `json:"project"` +} + +// ToServerCreateInput used by disk schedule +func (req *DiskCreateInput) ToServerCreateInput() *ServerCreateInput { + return &ServerCreateInput{ + ServerConfigs: &ServerConfigs{ + PreferRegion: req.PreferRegion, + PreferZone: req.PreferZone, + PreferWire: req.PreferWire, + PreferHost: req.PreferHost, + Hypervisor: req.Hypervisor, + Disks: []*DiskConfig{req.DiskConfig}, + Project: req.Project, + }, + Name: req.Name, + } +} + +func (req *ServerCreateInput) ToDiskCreateInput() *DiskCreateInput { + return &DiskCreateInput{ + DiskConfig: req.Disks[0], + PreferRegion: req.PreferRegion, + PreferHost: req.PreferHost, + PreferZone: req.PreferZone, + PreferWire: req.PreferWire, + Name: req.Name, + Project: req.Project, + Hypervisor: req.Hypervisor, + } +} diff --git a/pkg/apis/compute/disk_const.go b/pkg/apis/compute/disk_const.go new file mode 100644 index 0000000000..b864252f35 --- /dev/null +++ b/pkg/apis/compute/disk_const.go @@ -0,0 +1,38 @@ +package compute + +const ( + DISK_INIT = "init" + DISK_REBUILD = "rebuild" + DISK_ALLOC_FAILED = "alloc_failed" + DISK_STARTALLOC = "start_alloc" + DISK_BACKUP_STARTALLOC = "backup_start_alloc" + DISK_BACKUP_ALLOC_FAILED = "backup_alloc_failed" + DISK_ALLOCATING = "allocating" + DISK_READY = "ready" + DISK_RESET = "reset" + DISK_DEALLOC = "deallocating" + DISK_DEALLOC_FAILED = "dealloc_failed" + DISK_UNKNOWN = "unknown" + DISK_DETACHING = "detaching" + DISK_ATTACHING = "attaching" + + DISK_START_SAVE = "start_save" + DISK_SAVING = "saving" + + DISK_START_RESIZE = "start_resize" + DISK_RESIZING = "resizing" + + DISK_START_MIGRATE = "start_migrate" + DISK_POST_MIGRATE = "post_migrate" + DISK_MIGRATING = "migrating" + + DISK_START_SNAPSHOT = "start_snapshot" + DISK_SNAPSHOTING = "snapshoting" + + DISK_TYPE_SYS = "sys" + DISK_TYPE_SWAP = "swap" + DISK_TYPE_DATA = "data" + DISK_TYPE_VOLUME = "volume" + + DISK_BACKING_IMAGE = "image" +) diff --git a/pkg/apis/compute/doc.go b/pkg/apis/compute/doc.go new file mode 100644 index 0000000000..062f6a74e8 --- /dev/null +++ b/pkg/apis/compute/doc.go @@ -0,0 +1 @@ +package compute // import "yunion.io/x/onecloud/pkg/apis/compute" diff --git a/pkg/apis/compute/isolated_device_const.go b/pkg/apis/compute/isolated_device_const.go new file mode 100644 index 0000000000..7f613d8146 --- /dev/null +++ b/pkg/apis/compute/isolated_device_const.go @@ -0,0 +1,26 @@ +package compute + +const ( + DIRECT_PCI_TYPE = "PCI" + GPU_HPC_TYPE = "GPU-HPC" // # for compute + GPU_VGA_TYPE = "GPU-VGA" // # for display + USB_TYPE = "USB" + NIC_TYPE = "NIC" + + NVIDIA_VENDOR_ID = "10de" + AMD_VENDOR_ID = "1002" +) + +var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} + +var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE} + +var ID_VENDOR_MAP = map[string]string{ + NVIDIA_VENDOR_ID: "NVIDIA", + AMD_VENDOR_ID: "AMD", +} + +var VENDOR_ID_MAP = map[string]string{ + "NVIDIA": NVIDIA_VENDOR_ID, + "AMD": AMD_VENDOR_ID, +} diff --git a/pkg/apis/compute/network_const.go b/pkg/apis/compute/network_const.go new file mode 100644 index 0000000000..432af60b95 --- /dev/null +++ b/pkg/apis/compute/network_const.go @@ -0,0 +1,38 @@ +package compute + +const ( + // # DEFAULT_BANDWIDTH = options.default_bandwidth + MAX_BANDWIDTH = 100000 + + NETWORK_TYPE_GUEST = "guest" + NETWORK_TYPE_BAREMETAL = "baremetal" + NETWORK_TYPE_CONTAINER = "container" + NETWORK_TYPE_PXE = "pxe" + NETWORK_TYPE_IPMI = "ipmi" + + STATIC_ALLOC = "static" + + MAX_NETWORK_NAME_LEN = 11 + + EXTRA_DNS_UPDATE_TARGETS = "__extra_dns_update_targets" + + NETWORK_STATUS_INIT = "init" + NETWORK_STATUS_PENDING = "pending" + NETWORK_STATUS_AVAILABLE = "available" + NETWORK_STATUS_FAILED = "failed" + NETWORK_STATUS_UNKNOWN = "unknown" + NETWORK_STATUS_START_DELETE = "start_delete" + NETWORK_STATUS_DELETING = "deleting" + NETWORK_STATUS_DELETED = "deleted" + NETWORK_STATUS_DELETE_FAILED = "delete_failed" +) + +var ( + ALL_NETWORK_TYPES = []string{ + NETWORK_TYPE_GUEST, + NETWORK_TYPE_BAREMETAL, + NETWORK_TYPE_CONTAINER, + NETWORK_TYPE_PXE, + NETWORK_TYPE_IPMI, + } +) diff --git a/pkg/apis/compute/schedtag_const.go b/pkg/apis/compute/schedtag_const.go new file mode 100644 index 0000000000..6e3a301207 --- /dev/null +++ b/pkg/apis/compute/schedtag_const.go @@ -0,0 +1,13 @@ +package compute + +const ( + STRATEGY_REQUIRE = "require" + STRATEGY_EXCLUDE = "exclude" + STRATEGY_PREFER = "prefer" + STRATEGY_AVOID = "avoid" + + // # container used aggregate + CONTAINER_AGGREGATE = "container" +) + +var STRATEGY_LIST = []string{STRATEGY_REQUIRE, STRATEGY_EXCLUDE, STRATEGY_PREFER, STRATEGY_AVOID} diff --git a/pkg/apis/compute/storage_const.go b/pkg/apis/compute/storage_const.go new file mode 100644 index 0000000000..82ebce7899 --- /dev/null +++ b/pkg/apis/compute/storage_const.go @@ -0,0 +1,78 @@ +package compute + +const ( + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_NFS = "nfs" + + STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + STORAGE_CLOUD_SSD = "cloud_ssd" + STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 + STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" //单块本地SSD盘, 容量最大不能超过800 GiB + + //Azure hdd and ssd storagetype + STORAGE_STANDARD_LRS = "standard_lrs" + STORAGE_STANDARDSSD_LRS = "standardssd_lrs" + STORAGE_PREMIUM_LRS = "premium_lrs" + + // aws storage type + STORAGE_GP2_SSD = "gp2" // aws general purpose ssd + STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD + STORAGE_SC1_HDD = "sc1" // aws Cold HDD + STORAGE_STANDARD_HDD = "standard" // aws Magnetic volumes + + // qcloud storage type + // STORAGE_CLOUD_SSD ="cloud_ssd" + STORAGE_LOCAL_BASIC = "local_basic" + STORAGE_LOCAL_SSD = "local_ssd" + STORAGE_CLOUD_BASIC = "cloud_basic" + STORAGE_CLOUD_PREMIUM = "cloud_premium" + + // huawei storage type + STORAGE_HUAWEI_SSD = "SSD" // 超高IO云硬盘 + STORAGE_HUAWEI_SAS = "SAS" // 高IO云硬盘 + STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘 + + // openstack + STORAGE_OPENSTACK_ISCSI = "iscsi" +) + +const ( + STORAGE_ENABLED = "enabled" + // STORAGE_DISABLED = "disabled" + STORAGE_OFFLINE = "offline" + STORAGE_ONLINE = "online" + + DISK_TYPE_ROTATE = "rotate" + DISK_TYPE_SSD = "ssd" + DISK_TYPE_HYBRID = "hybrid" +) + +var ( + DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} + STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} + STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES + STORAGE_ALL_TYPES = []string{ + STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_NFS, + } + STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, + STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_EPHEMERAL_SSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS, + STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, + STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, + STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, + STORAGE_OPENSTACK_ISCSI, + } + + STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} +) diff --git a/pkg/apis/doc.go b/pkg/apis/doc.go new file mode 100644 index 0000000000..ed44acc20d --- /dev/null +++ b/pkg/apis/doc.go @@ -0,0 +1 @@ +package apis // import "yunion.io/x/onecloud/pkg/apis" diff --git a/pkg/apis/scheduler/api.go b/pkg/apis/scheduler/api.go new file mode 100644 index 0000000000..25dfc24fb5 --- /dev/null +++ b/pkg/apis/scheduler/api.go @@ -0,0 +1,82 @@ +package scheduler + +import ( + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +type ScheduleBaseConfig struct { + BestEffort bool `json:"best_effort"` + SuggestionLimit int64 `json:"suggestion_limit"` + SuggestionAll bool `json:"suggestion_all"` + IgnoreFilters []string `json:"ignore_filters"` + SessionId string `json:"session_id"` + + // usedby test api + RecordLog bool `json:"record_to_history"` + Details bool `json:"details"` +} + +type ForGuest struct { + Id string `json:"id"` + Name string `json:"name"` +} + +type GroupRelation struct { + GroupId string `json:"group_id"` + Strategy string `json:"strategy"` + Scope string `json:"scope"` +} + +type ServerConfig struct { + *compute.ServerConfigs + + Memory int `json:"vmem_size"` + Ncpu int `json:"vcpu_count"` + Name string `json:"name"` + GuestStatus string `json:"guest_status"` + + // HostId used by migrate + HostId string `json:"host_id"` + + // DEPRECATED + Metadata map[string]string `json:"__meta__"` + ForGuests []*ForGuest `json:"for_guests"` + GroupRelations []*GroupRelation `json:"group_releations"` + Groups interface{} `json:"groups"` + Id string `json:"id"` +} + +// ScheduleInput used by scheduler sync-schedule/test/forecast api +type ScheduleInput struct { + apis.Meta + + ScheduleBaseConfig + + ServerConfig +} + +type CandidateDisk struct { + Index int `json:"index"` + StorageId string `json:"storage_id"` +} + +type CandidateResource struct { + HostId string `json:"host_id"` + Name string `json:"name"` + Disks []*CandidateDisk `json:"disks"` + + // used by backup schedule + BackupCandidate *CandidateResource `json:"backup_candidate"` + IsMaster bool `json:"is_master"` + IsSlave bool `json:"is_slave"` + + // Error means no candidate found, include reasons + Error string `json:"error"` +} + +type ScheduleOutput struct { + apis.Meta + + Candidates []*CandidateResource `json:"candidates"` +} diff --git a/pkg/apis/scheduler/doc.go b/pkg/apis/scheduler/doc.go new file mode 100644 index 0000000000..5aeecee35f --- /dev/null +++ b/pkg/apis/scheduler/doc.go @@ -0,0 +1 @@ +package scheduler // import "yunion.io/x/onecloud/pkg/apis/scheduler" diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 7b77260d83..0e2fab8e94 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/pkg/util/workqueue" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" o "yunion.io/x/onecloud/pkg/baremetal/options" "yunion.io/x/onecloud/pkg/baremetal/profiles" "yunion.io/x/onecloud/pkg/baremetal/pxe" @@ -1162,7 +1163,7 @@ func (s *SBaremetalServer) GetRootTemplateId() string { return id } -func (s *SBaremetalServer) GetDiskConfig() ([]*baremetal.BaremetalDiskConfig, error) { +func (s *SBaremetalServer) GetDiskConfig() ([]*api.BaremetalDiskConfig, error) { layouts := make([]baremetal.Layout, 0) err := s.desc.Unmarshal(&layouts, "disk_config") if err != nil { diff --git a/pkg/baremetal/utils/raid/drivers/drivers.go b/pkg/baremetal/utils/raid/drivers/drivers.go index ab7518dbcd..e97b908a9f 100644 --- a/pkg/baremetal/utils/raid/drivers/drivers.go +++ b/pkg/baremetal/utils/raid/drivers/drivers.go @@ -3,6 +3,7 @@ package drivers import ( "fmt" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" _ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/hpssactl" _ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/megactl" @@ -28,7 +29,7 @@ func GetDrivers(term *ssh.Client) []raid.IRaidDriver { return ret } -func BuildRaid(driver raid.IRaidDriver, confs []*baremetal.BaremetalDiskConfig, adapterIdx int) error { +func BuildRaid(driver raid.IRaidDriver, confs []*api.BaremetalDiskConfig, adapterIdx int) error { if err := driver.PreBuildRaid(confs, adapterIdx); err != nil { return fmt.Errorf("PreBuildRaid: %v", err) } @@ -48,7 +49,7 @@ func BuildRaid(driver raid.IRaidDriver, confs []*baremetal.BaremetalDiskConfig, return nil } -func buildRaid(adapter raid.IRaidAdapter, confs []*baremetal.BaremetalDiskConfig) error { +func buildRaid(adapter raid.IRaidAdapter, confs []*api.BaremetalDiskConfig) error { if err := adapter.PreBuildRaid(confs); err != nil { return fmt.Errorf("PreBuildRaid: %v", err) } diff --git a/pkg/baremetal/utils/raid/hpssactl/hpssactl.go b/pkg/baremetal/utils/raid/hpssactl/hpssactl.go index c2cfaa9587..6bc1560d73 100644 --- a/pkg/baremetal/utils/raid/hpssactl/hpssactl.go +++ b/pkg/baremetal/utils/raid/hpssactl/hpssactl.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/regutils2" @@ -105,7 +106,7 @@ func (adapter *HPSARaidAdaptor) GetIndex() int { return adapter.index } -func (adapter *HPSARaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } @@ -163,7 +164,7 @@ func (adapter *HPSARaidAdaptor) GetDevices() []*baremetal.BaremetalStorage { return ret } -func (adapter *HPSARaidAdaptor) conf2Params(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *HPSARaidAdaptor) conf2Params(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.Direct != nil { if *(conf.Direct) { @@ -195,7 +196,7 @@ func (adapter *HPSARaidAdaptor) getLastArray() (string, error) { return "", nil } -func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { labels := []string{} for _, dev := range devs { labels = append(labels, fmt.Sprintf("%s", GetSpecString(dev))) @@ -235,19 +236,19 @@ func (adapter *HPSARaidAdaptor) buildRaid(level string, devs []*baremetal.Bareme return err } -func (adapter *HPSARaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("0", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("1", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("5", devs, conf) } -func (adapter *HPSARaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *HPSARaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("10", devs, conf) } @@ -256,7 +257,7 @@ func (adapter *HPSARaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStorage // WT|WB] [NORA|RA] [Direct|Cached] [CachedBadBBU|NoCachedBadBBU] useWT := true useDirect := true - if err := adapter.buildRaid("0", []*baremetal.BaremetalStorage{d}, &baremetal.BaremetalDiskConfig{WT: &useWT, Direct: &useDirect}); err != nil { + if err := adapter.buildRaid("0", []*baremetal.BaremetalStorage{d}, &api.BaremetalDiskConfig{WT: &useWT, Direct: &useDirect}); err != nil { return err } } @@ -354,7 +355,7 @@ func (r *HPSARaid) parsePhyDevs(lines []string) error { return nil } -func (r *HPSARaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *HPSARaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/baremetal/utils/raid/interface.go b/pkg/baremetal/utils/raid/interface.go index a49632c025..638493a774 100644 --- a/pkg/baremetal/utils/raid/interface.go +++ b/pkg/baremetal/utils/raid/interface.go @@ -1,6 +1,7 @@ package raid import ( + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/baremetal" ) @@ -8,21 +9,21 @@ type IRaidDriver interface { ParsePhyDevs() error GetName() string GetAdapters() []IRaidAdapter - PreBuildRaid(confs []*baremetal.BaremetalDiskConfig, adapterIdx int) error + PreBuildRaid(confs []*api.BaremetalDiskConfig, adapterIdx int) error CleanRaid() error } type IRaidAdapter interface { GetIndex() int - PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error + PreBuildRaid(confs []*api.BaremetalDiskConfig) error GetLogicVolumes() ([]int, error) RemoveLogicVolumes() error GetDevices() []*baremetal.BaremetalStorage - BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error - BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error + BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error + BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error BuildNoneRaid(devs []*baremetal.BaremetalStorage) error } diff --git a/pkg/baremetal/utils/raid/megactl/megactl.go b/pkg/baremetal/utils/raid/megactl/megactl.go index 31ade2a507..42eab7dc21 100644 --- a/pkg/baremetal/utils/raid/megactl/megactl.go +++ b/pkg/baremetal/utils/raid/megactl/megactl.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/pkg/util/stringutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/regutils2" @@ -225,12 +226,12 @@ func (adapter *MegaRaidAdaptor) parseLogicVolumes(lines []string) []int { return lvIdx } -func (adapter *MegaRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { adapter.clearJBODDisks() return nil } -func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *api.BaremetalDiskConfig) []string { params := []string{} szStr := []string{} if len(conf.Size) > 0 { @@ -242,7 +243,7 @@ func (adapter *MegaRaidAdaptor) conf2ParamsStorcliSize(conf *baremetal.Baremetal return params } -func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *baremetal.BaremetalDiskConfig) []string { +func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.WT != nil { if *conf.WT { @@ -278,7 +279,7 @@ func (adapter *MegaRaidAdaptor) conf2ParamsStorcli(conf *baremetal.BaremetalDisk return params } -func conf2Params(conf *baremetal.BaremetalDiskConfig) []string { +func conf2Params(conf *api.BaremetalDiskConfig) []string { params := []string{} if conf.WT != nil { if *conf.WT { @@ -319,35 +320,35 @@ func conf2Params(conf *baremetal.BaremetalDiskConfig) []string { return params } -func (adapter *MegaRaidAdaptor) storcliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 0) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 0) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 1) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 1) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 5) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.megacliBuildRaid(devs, conf, 5) } -func (adapter *MegaRaidAdaptor) storcliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.storcliBuildRaid(devs, conf, 10) } -func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(devs)%2 != 0 { return fmt.Errorf("Odd number of %d devs", len(devs)) } @@ -367,7 +368,7 @@ func (adapter *MegaRaidAdaptor) megacliBuildRaid10(devs []*baremetal.BaremetalSt return err } -func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig, level uint) error { +func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig, level uint) error { args := []string{} args = append(args, fmt.Sprintf("/c%d", adapter.index)) args = append(args, "add", "vd", fmt.Sprintf("type=r%d", level)) @@ -387,7 +388,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildRaid(devs []*baremetal.BaremetalStor return err } -func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig, level uint) error { +func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig, level uint) error { labels := []string{} for _, dev := range devs { labels = append(labels, GetSpecString(dev)) @@ -403,8 +404,8 @@ func (adapter *MegaRaidAdaptor) megacliBuildRaid(devs []*baremetal.BaremetalStor func cliBuildRaid( devs []*baremetal.BaremetalStorage, - conf *baremetal.BaremetalDiskConfig, - funcs ...func([]*baremetal.BaremetalStorage, *baremetal.BaremetalDiskConfig) error, + conf *api.BaremetalDiskConfig, + funcs ...func([]*baremetal.BaremetalStorage, *api.BaremetalDiskConfig) error, ) error { var err error for _, f := range funcs { @@ -416,19 +417,19 @@ func cliBuildRaid( return err } -func (adapter *MegaRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid0, adapter.storcliBuildRaid0) } -func (adapter *MegaRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid1, adapter.storcliBuildRaid1) } -func (adapter *MegaRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid5, adapter.storcliBuildRaid5) } -func (adapter *MegaRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return cliBuildRaid(devs, conf, adapter.megacliBuildRaid10, adapter.storcliBuildRaid10) } @@ -492,7 +493,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildJBOD(devs []*baremetal.BaremetalStor return nil } -func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { err := adapter.storcliBuildJBOD(devs) if err == nil { return nil @@ -513,7 +514,7 @@ func (adapter *MegaRaidAdaptor) storcliBuildNoRaid(devs []*baremetal.BaremetalSt return err } -func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MegaRaidAdaptor) megacliBuildNoRaid(devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { err := adapter.megacliBuildJBOD(devs) if err == nil { return nil @@ -746,7 +747,7 @@ func (raid *MegaRaid) CleanRaid() error { return nil } -func (raid *MegaRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (raid *MegaRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return raid.clearForeignState() } diff --git a/pkg/baremetal/utils/raid/mvcli/mvcli.go b/pkg/baremetal/utils/raid/mvcli/mvcli.go index be712c5957..9bb612e86d 100644 --- a/pkg/baremetal/utils/raid/mvcli/mvcli.go +++ b/pkg/baremetal/utils/raid/mvcli/mvcli.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/stringutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/ssh" @@ -185,11 +186,11 @@ func (adapter *MarvelRaidAdaptor) removeLogicVolume(idx int) error { return err } -func (adapter *MarvelRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } -func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, _ *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, _ *api.BaremetalDiskConfig) error { pds := []string{} for _, dev := range devs { pds = append(pds, fmt.Sprintf("%s", GetSpecString(dev))) @@ -200,20 +201,20 @@ func (adapter *MarvelRaidAdaptor) buildRaid(level string, devs []*baremetal.Bare return err } -func (adapter *MarvelRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r0", devs, conf) } -func (adapter *MarvelRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r1", devs, conf) } -func (adapter *MarvelRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { //return adapter.buildRaid("-r5", devs, conf) return fmt.Errorf("BuildRaid5 not impl") } -func (adapter *MarvelRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *MarvelRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("-r10", devs, conf) } @@ -276,7 +277,7 @@ func (r *MarvelRaid) parseAdapters(lines []string) error { return nil } -func (r *MarvelRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *MarvelRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/baremetal/utils/raid/sas2iru/sas2iru.go b/pkg/baremetal/utils/raid/sas2iru/sas2iru.go index bdc2f5b1f1..a146e0edf8 100644 --- a/pkg/baremetal/utils/raid/sas2iru/sas2iru.go +++ b/pkg/baremetal/utils/raid/sas2iru/sas2iru.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/stringutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/baremetal/utils/raid" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/util/ssh" @@ -225,11 +226,11 @@ func (adapter *Mpt2SASRaidAdaptor) setBootIR() error { return nil } -func (adapter *Mpt2SASRaidAdaptor) PreBuildRaid(confs []*baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) PreBuildRaid(confs []*api.BaremetalDiskConfig) error { return nil } -func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(conf.Size) > 1 { return fmt.Errorf("Subdivide sub-size not supported") } @@ -253,19 +254,19 @@ func (adapter *Mpt2SASRaidAdaptor) buildRaid(level string, devs []*baremetal.Bar return nil } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid0(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("RAID0", devs, conf) } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid1(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return adapter.buildRaid("RAID1", devs, conf) } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid5(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { return fmt.Errorf("Not impl") } -func (adapter *Mpt2SASRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *baremetal.BaremetalDiskConfig) error { +func (adapter *Mpt2SASRaidAdaptor) BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error { if len(devs) > 10 { return fmt.Errorf("RAID10 supports no more than 10 disks") } @@ -342,7 +343,7 @@ func (r *Mpt2SASRaid) GetCommand(args ...string) string { return raid.GetCommand(r.utility, args...) } -func (r *Mpt2SASRaid) PreBuildRaid(_ []*baremetal.BaremetalDiskConfig, _ int) error { +func (r *Mpt2SASRaid) PreBuildRaid(_ []*api.BaremetalDiskConfig, _ int) error { return nil } diff --git a/pkg/cloudcommon/cmdline/doc.go b/pkg/cloudcommon/cmdline/doc.go new file mode 100644 index 0000000000..af392c0b87 --- /dev/null +++ b/pkg/cloudcommon/cmdline/doc.go @@ -0,0 +1 @@ +package cmdline // import "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" diff --git a/pkg/cloudcommon/cmdline/helper.go b/pkg/cloudcommon/cmdline/helper.go new file mode 100644 index 0000000000..ee9184c081 --- /dev/null +++ b/pkg/cloudcommon/cmdline/helper.go @@ -0,0 +1,406 @@ +package cmdline + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/regutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/apis/scheduler" +) + +type resourceConfigOutput struct { + keyword string + keywordPlural string +} + +func newResouceConfigOutput(kw, kws string) *resourceConfigOutput { + return &resourceConfigOutput{keyword: kw, keywordPlural: kws} +} + +func (o resourceConfigOutput) Keyword() string { + return o.keyword +} + +func (o resourceConfigOutput) KeywordPlural() string { + return o.keywordPlural +} + +type iResourcesOutput interface { + Keyword() string + KeywordPlural() string + NewConfig() interface{} + ParseDesc(desc string, idx int) (interface{}, error) + Add(config interface{}) + Resources() interface{} +} + +func fetchResourceConfigsByJSON( + obj jsonutils.JSONObject, + output iResourcesOutput, +) error { + if obj == nil { + obj = jsonutils.NewDict() + } + keywordPlural := output.KeywordPlural() + keyword := output.Keyword() + if obj.Contains(keywordPlural) { + err := obj.Unmarshal(output.Resources(), keywordPlural) + return err + } + + configs := jsonutils.GetArrayOfPrefix(obj, keyword) + for idx, config := range configs { + configJson, ok := config.(*jsonutils.JSONDict) + resConfig := output.NewConfig() + if ok { + if err := configJson.Unmarshal(resConfig); err != nil { + return err + } + } else { + configStr, err := config.GetString() + if err != nil { + return err + } + resConfig, err = output.ParseDesc(configStr, idx) + if err != nil { + return err + } + } + output.Add(resConfig) + } + return nil +} + +type diskConfigOutput struct { + *resourceConfigOutput + disks []*compute.DiskConfig +} + +func newDiskConfigOutput() *diskConfigOutput { + return &diskConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("disk", "disks"), + } +} + +func (output *diskConfigOutput) Disks() []*compute.DiskConfig { + return output.disks +} + +func (output *diskConfigOutput) Resources() interface{} { + return &output.disks +} + +func (output *diskConfigOutput) NewConfig() interface{} { + return new(compute.DiskConfig) +} + +func (output *diskConfigOutput) Add(config interface{}) { + output.disks = append(output.disks, config.(*compute.DiskConfig)) +} + +func (output *diskConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseDiskConfig(desc, idx) +} + +func FetchDiskConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.DiskConfig, error) { + output := newDiskConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.Disks(), err +} + +type netConfigOutput struct { + *resourceConfigOutput + nets []*compute.NetworkConfig +} + +func newNetworkConfigOutput() *netConfigOutput { + return &netConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("net", "nets"), + } +} + +func (output *netConfigOutput) Networks() []*compute.NetworkConfig { + return output.nets +} + +func (output *netConfigOutput) Resources() interface{} { + return &output.nets +} + +func (output *netConfigOutput) NewConfig() interface{} { + return new(compute.NetworkConfig) +} + +func (output *netConfigOutput) Add(config interface{}) { + output.nets = append(output.nets, config.(*compute.NetworkConfig)) +} + +func (output *netConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseNetworkConfig(desc, idx) +} + +func FetchNetworkConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.NetworkConfig, error) { + output := newNetworkConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.Networks(), err +} + +type schedtagConfigOutput struct { + *resourceConfigOutput + tags []*compute.SchedtagConfig +} + +func newSchedtagConfigOutput() *schedtagConfigOutput { + return &schedtagConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("schedtag", "schedtags"), + } +} + +func (output *schedtagConfigOutput) Schedtags() []*compute.SchedtagConfig { + return output.tags +} + +func (output *schedtagConfigOutput) Resources() interface{} { + return &output.tags +} + +func (output *schedtagConfigOutput) NewConfig() interface{} { + return new(compute.SchedtagConfig) +} + +func (output *schedtagConfigOutput) Add(config interface{}) { + output.tags = append(output.tags, config.(*compute.SchedtagConfig)) +} + +func (output *schedtagConfigOutput) ParseDesc(desc string, _ int) (interface{}, error) { + return ParseSchedtagConfig(desc) +} + +func FetchSchedtagConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.SchedtagConfig, error) { + output := newSchedtagConfigOutput() + if err := fetchResourceConfigsByJSON(obj, output); err != nil { + return nil, err + } + if len(output.tags) == 0 { + // compatible with old api + schedtags, _ := obj.GetMap("aggregate_strategy") + if schedtags != nil { + for id, strategyObj := range schedtags { + strategy, _ := strategyObj.GetString() + output.tags = append(output.tags, &compute.SchedtagConfig{ + Id: id, + Strategy: strategy, + }) + } + } + } + return output.tags, nil +} + +type isoDevConfigOutput struct { + *resourceConfigOutput + devs []*compute.IsolatedDeviceConfig +} + +func newIsoDevConfigOutput() *isoDevConfigOutput { + return &isoDevConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("isolated_device", "isolated_devices"), + } +} + +func (output *isoDevConfigOutput) Devs() []*compute.IsolatedDeviceConfig { + return output.devs +} + +func (output *isoDevConfigOutput) Resources() interface{} { + return &output.devs +} + +func (output *isoDevConfigOutput) NewConfig() interface{} { + return new(compute.IsolatedDeviceConfig) +} + +func (output *isoDevConfigOutput) Add(config interface{}) { + output.devs = append(output.devs, config.(*compute.IsolatedDeviceConfig)) +} + +func (output *isoDevConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseIsolatedDevice(desc, idx) +} + +func FetchIsolatedDeviceConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.IsolatedDeviceConfig, error) { + output := newIsoDevConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.devs, err +} + +type bmDiskConfigOutput struct { + *resourceConfigOutput + configs []*compute.BaremetalDiskConfig +} + +func newbmDiskConfigOutput() *bmDiskConfigOutput { + return &bmDiskConfigOutput{ + resourceConfigOutput: newResouceConfigOutput("baremetal_disk_config", "baremetal_disk_configs"), + } +} + +func (output *bmDiskConfigOutput) Devs() []*compute.BaremetalDiskConfig { + return output.configs +} + +func (output *bmDiskConfigOutput) Resources() interface{} { + return &output.configs +} + +func (output *bmDiskConfigOutput) NewConfig() interface{} { + return new(compute.BaremetalDiskConfig) +} + +func (output *bmDiskConfigOutput) Add(config interface{}) { + output.configs = append(output.configs, config.(*compute.BaremetalDiskConfig)) +} + +func (output *bmDiskConfigOutput) ParseDesc(desc string, idx int) (interface{}, error) { + return ParseBaremetalDiskConfig(desc) +} + +func FetchBaremetalDiskConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.BaremetalDiskConfig, error) { + output := newbmDiskConfigOutput() + err := fetchResourceConfigsByJSON(obj, output) + return output.configs, err +} + +func FetchServerConfigsByJSON(obj jsonutils.JSONObject) (*compute.ServerConfigs, error) { + conf := new(compute.ServerConfigs) + if err := obj.Unmarshal(conf); err != nil { + return nil, err + } + + var err error + conf.Disks, err = FetchDiskConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.Networks, err = FetchNetworkConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.Schedtags, err = FetchSchedtagConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.IsolatedDevices, err = FetchIsolatedDeviceConfigsByJSON(obj) + if err != nil { + return nil, err + } + conf.BaremetalDiskConfigs, err = FetchBaremetalDiskConfigsByJSON(obj) + if err != nil { + return nil, err + } + return conf, nil +} + +func FetchScheduleInputByJSON(obj jsonutils.JSONObject) (*scheduler.ScheduleInput, error) { + input := new(scheduler.ScheduleInput) + err := obj.Unmarshal(input) + if err != nil { + return nil, err + } + conf := &input.ServerConfig + conf.ServerConfigs, err = FetchServerConfigsByJSON(obj) + if err != nil { + return nil, err + } + return input, nil +} + +func FetchDeployConfigsByJSON(obj jsonutils.JSONObject) ([]*compute.DeployConfig, error) { + deploys := make([]*compute.DeployConfig, 0) + if obj.Contains("deploy_configs") { + err := obj.Unmarshal(&deploys, "deploy_configs") + return deploys, err + } + for idx := 0; obj.Contains(fmt.Sprintf("deploy.%d.path", idx)); idx += 1 { + path, _ := obj.GetString(fmt.Sprintf("deploy.%d.path", idx)) + action, _ := obj.GetString(fmt.Sprintf("deploy.%d.action", idx)) + content, _ := obj.GetString(fmt.Sprintf("deploy.%d.content", idx)) + deploys = append(deploys, &compute.DeployConfig{Path: path, Action: action, Content: content}) + } + return deploys, nil +} + +func FetchServerCreateInputByJSON(obj jsonutils.JSONObject) (*compute.ServerCreateInput, error) { + input := new(compute.ServerCreateInput) + // compatible with vmem_size string + memSizeObj, _ := obj.Get("vmem_size") + if memSizeObj != nil { + if _, ok := memSizeObj.(*jsonutils.JSONString); ok { + memStr, err := memSizeObj.GetString() + if err != nil { + return nil, err + } + if !regutils.MatchSize(memStr) { + return nil, fmt.Errorf("Invalid memory size string: %s, Memory size must be number[+unit], like 256M, 1G or 256", memStr) + } + vmemSize, err := fileutils.GetSizeMb(memStr, 'M', 1024) + if err != nil { + return nil, err + } + obj.(*jsonutils.JSONDict).Set("vmem_size", jsonutils.NewInt(int64(vmemSize))) + } + } + + err := obj.Unmarshal(input) + if err != nil { + return nil, err + } + config, err := FetchServerConfigsByJSON(obj) + if err != nil { + return nil, err + } + input.ServerConfigs = config + + deployConfigs, err := FetchDeployConfigsByJSON(obj) + if err != nil { + return nil, err + } + input.DeployConfigs = deployConfigs + + if skuName := jsonutils.GetAnyString(obj, []string{"sku", "flavor", "instance_type"}); skuName != "" { + input.InstanceType = skuName + } + if keypair := jsonutils.GetAnyString(obj, []string{"keypair", "keypair_id"}); keypair != "" { + input.Keypair = keypair + } + if secgroup := jsonutils.GetAnyString(obj, []string{"secgroup", "secgroup_id", "secgrp_id"}); len(secgroup) != 0 { + input.Secgroup = secgroup + } + + return input, nil +} + +func FetchDiskCreateInputByJSON(data jsonutils.JSONObject) (*compute.DiskCreateInput, error) { + config := new(compute.DiskConfig) + input := &compute.DiskCreateInput{ + DiskConfig: config, + } + if err := data.Unmarshal(input); err != nil { + return nil, err + } + if data.Contains("disk") { + desc, err := data.GetString("disk") + config, err = ParseDiskConfig(desc, -1) + if err != nil { + return nil, err + } + input.DiskConfig = config + } + if storageId := jsonutils.GetAnyString(data, []string{"storage_id", "storage"}); storageId != "" { + input.Storage = storageId + } + return input, nil +} diff --git a/pkg/cloudcommon/cmdline/helper_test.go b/pkg/cloudcommon/cmdline/helper_test.go new file mode 100644 index 0000000000..a03e8c62ed --- /dev/null +++ b/pkg/cloudcommon/cmdline/helper_test.go @@ -0,0 +1,87 @@ +package cmdline + +import ( + "reflect" + "testing" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +func TestFetchServerConfigsByJSON(t *testing.T) { + type args struct { + obj jsonutils.JSONObject + } + tests := []struct { + name string + args args + want *compute.ServerConfigs + wantErr bool + }{ + { + name: "all configs", + args: args{jsonutils.Marshal(map[string]string{ + "disk.0": "1g:centos", + "net.0": "192.168.222.3:inf0", + "schedtag.0": "ssd:require", + "schedtag.1": "container:exclude", + "isolated_device.0": "vendor=nvidia:p400", + "baremetal_disk_config.0": "raid0:[1,2]", + })}, + want: &compute.ServerConfigs{ + Disks: []*compute.DiskConfig{ + { + Index: 0, + SizeMb: 1024, + ImageId: "centos", + Medium: "hybrid", + }, + }, + Networks: []*compute.NetworkConfig{ + { + Network: "inf0", + Address: "192.168.222.3", + }, + }, + Schedtags: []*compute.SchedtagConfig{ + { + Id: "ssd", + Strategy: "require", + }, + { + Id: "container", + Strategy: "exclude", + }, + }, + IsolatedDevices: []*compute.IsolatedDeviceConfig{ + { + Vendor: "nvidia", + Model: "p400", + }, + }, + BaremetalDiskConfigs: []*compute.BaremetalDiskConfig{ + { + Type: "hybrid", + Conf: "raid0", + Range: []int64{1, 2}, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FetchServerConfigsByJSON(tt.args.obj) + if (err != nil) != tt.wantErr { + t.Errorf("FetchServerConfigsByJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotJ := jsonutils.Marshal(got).String() + wantJ := jsonutils.Marshal(tt.want).String() + if !reflect.DeepEqual(gotJ, wantJ) { + t.Errorf("FetchServerConfigsByJSON() = %v, want %v", gotJ, wantJ) + } + }) + } +} diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go new file mode 100644 index 0000000000..23cd42e3f6 --- /dev/null +++ b/pkg/cloudcommon/cmdline/parser.go @@ -0,0 +1,349 @@ +package cmdline + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/apis/compute" +) + +var ( + ErrorEmptyDesc = errors.New("Empty description") +) + +// ParseSchedtagConfig desc format: : +func ParseSchedtagConfig(desc string) (*compute.SchedtagConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("Invalid desc: %s", desc) + } + strategy := parts[1] + if !utils.IsInStringArray(strategy, compute.STRATEGY_LIST) { + return nil, fmt.Errorf("Invalid strategy: %s", strategy) + } + return &compute.SchedtagConfig{ + Id: parts[0], + Strategy: parts[1], + }, nil +} + +// ParseResourceSchedtagConfig desc format: :: +func ParseResourceSchedtagConfig(desc string) (int, *compute.SchedtagConfig, error) { + if len(desc) == 0 { + return 0, nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + if len(parts) != 3 { + return 0, nil, fmt.Errorf("Invalid desc: %s", desc) + } + idx, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, nil, err + } + tag, err := ParseSchedtagConfig(fmt.Sprintf("%s:%s", parts[1], parts[2])) + if err != nil { + return 0, nil, err + } + return idx, tag, nil +} + +func ParseDiskConfig(diskStr string, idx int) (*compute.DiskConfig, error) { + if len(diskStr) == 0 { + return nil, ErrorEmptyDesc + } + diskConfig := new(compute.DiskConfig) + diskConfig.Index = idx + + // default backend and medium type + diskConfig.Backend = "" // STORAGE_LOCAL + diskConfig.Medium = compute.DISK_TYPE_HYBRID + + parts := strings.Split(diskStr, ":") + for _, p := range parts { + if len(p) == 0 { + continue + } + if regutils.MatchSize(p) { + diskConfig.SizeMb, _ = fileutils.GetSizeMb(p, 'M', 1024) + } else if utils.IsInStringArray(p, osprofile.FS_TYPES) { + diskConfig.Fs = p + } else if utils.IsInStringArray(p, osprofile.IMAGE_FORMAT_TYPES) { + diskConfig.Format = p + } else if utils.IsInStringArray(p, osprofile.DISK_DRIVERS) { + diskConfig.Driver = p + } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { + diskConfig.Cache = p + } else if utils.IsInStringArray(p, compute.DISK_TYPES) { + diskConfig.Medium = p + } else if utils.IsInStringArray(p, []string{compute.DISK_TYPE_VOLUME}) { + diskConfig.DiskType = p + } else if p[0] == '/' { + diskConfig.Mountpoint = p + } else if p == "autoextend" { + diskConfig.SizeMb = -1 + } else if strings.HasPrefix(p, "snapshot-") { + // HACK: use snapshot creat disk format snapshot-id + // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b + diskConfig.SnapshotId = p[len("snapshot-"):] + } else if len(p) > 0 { + diskConfig.ImageId = p + } + } + + return diskConfig, nil +} + +func ParseNetworkConfigByJSON(desc jsonutils.JSONObject, idx int) (*compute.NetworkConfig, error) { + if _, ok := desc.(*jsonutils.JSONString); ok { + descStr, _ := desc.GetString() + return ParseNetworkConfig(descStr, idx) + } + conf := new(compute.NetworkConfig) + conf.Index = idx + err := desc.Unmarshal(conf) + return conf, err +} + +func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + parts := strings.Split(desc, ":") + netConfig := new(compute.NetworkConfig) + netConfig.Index = idx + for _, p := range parts { + if len(p) == 0 { + continue + } + if regutils.MatchIP4Addr(p) { + netConfig.Address = p + } else if regutils.MatchIP6Addr(p) { + netConfig.Address6 = p + } else if regutils.MatchCompactMacAddr(p) { + netConfig.Mac = netutils.MacUnpackHex(p) + } else if strings.HasPrefix(p, "wire=") { + netConfig.Wire = p[len("wire="):] + } else if p == "[random_exit]" { + netConfig.Exit = true + } else if p == "[random]" { + netConfig.Exit = false + } else if p == "[private]" { + netConfig.Private = true + } else if p == "[reserved]" { + netConfig.Reserved = true + } else if p == "[teaming]" { + netConfig.RequireTeaming = true + } else if p == "[try-teaming]" { + netConfig.TryTeaming = true + } else if strings.HasPrefix(p, "standby-port=") { + netConfig.StandbyPortCount, _ = strconv.Atoi(p[len("standby-port="):]) + } else if strings.HasPrefix(p, "standby-addr=") { + netConfig.StandbyAddrCount, _ = strconv.Atoi(p[len("standby-addr="):]) + } else if utils.IsInStringArray(p, []string{"virtio", "e1000", "vmxnet3"}) { + netConfig.Driver = p + } else if regutils.MatchSize(p) { + bw, err := fileutils.GetSizeMb(p, 'M', 1000) + if err != nil { + return nil, err + } + netConfig.BwLimit = bw + } else if p == "[vip]" { + netConfig.Vip = true + } else if utils.IsInStringArray(p, compute.ALL_NETWORK_TYPES) { + netConfig.NetType = p + } else { + netConfig.Network = p + } + } + return netConfig, nil +} + +func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, error) { + if len(desc) == 0 { + return nil, ErrorEmptyDesc + } + if idx < 0 { + return nil, fmt.Errorf("Invalid index: %d", idx) + } + dev := new(compute.IsolatedDeviceConfig) + parts := strings.Split(desc, ":") + for _, p := range parts { + if regutils.MatchUUIDExact(p) { + dev.Id = p + } else if utils.IsInStringArray(p, compute.VALID_PASSTHROUGH_TYPES) { + dev.DevType = p + } else if strings.HasPrefix(p, "vendor=") { + dev.Vendor = p[len("vendor="):] + } else { + dev.Model = p + } + } + return dev, nil +} + +func ParseBaremetalDiskConfig(desc string) (*compute.BaremetalDiskConfig, error) { + bdc := new(compute.BaremetalDiskConfig) + bdc.Type = compute.DISK_TYPE_HYBRID + bdc.Conf = compute.DISK_CONF_NONE + bdc.Count = 0 + desc = strings.ToLower(desc) + if len(desc) == 0 { + return bdc, nil + } + + parts := strings.Split(desc, ":") + drvMap := make(map[string]string) + for _, drv := range compute.DISK_DRIVERS.List() { + drvMap[strings.ToLower(drv)] = drv + } + + for _, p := range parts { + if len(p) == 0 { + continue + } else if utils.IsInStringArray(p, compute.DISK_TYPES) { + bdc.Type = p + } else if compute.DISK_CONFS.Has(p) { + bdc.Conf = p + } else if drv, ok := drvMap[p]; ok { + bdc.Driver = drv + } else if utils.IsMatchInteger(p) { + bdc.Count, _ = strconv.ParseInt(p, 0, 0) + } else if len(p) > 2 && p[0] == '[' && p[len(p)-1] == ']' { + rg, err1 := ParseRange(p[1:(len(p) - 1)]) + if err1 != nil { + return nil, err1 + } + bdc.Range = rg + } else if len(p) > 2 && p[0] == '(' && p[len(p)-1] == ')' { + bdc.Splits = p[1 : len(p)-1] + } else if utils.HasPrefix(p, "strip") { + strip := parseStrip(p[len("strip"):], "k") + bdc.Strip = &strip + } else if utils.HasPrefix(p, "adapter") { + ada, _ := strconv.ParseInt(p[len("adapter"):], 0, 64) + pada := int(ada) + bdc.Adapter = &pada + } else if p == "ra" { + hasRA := true + bdc.RA = &hasRA + } else if p == "nora" { + noRA := false + bdc.RA = &noRA + } else if p == "wt" { + wt := true + bdc.WT = &wt + } else if p == "wb" { + wt := false + bdc.WT = &wt + } else if p == "direct" { + direct := true + bdc.Direct = &direct + } else if p == "cached" { + direct := false + bdc.Direct = &direct + } else if p == "cachedbadbbu" { + cached := true + bdc.Cachedbadbbu = &cached + } else if p == "nocachedbadbbu" { + cached := false + bdc.Cachedbadbbu = &cached + } else { + return nil, fmt.Errorf("ParseDiskConfig unkown option %q", p) + } + } + + return bdc, nil +} + +func ParseRange(rangeStr string) (ret []int64, err error) { + rss := regexp.MustCompile(`[\s,]+`).Split(rangeStr, -1) + intSet := sets.NewInt64() + + for _, rs := range rss { + r, err1 := _parseRange(rs) + if err1 != nil { + err = err1 + return + } + intSet.Insert(r...) + } + ret = intSet.List() + return +} + +// range string should be: "1-3", "3" +func _parseRange(str string) (ret []int64, err error) { + if len(str) == 0 { + return + } + + // exclude "," symbol + if len(str) == 1 && !utils.IsMatchInteger(str) { + return + } + + // add int string + if utils.IsMatchInteger(str) { + i, _ := strconv.ParseInt(str, 10, 64) + ret = append(ret, i) + return + } + + // add rang like string, "2-10" etc. + ret, err = parseRangeStr(str) + return +} + +// return bytes +func parseStrip(stripStr string, defaultSize string) int64 { + size, _ := utils.GetSize(stripStr, defaultSize, 1024) + return size / 1024 +} + +func parseRangeStr(str string) (ret []int64, err error) { + im := utils.IsMatchInteger + errGen := func(e string) error { + return fmt.Errorf("Incorrect range str: %q", e) + } + rs := strings.Split(str, "-") + if len(rs) != 2 { + err = errGen(str) + return + } + + bs, es := rs[0], rs[1] + if !im(bs) { + err = errGen(str) + return + } + if !im(es) { + err = errGen(str) + return + } + + begin, _ := strconv.ParseInt(bs, 10, 64) + end, _ := strconv.ParseInt(es, 10, 64) + + if begin > end { + begin, end = end, begin + } + + for i := begin; i <= end; i++ { + ret = append(ret, i) + } + return +} diff --git a/pkg/cloudcommon/cmdline/parser_test.go b/pkg/cloudcommon/cmdline/parser_test.go new file mode 100644 index 0000000000..807a4b6eff --- /dev/null +++ b/pkg/cloudcommon/cmdline/parser_test.go @@ -0,0 +1,177 @@ +package cmdline + +import ( + "reflect" + "testing" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/apis/compute" +) + +func TestParseSchedtagConfig(t *testing.T) { + type args struct { + desc string + } + tests := []struct { + name string + args args + want *compute.SchedtagConfig + wantErr bool + }{ + { + name: "empty input", + args: args{""}, + want: nil, + wantErr: true, + }, + { + name: "normal", + args: args{"ssd:require"}, + want: &compute.SchedtagConfig{Id: "ssd", Strategy: "require"}, + wantErr: false, + }, + { + name: "invalid strategy", + args: args{"ssd:require2"}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSchedtagConfig(tt.args.desc) + if (err != nil) != tt.wantErr { + t.Errorf("ParseSchedtagConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseSchedtagConfig() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseRange(t *testing.T) { + type args struct { + rangeStr string + } + tests := []struct { + name string + args args + wantRet []int64 + wantErr bool + }{ + { + name: "range number and , test", + args: args{"11-7,, 9-9, 10, 15, 1-2,4-5,,"}, + wantRet: []int64{1, 2, 4, 5, 7, 8, 9, 10, 11, 15}, + wantErr: false, + }, + { + name: "range,range", + args: args{"1-2,4-5"}, + wantRet: []int64{1, 2, 4, 5}, + wantErr: false, + }, + { + name: "numbers", + args: args{"1, 1"}, + wantRet: []int64{1}, + wantErr: false, + }, + { + name: "numbersErr", + args: args{"1-, 1"}, + wantRet: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRet, err := ParseRange(tt.args.rangeStr) + if (err != nil) != tt.wantErr { + t.Errorf("ParseRange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRet, tt.wantRet) { + t.Errorf("ParseRange() = %v, want %v", gotRet, tt.wantRet) + } + }) + } +} + +func TestFetchDiskConfigsByJSON(t *testing.T) { + type args struct { + obj jsonutils.JSONObject + } + + testDisks := []*compute.DiskConfig{ + { + SizeMb: 1024, + Medium: compute.DISK_TYPE_SSD, + }, + { + SizeMb: 2048, + Medium: compute.DISK_TYPE_ROTATE, + }, + } + + testDisksObj := map[string][]*compute.DiskConfig{"disks": testDisks} + + tests := []struct { + name string + args args + want []*compute.DiskConfig + wantErr bool + }{ + { + name: "empty", + args: args{nil}, + want: nil, + wantErr: false, + }, + { + name: "parse strings", + args: args{obj: jsonutils.Marshal( + map[string]string{ + "disk.0": "10g:/data1", + "disk.1": "20g:ext4:ssd", + }, + )}, + want: []*compute.DiskConfig{ + { + Index: 0, + SizeMb: 10 * 1024, + Mountpoint: "/data1", + Medium: compute.DISK_TYPE_HYBRID, + }, + { + Index: 1, + SizeMb: 20 * 1024, + Fs: "ext4", + Medium: compute.DISK_TYPE_SSD, + }, + }, + wantErr: false, + }, + { + name: "parse struct", + args: args{obj: jsonutils.Marshal(testDisksObj)}, + want: testDisks, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FetchDiskConfigsByJSON(tt.args.obj) + if (err != nil) != tt.wantErr { + t.Errorf("FetchDiskConfigsByJSON() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FetchDiskConfigsByJSON() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/compute/baremetal/diskconfig.go b/pkg/compute/baremetal/diskconfig.go index 9eae1f6808..fd1cd6d4b5 100644 --- a/pkg/compute/baremetal/diskconfig.go +++ b/pkg/compute/baremetal/diskconfig.go @@ -4,171 +4,27 @@ import ( "encoding/json" "fmt" "reflect" - "regexp" "strconv" "strings" "yunion.io/x/log" "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" ) -// return bytes -func parseStrip(stripStr string, defaultSize string) int64 { - size, _ := utils.GetSize(stripStr, defaultSize, 1024) - return size / 1024 -} - -func parseRangeStr(str string) (ret []int64, err error) { - im := utils.IsMatchInteger - errGen := func(e string) error { - return fmt.Errorf("Incorrect range str: %q", e) +func ParseDiskConfig(desc string) (api.BaremetalDiskConfig, error) { + conf, err := cmdline.ParseBaremetalDiskConfig(desc) + if err != nil { + return api.BaremetalDiskConfig{}, err } - rs := strings.Split(str, "-") - if len(rs) != 2 { - err = errGen(str) - return - } - - bs, es := rs[0], rs[1] - if !im(bs) { - err = errGen(str) - return - } - if !im(es) { - err = errGen(str) - return - } - - begin, _ := strconv.ParseInt(bs, 10, 64) - end, _ := strconv.ParseInt(es, 10, 64) - - if begin > end { - begin, end = end, begin - } - - for i := begin; i <= end; i++ { - ret = append(ret, i) - } - return -} - -// range string should be: "1-3", "3" -func _parseRange(str string) (ret []int64, err error) { - if len(str) == 0 { - return - } - - // exclude "," symbol - if len(str) == 1 && !utils.IsMatchInteger(str) { - return - } - - // add int string - if utils.IsMatchInteger(str) { - i, _ := strconv.ParseInt(str, 10, 64) - ret = append(ret, i) - return - } - - // add rang like string, "2-10" etc. - ret, err = parseRangeStr(str) - return -} - -func ParseRange(rangeStr string) (ret []int64, err error) { - rss := regexp.MustCompile(`[\s,]+`).Split(rangeStr, -1) - intSet := sets.NewInt64() - - for _, rs := range rss { - r, err1 := _parseRange(rs) - if err1 != nil { - err = err1 - return - } - intSet.Insert(r...) - } - ret = intSet.List() - return -} - -func ParseDiskConfig(desc string) (bdc BaremetalDiskConfig, err error) { - bdc.Type = DISK_TYPE_HYBRID - bdc.Conf = DISK_CONF_NONE - bdc.Count = 0 - - desc = strings.ToLower(desc) - if desc == "" { - return - } - - parts := strings.Split(desc, ":") - drvMap := make(map[string]string) - for _, drv := range DISK_DRIVERS.List() { - drvMap[strings.ToLower(drv)] = drv - } - for _, p := range parts { - if len(p) == 0 { - continue - } else if DISK_TYPES.Has(p) { - bdc.Type = p - } else if DISK_CONFS.Has(p) { - bdc.Conf = p - } else if drv, ok := drvMap[p]; ok { - bdc.Driver = drv - } else if utils.IsMatchInteger(p) { - bdc.Count, _ = strconv.ParseInt(p, 0, 0) - } else if len(p) > 2 && p[0] == '[' && p[len(p)-1] == ']' { - rg, err1 := ParseRange(p[1:(len(p) - 1)]) - if err1 != nil { - err = err1 - return - } - bdc.Range = rg - } else if len(p) > 2 && p[0] == '(' && p[len(p)-1] == ')' { - bdc.Splits = p[1 : len(p)-1] - } else if utils.HasPrefix(p, "strip") { - strip := parseStrip(p[len("strip"):], "k") - bdc.Strip = &strip - } else if utils.HasPrefix(p, "adapter") { - ada, _ := strconv.ParseInt(p[len("adapter"):], 0, 64) - pada := int(ada) - bdc.Adapter = &pada - } else if p == "ra" { - hasRA := true - bdc.RA = &hasRA - } else if p == "nora" { - noRA := false - bdc.RA = &noRA - } else if p == "wt" { - wt := true - bdc.WT = &wt - } else if p == "wb" { - wt := false - bdc.WT = &wt - } else if p == "direct" { - direct := true - bdc.Direct = &direct - } else if p == "cached" { - direct := false - bdc.Direct = &direct - } else if p == "cachedbadbbu" { - cached := true - bdc.Cachedbadbbu = &cached - } else if p == "nocachedbadbbu" { - cached := false - bdc.Cachedbadbbu = &cached - } else { - err = fmt.Errorf("ParseDiskConfig unkown option %q", p) - return - } - } - - return + return *conf, nil } func isDiskConfigStorageMatch( - config *BaremetalDiskConfig, + config *api.BaremetalDiskConfig, confDriver *string, confAdapter *int, storage *BaremetalStorage, @@ -198,7 +54,7 @@ func isDiskConfigStorageMatch( return false } -func RetrieveStorages(diskConfig *BaremetalDiskConfig, storages []*BaremetalStorage) (selected, rest []*BaremetalStorage) { +func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*BaremetalStorage) (selected, rest []*BaremetalStorage) { var confDriver *string = nil var confAdapter *int = nil @@ -259,9 +115,9 @@ func RequireEvenDisks(diskConfig string) bool { } type Layout struct { - Disks []*BaremetalStorage `json:"disks"` - Conf *BaremetalDiskConfig `json:"conf"` - Size int64 `json:"size"` + Disks []*BaremetalStorage `json:"disks"` + Conf *api.BaremetalDiskConfig `json:"conf"` + Size int64 `json:"size"` } func (l Layout) String() string { @@ -280,7 +136,7 @@ func RetrieveStorageDrivers(storages []*BaremetalStorage) sets.String { } func MeetConfig( - conf *BaremetalDiskConfig, + conf *api.BaremetalDiskConfig, storages []*BaremetalStorage, ) error { storageDrvs := RetrieveStorageDrivers(storages) @@ -417,9 +273,9 @@ func ExpandNoneConf(layouts []Layout) (ret []Layout) { return ret } -func GetLayoutRaidConfig(layouts []Layout) []*BaremetalDiskConfig { +func GetLayoutRaidConfig(layouts []Layout) []*api.BaremetalDiskConfig { var disk []*BaremetalStorage - ret := make([]*BaremetalDiskConfig, 0) + ret := make([]*api.BaremetalDiskConfig, 0) for _, layout := range layouts { if layout.Conf.Conf == DISK_CONF_NONE && sets.NewString(DISK_DRIVER_LINUX, DISK_DRIVER_PCIE).Has(layout.Disks[0].Driver) { @@ -444,10 +300,10 @@ func GetLayoutRaidConfig(layouts []Layout) []*BaremetalDiskConfig { return ret } -func CalculateLayout(confs []*BaremetalDiskConfig, storages []*BaremetalStorage) (layouts []Layout, err error) { +func CalculateLayout(confs []*api.BaremetalDiskConfig, storages []*BaremetalStorage) (layouts []Layout, err error) { var confIdx = 0 for len(storages) > 0 { - var conf *BaremetalDiskConfig + var conf *api.BaremetalDiskConfig if confIdx < len(confs) { conf = confs[confIdx] confIdx += 1 @@ -544,8 +400,8 @@ func CheckDisksAllocable(layouts []Layout, disks []*Disk) bool { return true } -func NewBaremetalDiskConfigs(dss ...string) ([]*BaremetalDiskConfig, error) { - ret := make([]*BaremetalDiskConfig, 0) +func NewBaremetalDiskConfigs(dss ...string) ([]*api.BaremetalDiskConfig, error) { + ret := make([]*api.BaremetalDiskConfig, 0) for _, ds := range dss { r, err := ParseDiskConfig(ds) if err != nil { @@ -671,7 +527,7 @@ func GetDiskConfigurations(layouts []Layout) []DiskConfiguration { type DriverAdapterDiskConfig struct { Driver string Adapter int - Configs []*BaremetalDiskConfig + Configs []*api.BaremetalDiskConfig } func GroupLayoutResultsByDriverAdapter(layouts []Layout) []*DriverAdapterDiskConfig { @@ -687,7 +543,7 @@ func GroupLayoutResultsByDriverAdapter(layouts []Layout) []*DriverAdapterDiskCon item := &DriverAdapterDiskConfig{ Driver: driver, Adapter: adapter, - Configs: []*BaremetalDiskConfig{layout.Conf}, + Configs: []*api.BaremetalDiskConfig{layout.Conf}, } ret = append(ret, item) tbl[key] = item diff --git a/pkg/compute/baremetal/diskconfig_test.go b/pkg/compute/baremetal/diskconfig_test.go index 388030f4f2..03e85fb797 100644 --- a/pkg/compute/baremetal/diskconfig_test.go +++ b/pkg/compute/baremetal/diskconfig_test.go @@ -6,6 +6,8 @@ import ( "testing" "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/compute" ) func TestParseDiskConfig(t *testing.T) { @@ -22,13 +24,13 @@ func TestParseDiskConfig(t *testing.T) { tests := []struct { name string args args - wantBdc BaremetalDiskConfig + wantBdc api.BaremetalDiskConfig wantErr bool }{ { name: "rotate:[1-2,4-5]:MegaRaid", args: args{"rotate:[1-2,4-5]:MegaRaid"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_NONE, Driver: DISK_DRIVER_MEGARAID, @@ -40,7 +42,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[1-2,4,6]:raid10:marvelraid", args: args{"rotate:[1-2,4,6]:raid10:marvelraid"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Driver: DISK_DRIVER_MARVELRAID, @@ -52,7 +54,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4,6]:raid10", args: args{"rotate:[4,6]:raid10"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -63,7 +65,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4]:raid10:(40%, )", args: args{"rotate:[4]:raid10:(40%, )"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -75,7 +77,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "[12-13]:raid1:(100g,32g,):adapter0:strip64k", args: args{"[12-13]:raid1:(100g,32g,):adapter0:strip64k"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID1, Count: 0, @@ -89,7 +91,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "6:raid5:adapter2", args: args{"6:raid5:adapter2"}, - wantBdc: BaremetalDiskConfig{ + wantBdc: api.BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID5, Count: 6, @@ -112,55 +114,6 @@ func TestParseDiskConfig(t *testing.T) { } } -func TestParseRange(t *testing.T) { - type args struct { - rangeStr string - } - tests := []struct { - name string - args args - wantRet []int64 - wantErr bool - }{ - { - name: "range number and , test", - args: args{"11-7,, 9-9, 10, 15, 1-2,4-5,,"}, - wantRet: []int64{1, 2, 4, 5, 7, 8, 9, 10, 11, 15}, - wantErr: false, - }, - { - name: "range,range", - args: args{"1-2,4-5"}, - wantRet: []int64{1, 2, 4, 5}, - wantErr: false, - }, - { - name: "numbers", - args: args{"1, 1"}, - wantRet: []int64{1}, - wantErr: false, - }, - { - name: "numbersErr", - args: args{"1-, 1"}, - wantRet: nil, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotRet, err := ParseRange(tt.args.rangeStr) - if (err != nil) != tt.wantErr { - t.Errorf("ParseRange() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(gotRet, tt.wantRet) { - t.Errorf("ParseRange() = %v, want %v", gotRet, tt.wantRet) - } - }) - } -} - var ( testStorages = []*BaremetalStorage{ // 0-11 disk on adapter2 @@ -516,7 +469,7 @@ func TestCheckDisksAllocable(t *testing.T) { } layout, err := CalculateLayout(confs, testStorages) - defaultLayout, err := CalculateLayout([]*BaremetalDiskConfig{&BaremetalDefaultDiskConfig}, testStorages[12:]) + defaultLayout, err := CalculateLayout([]*api.BaremetalDiskConfig{&BaremetalDefaultDiskConfig}, testStorages[12:]) if err != nil { t.Fatalf("CalculateLayout err: %v", err) } diff --git a/pkg/compute/baremetal/types.go b/pkg/compute/baremetal/types.go index 384178d9c6..9f8bc930b2 100644 --- a/pkg/compute/baremetal/types.go +++ b/pkg/compute/baremetal/types.go @@ -3,66 +3,45 @@ package baremetal import ( "errors" - "yunion.io/x/pkg/util/sets" + api "yunion.io/x/onecloud/pkg/apis/compute" ) const ( - DISK_CONF_RAID0 = "raid0" - DISK_CONF_RAID1 = "raid1" - DISK_CONF_RAID5 = "raid5" - DISK_CONF_RAID10 = "raid10" - DISK_CONF_NONE = "none" + DISK_CONF_RAID0 = api.DISK_CONF_RAID0 + DISK_CONF_RAID1 = api.DISK_CONF_RAID1 + DISK_CONF_RAID5 = api.DISK_CONF_RAID5 + DISK_CONF_RAID10 = api.DISK_CONF_RAID10 + DISK_CONF_NONE = api.DISK_CONF_NONE - DEFAULT_DISK_CONF = DISK_CONF_NONE + DEFAULT_DISK_CONF = api.DEFAULT_DISK_CONF - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" - DISK_TYPE_HYBRID = "hybrid" + DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE + DISK_TYPE_SSD = api.DISK_TYPE_SSD + DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID - DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + DEFAULT_DISK_TYPE = api.DEFAULT_DISK_TYPE - DISK_DRIVER_MEGARAID = "MegaRaid" - DISK_DRIVER_LINUX = "Linux" - DISK_DRIVER_HPSARAID = "HPSARaid" - DISK_DRIVER_MPT2SAS = "Mpt2SAS" - DISK_DRIVER_MARVELRAID = "MarvelRaid" - DISK_DRIVER_PCIE = "PCIE" + DISK_DRIVER_MEGARAID = api.DISK_DRIVER_MEGARAID + DISK_DRIVER_LINUX = api.DISK_DRIVER_LINUX + DISK_DRIVER_HPSARAID = api.DISK_DRIVER_HPSARAID + DISK_DRIVER_MPT2SAS = api.DISK_DRIVER_MPT2SAS + DISK_DRIVER_MARVELRAID = api.DISK_DRIVER_MARVELRAID + DISK_DRIVER_PCIE = api.DISK_DRIVER_PCIE - HDD_DISK_SPEC_TYPE = "HDD" - SSD_DISK_SPEC_TYPE = "SSD" + HDD_DISK_SPEC_TYPE = api.HDD_DISK_SPEC_TYPE + SSD_DISK_SPEC_TYPE = api.SSD_DISK_SPEC_TYPE ) var ( - BaremetalDefaultDiskConfig = BaremetalDiskConfig{ - Type: DISK_TYPE_HYBRID, - Conf: DISK_CONF_NONE, - Count: 0, - } + BaremetalDefaultDiskConfig = api.BaremetalDefaultDiskConfig - DISK_CONFS = sets.NewString( - DISK_CONF_RAID0, - DISK_CONF_RAID1, - DISK_CONF_RAID5, - DISK_CONF_RAID10, - DISK_CONF_NONE, - ) + DISK_CONFS = api.DISK_CONFS - DISK_TYPES = sets.NewString( - DISK_TYPE_ROTATE, - DISK_TYPE_SSD, - DISK_TYPE_HYBRID, - ) + DISK_TYPES = api.DISK_TYPES - DISK_DRIVERS_RAID = sets.NewString( - DISK_DRIVER_MEGARAID, - DISK_DRIVER_HPSARAID, - DISK_DRIVER_MPT2SAS, - DISK_DRIVER_MARVELRAID, - ) + DISK_DRIVERS_RAID = api.DISK_DRIVERS_RAID - DISK_DRIVERS = sets.NewString( - DISK_DRIVER_LINUX, - DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) + DISK_DRIVERS = api.DISK_DRIVERS ) var ( @@ -99,24 +78,6 @@ func (s BaremetalStorage) GetBlock() int64 { return s.Block } -type BaremetalDiskConfig struct { - // disk type - Type string `json:"type"` - // raid config - Conf string `json:"conf"` - Count int64 `json:"count"` - Range []int64 `json:"range"` - Splits string `json:"splits"` - Size []int64 `json:"size"` - Adapter *int `json:"adapter,omitempty"` - Driver string `json:"driver"` - Cachedbadbbu *bool `json:"cachedbadbbu,omitempty"` - Strip *int64 `json:"strip,omitempty"` - RA *bool `json:"ra,omitempty"` - WT *bool `json:"wt,omitempty"` - Direct *bool `json:"direct,omitempty"` -} - type Disk struct { Backend string `json:"backend"` ImageID string `json:"image_id"` diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index dda30729f8..95bc091a6b 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -36,6 +36,16 @@ func (self *SAliyunGuestDriver) GetMinimalSysDiskSizeGb() int { return 20 } +func (self *SAliyunGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_CLOUD_EFFICIENCY, + models.STORAGE_CLOUD_SSD, + models.STORAGE_CLOUD_ESSD, + models.STORAGE_PUBLIC_CLOUD, + models.STORAGE_EPHEMERAL_SSD, + } +} + func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -43,13 +53,7 @@ func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend st return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_CLOUD_EFFICIENCY, - models.STORAGE_CLOUD_SSD, - models.STORAGE_CLOUD_ESSD, - models.STORAGE_PUBLIC_CLOUD, - models.STORAGE_EPHEMERAL_SSD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -96,19 +100,15 @@ func (self *SAliyunGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - for i := 0; data.Contains(fmt.Sprintf("disk.%d", i)); i++ { - disk := models.SDiskConfig{} - if err := data.Unmarshal(&disk, fmt.Sprintf("disk.%d", i)); err != nil { - return nil, httperrors.NewInputParameterError("invalid diskinfo of index %d", i) - } + for i, disk := range input.Disks { if i == 0 && (disk.SizeMb < 20*1024 || disk.SizeMb > 500*1024) { return nil, httperrors.NewInputParameterError("The system disk size must be in the range of 20GB ~ 500Gb") } @@ -127,7 +127,7 @@ func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred } } } - return data, nil + return input, nil } func (self *SAliyunGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index f473b4c871..1d50455754 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -69,6 +69,16 @@ func (self *SAwsGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SAwsGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_GP2_SSD, + models.STORAGE_IO1_SSD, + models.STORAGE_ST1_HDD, + models.STORAGE_SC1_HDD, + models.STORAGE_STANDARD_HDD, + } +} + func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -77,13 +87,7 @@ func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend strin } } - for _, stype := range []string{ - models.STORAGE_GP2_SSD, - models.STORAGE_IO1_SSD, - models.STORAGE_ST1_HDD, - models.STORAGE_SC1_HDD, - models.STORAGE_STANDARD_HDD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -117,8 +121,8 @@ func (self *SAwsGuestDriver) RequestDetachDisk(ctx context.Context, guest *model return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAwsGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) } func (self *SAwsGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index e1f558ab61..aef2e4b74e 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -9,6 +9,8 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -40,6 +42,14 @@ func (self *SAzureGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SAzureGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_STANDARD_LRS, + models.STORAGE_STANDARDSSD_LRS, + models.STORAGE_PREMIUM_LRS, + } +} + func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -47,11 +57,7 @@ func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend str return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_STANDARD_LRS, - models.STORAGE_STANDARDSSD_LRS, - models.STORAGE_PREMIUM_LRS, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -102,26 +108,30 @@ func (self *SAzureGuestDriver) RequestDetachDisk(ctx context.Context, guest *mod return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SAzureGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - return data, nil + return input, nil } func (self *SAzureGuestDriver) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) + input, err := cmdline.FetchServerCreateInputByJSON(data) if err != nil { return nil, err } - if data.Contains("name") { + input, err = self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) + if err != nil { + return nil, err + } + if input.Name != "" { return nil, httperrors.NewInputParameterError("cannot support change azure instance name") } - return data, nil + return input.JSON(input), nil } func (self *SAzureGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index 4caebf8a72..101fb44c68 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -55,32 +56,21 @@ func (self *SBaremetalGuestDriver) GetMaxVMemSizeGB() int { return 4096 } -func (self *SBaremetalGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params *jsonutils.JSONDict) error { +func (self *SBaremetalGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, confs []*api.BaremetalDiskConfig) error { baremetalStorage := models.ConvertStorageInfo2BaremetalStorages(host.StorageInfo) if baremetalStorage == nil { return fmt.Errorf("Convert storage info error") } - confs, err := params.GetArray("baremetal_disk_config") - if err != nil { + if len(confs) == 0 { parsedConf, _ := baremetal.ParseDiskConfig("") - nConfs := []*baremetal.BaremetalDiskConfig{&parsedConf} + nConfs := []*api.BaremetalDiskConfig{&parsedConf} layouts, err := baremetal.CalculateLayout(nConfs, baremetalStorage) if err != nil { return err } return host.UpdateDiskConfig(userCred, layouts) } else { - var nConfs = make([]*baremetal.BaremetalDiskConfig, 0) - for i := 0; i < len(confs); i++ { - parsedConf := &baremetal.BaremetalDiskConfig{} - err := confs[i].Unmarshal(parsedConf) - if err != nil { - log.Errorln(err) - return err - } - nConfs = append(nConfs, parsedConf) - } - layouts, err := baremetal.CalculateLayout(nConfs, baremetalStorage) + layouts, err := baremetal.CalculateLayout(confs, baremetalStorage) if err != nil { return err } @@ -104,7 +94,7 @@ func (self *SBaremetalGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk return httperrors.NewUnsupportOperationError("Cannot resize disk for baremtal") } -func (self *SBaremetalGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { +func (self *SBaremetalGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { netifs, net := host.GetNetinterfacesWithIdAndCredential(netConfig.Network, userCred, netConfig.Reserved) if netifs != nil { nicCnt := 1 @@ -135,7 +125,7 @@ func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []string { return []string{models.NETWORK_TYPE_BAREMETAL, models.NETWORK_TYPE_GUEST} } -func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { +func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { netifs := host.GetNetInterfaces() netsAvaiable := make([]models.SNetwork, 0) netifIndexs := make(map[string][]models.SNetInterface, 0) @@ -203,6 +193,10 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct return nil, fmt.Errorf("No appropriate host virtual network...") } +func (self *SBaremetalGuestDriver) GetStorageTypes() []string { + return nil +} + func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { bs := host.GetBaremetalstorage() if bs == nil { @@ -298,35 +292,37 @@ func (self *SBaremetalGuestDriver) StartGuestSyncstatusTask(guest *models.SGuest return nil } -func (self *SBaremetalGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - var confs = make([]baremetal.BaremetalDiskConfig, 0) - jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") - for i := 0; i < len(jsonArray); i += 1 { - // for data.Contains(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) { - desc, _ := jsonArray[i].GetString() // data.GetString(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) - data.Remove(fmt.Sprintf("baremetal_disk_config.%d", i)) - bmConf, err := baremetal.ParseDiskConfig(desc) - if err != nil { - return nil, httperrors.NewInputParameterError("baremetal_disk_config.%d", i) - } - confs = append(confs, bmConf) - } - if len(confs) == 0 { - bmConf, _ := baremetal.ParseDiskConfig("") - confs = append(confs, bmConf) - } - data.Set("baremetal_disk_config", jsonutils.Marshal(confs)) - if !data.Contains("disk.0") { +func (self *SBaremetalGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + /*var confs = make([]api.BaremetalDiskConfig, 0)*/ + //jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") + //for i := 0; i < len(jsonArray); i += 1 { + //// for data.Contains(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) { + //desc, _ := jsonArray[i].GetString() // data.GetString(fmt.Sprintf("baremetal_disk_config.%d", len(confs))) + //data.Remove(fmt.Sprintf("baremetal_disk_config.%d", i)) + //bmConf, err := baremetal.ParseDiskConfig(desc) + //if err != nil { + //return nil, httperrors.NewInputParameterError("baremetal_disk_config.%d", i) + //} + //confs = append(confs, bmConf) + //} + //if len(confs) == 0 { + //bmConf, _ := baremetal.ParseDiskConfig("") + //confs = append(confs, bmConf) + //} + /*data.Set("baremetal_disk_config", jsonutils.Marshal(confs))*/ + if len(input.Disks) <= 0 { return nil, httperrors.NewInputParameterError("Root disk must be present") } - disk0, _ := data.Get("disk.0") - if !disk0.Contains("image_id") { + + disk0 := input.Disks[0] + + if disk0.ImageId == "" { return nil, httperrors.NewInputParameterError("Root disk must have templete") } - return data, nil + return input, nil } -func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { if host.HostType != models.HOST_TYPE_BAREMETAL || !host.IsBaremetal { return nil, httperrors.NewInputParameterError("Host %s is not a baremetal", bmName) } @@ -336,9 +332,9 @@ func (self *SBaremetalGuestDriver) ValidateCreateHostData(ctx context.Context, u if host.GetBaremetalServer() != nil { return nil, httperrors.NewInsufficientResourceError("Baremetal %s is occupied", bmName) } - data.Set("vmem_size", jsonutils.NewInt(int64(host.MemSize))) - data.Set("vcpu_count", jsonutils.NewInt(int64(host.CpuCount))) - return data, nil + input.VmemSize = host.MemSize + input.VcpuCount = int(host.CpuCount) + return input, nil } func (self *SBaremetalGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { diff --git a/pkg/compute/guestdrivers/container.go b/pkg/compute/guestdrivers/container.go index 0be43be85b..d36fb073e6 100644 --- a/pkg/compute/guestdrivers/container.go +++ b/pkg/compute/guestdrivers/container.go @@ -28,20 +28,6 @@ func init() { models.RegisterGuestDriver(&driver) } -func (self *SContainerDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - vmemSize, vcpuCount, err := models.ValidateMemCpuData(data) - if err != nil { - return nil, err - } - if vmemSize > 0 { - data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") - } - if vcpuCount > 0 { - data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") - } - return data, nil -} - func (self *SContainerDriver) newUnsupportOperationError(option string) error { return httperrors.NewUnsupportOperationError("Container not support %s", option) } diff --git a/pkg/compute/guestdrivers/huawei.go b/pkg/compute/guestdrivers/huawei.go index 5a3831cedf..8d12b45d59 100644 --- a/pkg/compute/guestdrivers/huawei.go +++ b/pkg/compute/guestdrivers/huawei.go @@ -4,13 +4,11 @@ import ( "context" "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" ) @@ -35,6 +33,10 @@ func (self *SHuaweiGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } +func (self *SHuaweiGuestDriver) GetStorageTypes() []string { + return []string{models.STORAGE_HUAWEI_SATA, models.STORAGE_HUAWEI_SAS, models.STORAGE_HUAWEI_SSD} +} + func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i += 1 { @@ -43,7 +45,7 @@ func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend st } } - for _, stype := range []string{models.STORAGE_HUAWEI_SATA, models.STORAGE_HUAWEI_SAS, models.STORAGE_HUAWEI_SSD} { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i += 1 { if storages[i].StorageType == stype { return &storages[i] @@ -77,10 +79,6 @@ func (self *SHuaweiGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SHuaweiGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) -} - func (self *SHuaweiGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { if !utils.IsInStringArray(guest.Status, []string{models.VM_RUNNING, models.VM_READY}) { return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 3594b06d32..7f1c8466ee 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -348,7 +349,9 @@ func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *mo } func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), task.GetParams(), task.GetTaskId()) + input := new(api.ServerCreateInput) + task.GetParams().Unmarshal(input) + return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), input.Disks, task.GetTaskId()) } func (self *SKVMGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { diff --git a/pkg/compute/guestdrivers/openstack.go b/pkg/compute/guestdrivers/openstack.go index 66522963ec..e96b1e6bfe 100644 --- a/pkg/compute/guestdrivers/openstack.go +++ b/pkg/compute/guestdrivers/openstack.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/compare" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -42,6 +43,10 @@ func (self *SOpenStackGuestDriver) GetMinimalSysDiskSizeGb() int { return options.Options.DefaultDiskSizeMB / 1024 } +func (self *SOpenStackGuestDriver) GetStorageTypes() []string { + return []string{models.STORAGE_OPENSTACK_ISCSI} +} + func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i++ { @@ -49,7 +54,7 @@ func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend return &storages[i] } } - for _, stype := range []string{models.STORAGE_OPENSTACK_ISCSI} { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i++ { if storages[i].StorageType == stype { return &storages[i] @@ -83,15 +88,16 @@ func (self *SOpenStackGuestDriver) GetDeployStatus() ([]string, error) { return []string{models.VM_RUNNING}, nil } -func (self *SOpenStackGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SOpenStackGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + var err error + input, err = self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) >= 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - return data, nil + return input, nil } func (self *SOpenStackGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index 79e8781542..6586791324 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -38,6 +39,16 @@ func (self *SQcloudGuestDriver) GetMinimalSysDiskSizeGb() int { return 50 } +func (self *SQcloudGuestDriver) GetStorageTypes() []string { + return []string{ + models.STORAGE_CLOUD_BASIC, + models.STORAGE_CLOUD_PREMIUM, + models.STORAGE_CLOUD_SSD, + models.STORAGE_LOCAL_BASIC, + models.STORAGE_LOCAL_SSD, + } +} + func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { storages := host.GetAttachedStorages("") for i := 0; i < len(storages); i++ { @@ -45,13 +56,7 @@ func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend st return &storages[i] } } - for _, stype := range []string{ - models.STORAGE_CLOUD_BASIC, - models.STORAGE_CLOUD_PREMIUM, - models.STORAGE_CLOUD_SSD, - models.STORAGE_LOCAL_BASIC, - models.STORAGE_LOCAL_SSD, - } { + for _, stype := range self.GetStorageTypes() { for i := 0; i < len(storages); i++ { if storages[i].StorageType == stype { return &storages[i] @@ -99,18 +104,16 @@ func (self *SQcloudGuestDriver) RequestDetachDisk(ctx context.Context, guest *mo return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } -func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) +func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - if data.Contains("net.0") && data.Contains("net.1") { + if len(input.Networks) > 2 { return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") } - sysDisk := models.SDiskConfig{} - if err := data.Unmarshal(&sysDisk, "disk.0"); err != nil { - return nil, fmt.Errorf("Missing system disk information") - } + + sysDisk := input.Disks[0] switch sysDisk.Backend { case models.STORAGE_CLOUD_BASIC, models.STORAGE_CLOUD_SSD: if sysDisk.SizeMb > 500*1024 { @@ -122,11 +125,8 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred } } - for i := 1; data.Contains(fmt.Sprintf("disk.%d", i)); i++ { - disk := models.SDiskConfig{} - if err := data.Unmarshal(&disk, fmt.Sprintf("disk.%d", i)); err != nil { - return nil, httperrors.NewInputParameterError("invalid diskinfo of index %d", i) - } + for i := 1; i < len(input.Disks); i++ { + disk := input.Disks[i] switch disk.Backend { case models.STORAGE_CLOUD_BASIC: if disk.SizeMb < 10*1024 || disk.SizeMb > 16000*1024 { @@ -142,7 +142,7 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred } } } - return data, nil + return input, nil } func (self *SQcloudGuestDriver) GetGuestInitialStateAfterCreate() string { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index ea7db64abe..6b903da128 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -27,12 +28,12 @@ func (self *SVirtualizedGuestDriver) GetMaxVMemSizeGB() int { return 512 } -func (self *SVirtualizedGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params *jsonutils.JSONDict) error { +func (self *SVirtualizedGuestDriver) PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *models.SHost, params []*api.BaremetalDiskConfig) error { // do nothing return nil } -func (self *SVirtualizedGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { +func (self *SVirtualizedGuestDriver) GetNamedNetworkConfiguration(guest *models.SGuest, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig) (*models.SNetwork, []models.SNicConfig, models.IPAddlocationDirection) { net, _ := host.GetNetworkWithIdAndCredential(netConfig.Network, userCred, netConfig.Reserved) nicConfs := []models.SNicConfig{ { @@ -55,7 +56,7 @@ func (self *SVirtualizedGuestDriver) GetRandomNetworkTypes() []string { return []string{models.NETWORK_TYPE_GUEST} } -func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *models.SNetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { +func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { wirePattern = regexp.MustCompile(netConfig.Wire) @@ -115,6 +116,10 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, return gn, err } +func (self *SVirtualizedGuestDriver) GetStorageTypes() []string { + return nil +} + func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { return host.GetLeastUsedStorage(backend) } @@ -183,24 +188,24 @@ func (self *SVirtualizedGuestDriver) RequestStopGuestForDelete(ctx context.Conte return nil } -func (self *SVirtualizedGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - return data, nil +func (self *SVirtualizedGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + return input, nil } -func (self *SVirtualizedGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (self *SVirtualizedGuestDriver) ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *models.SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { if host.HostStatus != models.HOST_ONLINE { return nil, httperrors.NewInvalidStatusError("Host %s is not online", bmName) } - data.Add(jsonutils.NewString(host.Id), "prefer_host_id") + input.PreferHost = host.Id if host.IsPrepaidRecycle() { - data.Set("vmem_size", jsonutils.NewInt(int64(host.MemSize))) - data.Set("vcpu_count", jsonutils.NewInt(int64(host.CpuCount))) + input.VmemSize = host.MemSize + input.VcpuCount = int(host.CpuCount) if host.GetGuestCount() >= 1 { return nil, httperrors.NewInsufficientResourceError("host has been occupied") } } - return data, nil + return input, nil } func (self *SVirtualizedGuestDriver) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, data *jsonutils.JSONDict) error { diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 417232e60c..6af7fb6b33 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/baremetal" @@ -156,17 +157,17 @@ func (self *SBaseHostDriver) GetRaidScheme(host *models.SHost, raid string) (str candidates = []string{raid} } } - var conf []*baremetal.BaremetalDiskConfig + var conf []*api.BaremetalDiskConfig for i := 0; i < len(candidates); i++ { if candidates[i] == baremetal.DISK_CONF_NONE { - conf = []*baremetal.BaremetalDiskConfig{} + conf = []*api.BaremetalDiskConfig{} } else { parsedConf, err := baremetal.ParseDiskConfig(candidates[i]) if err != nil { log.Errorf("try raid %s failed: %s", candidates[i], err.Error()) return "", err } - conf = []*baremetal.BaremetalDiskConfig{&parsedConf} + conf = []*api.BaremetalDiskConfig{&parsedConf} } baremetalStorage := models.ConvertStorageInfo2BaremetalStorages(host.StorageInfo) if baremetalStorage == nil { diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index e551cb61f2..84d4d7f1ae 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -341,6 +341,7 @@ func (self *SKVMHostDriver) PrepareConvert(host *models.SHost, image, raid strin if err != nil { return nil, err } + // TODO: use api.ServerCreateInput var sysSize = "60g" if data.Contains("baremetal_disk_config.0") { jsonArray := jsonutils.GetArrayOfPrefix(data, "baremetal_disk_config") diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 2b757d7f6f..966cc2dbb1 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -13,12 +13,12 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/fileutils" - "yunion.io/x/pkg/util/osprofile" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -33,40 +33,40 @@ import ( ) const ( - DISK_INIT = "init" - DISK_REBUILD = "rebuild" - DISK_ALLOC_FAILED = "alloc_failed" - DISK_STARTALLOC = "start_alloc" - DISK_BACKUP_STARTALLOC = "backup_start_alloc" - DISK_BACKUP_ALLOC_FAILED = "backup_alloc_failed" - DISK_ALLOCATING = "allocating" - DISK_READY = "ready" - DISK_RESET = "reset" - DISK_DEALLOC = "deallocating" - DISK_DEALLOC_FAILED = "dealloc_failed" - DISK_UNKNOWN = "unknown" - DISK_DETACHING = "detaching" - DISK_ATTACHING = "attaching" + DISK_INIT = api.DISK_INIT + DISK_REBUILD = api.DISK_REBUILD + DISK_ALLOC_FAILED = api.DISK_ALLOC_FAILED + DISK_STARTALLOC = api.DISK_STARTALLOC + DISK_BACKUP_STARTALLOC = api.DISK_BACKUP_STARTALLOC + DISK_BACKUP_ALLOC_FAILED = api.DISK_BACKUP_ALLOC_FAILED + DISK_ALLOCATING = api.DISK_ALLOCATING + DISK_READY = api.DISK_READY + DISK_RESET = api.DISK_RESET + DISK_DEALLOC = api.DISK_DEALLOC + DISK_DEALLOC_FAILED = api.DISK_DEALLOC_FAILED + DISK_UNKNOWN = api.DISK_UNKNOWN + DISK_DETACHING = api.DISK_DETACHING + DISK_ATTACHING = api.DISK_ATTACHING - DISK_START_SAVE = "start_save" - DISK_SAVING = "saving" + DISK_START_SAVE = api.DISK_START_SAVE + DISK_SAVING = api.DISK_SAVING - DISK_START_RESIZE = "start_resize" - DISK_RESIZING = "resizing" + DISK_START_RESIZE = api.DISK_START_RESIZE + DISK_RESIZING = api.DISK_RESIZING - DISK_START_MIGRATE = "start_migrate" - DISK_POST_MIGRATE = "post_migrate" - DISK_MIGRATING = "migrating" + DISK_START_MIGRATE = api.DISK_START_MIGRATE + DISK_POST_MIGRATE = api.DISK_POST_MIGRATE + DISK_MIGRATING = api.DISK_MIGRATING - DISK_START_SNAPSHOT = "start_snapshot" - DISK_SNAPSHOTING = "snapshoting" + DISK_START_SNAPSHOT = api.DISK_START_SNAPSHOT + DISK_SNAPSHOTING = api.DISK_SNAPSHOTING - DISK_TYPE_SYS = "sys" - DISK_TYPE_SWAP = "swap" - DISK_TYPE_DATA = "data" - DISK_TYPE_VOLUME = "volume" + DISK_TYPE_SYS = api.DISK_TYPE_SYS + DISK_TYPE_SWAP = api.DISK_TYPE_SWAP + DISK_TYPE_DATA = api.DISK_TYPE_DATA + DISK_TYPE_VOLUME = api.DISK_TYPE_VOLUME - DISK_BACKING_IMAGE = "image" + DISK_BACKING_IMAGE = api.DISK_BACKING_IMAGE ) type SDiskManager struct { @@ -344,12 +344,11 @@ func (self *SDisk) GetRuningGuestCount() int { } func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - diskConfig := SDiskConfig{} - if err := data.Unmarshal(&diskConfig, "disk"); err != nil { + input := new(api.DiskCreateInput) + if err := data.Unmarshal(input); err != nil { return err - } else { - self.fetchDiskInfo(&diskConfig) } + self.fetchDiskInfo(input.DiskConfig) return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) } @@ -379,17 +378,18 @@ func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok } func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - disk, err := data.Get("disk") + input, err := cmdline.FetchDiskCreateInputByJSON(data) if err != nil { - return nil, httperrors.NewMissingParameterError("disk") + return nil, httperrors.NewInputParameterError("parse disk input: %v", err) } - - diskConfig, err := parseDiskInfo(ctx, userCred, disk) + diskConfig := input.DiskConfig + diskConfig, err = parseDiskInfo(ctx, userCred, diskConfig) if err != nil { return nil, err } + input.Project = ownerProjId - storageID := jsonutils.GetAnyString(data, []string{"storage_id", "storage"}) + storageID := input.Storage if storageID != "" { storageObj, err := StorageManager.FetchByIdOrName(nil, storageID) if err != nil { @@ -404,16 +404,15 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc if err != nil { return nil, err } - data.Add(jsonutils.NewString(storage.Id), "storage_id") + input.Storage = storage.Id } else { diskConfig.Backend = STORAGE_LOCAL - hypervisor, _ := data.GetString("hypervisor") - data, err = ValidateScheduleCreateData(ctx, userCred, data, hypervisor) + serverInput, err := ValidateScheduleCreateData(ctx, userCred, input.ToServerCreateInput(), input.Hypervisor) if err != nil { return nil, err } + input = serverInput.ToDiskCreateInput() } - data.Add(jsonutils.Marshal(diskConfig), "disk") if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil { return nil, err @@ -422,10 +421,10 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { return nil, httperrors.NewOutOfQuotaError("%s", err) } - return data, nil + return input.JSON(input), nil } -func (manager *SDiskManager) validateDiskOnStorage(diskConfig *SDiskConfig, storage *SStorage) error { +func (manager *SDiskManager) validateDiskOnStorage(diskConfig *api.DiskConfig, storage *SStorage) error { if !storage.Enabled { return httperrors.NewInputParameterError("Cannot create disk with disabled storage[%s]", storage.Name) } @@ -455,7 +454,27 @@ func (manager *SDiskManager) validateDiskOnStorage(diskConfig *SDiskConfig, stor return nil } -func (disk *SDisk) SetStorageByHost(hostId string, diskConfig *SDiskConfig) error { +func (disk *SDisk) SetStorage(storageId string, diskConfig *api.DiskConfig) error { + backend := diskConfig.Backend + if backend == "" { + return fmt.Errorf("Backend is empty") + } + storage := StorageManager.FetchStorageById(storageId) + if storage == nil { + return fmt.Errorf("Not found backend %s storage %s", backend, storageId) + } + err := DiskManager.validateDiskOnStorage(diskConfig, storage) + if err != nil { + return err + } + _, err = db.Update(disk, func() error { + disk.StorageId = storage.Id + return nil + }) + return err +} + +func (disk *SDisk) SetStorageByHost(hostId string, diskConfig *api.DiskConfig) error { host := HostManager.FetchHostById(hostId) backend := diskConfig.Backend if backend == "" { @@ -495,16 +514,16 @@ func getDiskResourceRequirements(ctx context.Context, userCred mcclient.TokenCre } } -func (manager *SDiskManager) convertToBatchCreateData(data jsonutils.JSONObject) *jsonutils.JSONDict { +/*func (manager *SDiskManager) convertToBatchCreateData(data jsonutils.JSONObject) *jsonutils.JSONDict { diskConfig, _ := data.Get("disk") newData := data.(*jsonutils.JSONDict).CopyExcludes("disk") newData.Add(diskConfig, "disk.0") return newData -} +}*/ func (manager *SDiskManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { pendingUsage := getDiskResourceRequirements(ctx, userCred, data, len(items)) - RunBatchCreateTask(ctx, items, userCred, manager.convertToBatchCreateData(data), pendingUsage, "DiskBatchCreateTask") + RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "DiskBatchCreateTask") } func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { @@ -1142,97 +1161,28 @@ func totalDiskSize(projectId string, active tristate.TriState, ready tristate.Tr return size } -type SDiskConfig struct { - ImageId string `json:"image_id"` - - SnapshotId string `json:"snapshot_id"` - DiskType string `json:"disk_type"` // sys, data, swap, volume - - // ImageDiskFormat string - SizeMb int `json:"size"` // MB - Fs string `json:"fs"` // file system - Format string `json:"format"` // - Driver string `json:"driver"` // - Cache string `json:"cache"` // - Mountpoint string `json:"mountpoint"` // - Backend string `json:"backend"` // stroageType - Medium string `json:"medium"` - ImageProperties map[string]string `json:"image_properties"` - - Schedtags map[string]string `json:"schedtags"` - - DiskId string `json:"-"` // import only -} - -func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SDiskConfig, error) { - diskConfig := SDiskConfig{} - - diskJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := diskJson.Unmarshal(&diskConfig) - if err != nil { +func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info *api.DiskConfig) (*api.DiskConfig, error) { + if info.SnapshotId != "" { + if err := fillDiskConfigBySnapshot(userCred, info, info.SnapshotId); err != nil { return nil, err } - return &diskConfig, nil } - - // default backend and medium type - diskConfig.Backend = "" // STORAGE_LOCAL - diskConfig.Medium = DISK_TYPE_HYBRID - - diskStr, err := info.GetString() - if err != nil { - log.Errorf("invalid diskinfo format %s", err) - return nil, err - } - parts := strings.Split(diskStr, ":") - for _, p := range parts { - if len(p) == 0 { - continue - } - if regutils.MatchSize(p) { - diskConfig.SizeMb, _ = fileutils.GetSizeMb(p, 'M', 1024) - } else if utils.IsInStringArray(p, osprofile.FS_TYPES) { - diskConfig.Fs = p - } else if utils.IsInStringArray(p, osprofile.IMAGE_FORMAT_TYPES) { - diskConfig.Format = p - } else if utils.IsInStringArray(p, osprofile.DISK_DRIVERS) { - diskConfig.Driver = p - } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { - diskConfig.Cache = p - } else if utils.IsInStringArray(p, DISK_TYPES) { - diskConfig.Medium = p - } else if utils.IsInStringArray(p, []string{DISK_TYPE_VOLUME}) { - diskConfig.DiskType = p - } else if p[0] == '/' { - diskConfig.Mountpoint = p - } else if p == "autoextend" { - diskConfig.SizeMb = -1 - } else if storageType, exist := StorageManager.IsStorageTypeExist(p); exist { - diskConfig.Backend = storageType - } else if strings.HasPrefix(p, "snapshot-") { - // HACK: use snapshot creat disk format snapshot-id - // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b - if err := fillDiskConfigBySnapshot(userCred, &diskConfig, p[len("snapshot-"):]); err != nil { - return nil, err - } - } else if len(p) > 0 { - if err := fillDiskConfigByImage(ctx, userCred, &diskConfig, p); err != nil { - return nil, err - } + if info.ImageId != "" { + if err := fillDiskConfigByImage(ctx, userCred, info, info.ImageId); err != nil { + return nil, err } } // XXX: do not set default disk size here, set it by each hypervisor driver // if len(diskConfig.ImageId) > 0 && diskConfig.SizeMb == 0 { // diskConfig.SizeMb = options.Options.DefaultDiskSize // MB // else - if len(diskConfig.ImageId) == 0 && diskConfig.SizeMb == 0 { + if len(info.ImageId) == 0 && info.SizeMb == 0 { return nil, httperrors.NewInputParameterError("Diskinfo not contains either imageID or size") } - return &diskConfig, nil + return info, nil } -func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDiskConfig, snapshotId string) error { +func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *api.DiskConfig, snapshotId string) error { iSnapshot, err := SnapshotManager.FetchByIdOrName(userCred, snapshotId) if err != nil { if err == sql.ErrNoRows { @@ -1260,7 +1210,7 @@ func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDi } func fillDiskConfigByImage(ctx context.Context, userCred mcclient.TokenCredential, - diskConfig *SDiskConfig, imageId string) error { + diskConfig *api.DiskConfig, imageId string) error { if userCred == nil { diskConfig.ImageId = imageId } else { @@ -1298,7 +1248,7 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, imageI return image, nil } -func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { +func (self *SDisk) fetchDiskInfo(diskConfig *api.DiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId self.DiskType = DISK_TYPE_SYS @@ -1343,6 +1293,7 @@ type DiskInfo struct { DiskType string } +// DEPRECATE: will be remove in future, use ToDiskConfig func (self *SDisk) ToDiskInfo() DiskInfo { ret := DiskInfo{ ImageId: self.GetTemplateId(), @@ -1362,6 +1313,26 @@ func (self *SDisk) ToDiskInfo() DiskInfo { return ret } +func (self *SDisk) ToDiskConfig() *api.DiskConfig { + ret := &api.DiskConfig{ + Index: -1, + ImageId: self.GetTemplateId(), + Fs: self.GetFsFormat(), + Mountpoint: self.GetMountPoint(), + Format: self.DiskFormat, + SizeMb: self.DiskSize, + DiskType: self.DiskType, + } + storage := self.GetStorage() + if storage == nil { + return ret + } + ret.Storage = storage.Id + ret.Backend = storage.StorageType + ret.Medium = storage.MediumType + return ret +} + func (self *SDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { // override log.Infof("disk delete do nothing") diff --git a/pkg/compute/models/dynamicschedtags.go b/pkg/compute/models/dynamicschedtags.go index a2c0059c3f..5fe7c7903e 100644 --- a/pkg/compute/models/dynamicschedtags.go +++ b/pkg/compute/models/dynamicschedtags.go @@ -193,7 +193,7 @@ func (self *SDynamicschedtag) PerformEvaluate(ctx context.Context, userCred mccl hostDesc := jsonutils.Marshal(host) params := jsonutils.NewDict() - params.Add(srvDesc, "server") + params.Add(srvDesc.JSON(srvDesc), "server") params.Add(hostDesc, "host") meet, err := conditionparser.Eval(self.Condition, params) @@ -201,7 +201,7 @@ func (self *SDynamicschedtag) PerformEvaluate(ctx context.Context, userCred mccl return nil, err } result := jsonutils.NewDict() - result.Add(srvDesc, "server") + result.Add(srvDesc.JSON(srvDesc), "server") result.Add(hostDesc, "host") if meet { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 8e9192e232..88b93a234b 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -18,6 +18,9 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -703,8 +706,8 @@ func (self *SGuest) StartGueststartTask(ctx context.Context, userCred mcclient.T return nil } -func (self *SGuest) StartGuestCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error { - return self.GetDriver().StartGuestCreateTask(self, ctx, userCred, params, pendingUsage, parentTaskId) +func (self *SGuest) StartGuestCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, pendingUsage quotas.IQuota, parentTaskId string) error { + return self.GetDriver().StartGuestCreateTask(self, ctx, userCred, input.JSON(input), pendingUsage, parentTaskId) } func (self *SGuest) StartSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { @@ -1118,10 +1121,13 @@ func (self *SGuest) AllowPerformCreatedisk(ctx context.Context, userCred mcclien func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { diskSize := 0 - disksConf := jsonutils.NewDict() + disksConf := make([]*api.DiskConfig, 0) diskSizes := make(map[string]int, 0) - diskDefArray := jsonutils.GetArrayOfPrefix(data, "disk") + diskDefArray, err := cmdline.FetchDiskConfigsByJSON(data) + if err != nil { + return nil, err + } if len(diskDefArray) == 0 { logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, "No Disk Info Provided", userCred, false) return nil, httperrors.NewBadRequestError("No Disk Info Provided") @@ -1136,7 +1142,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok if len(diskInfo.Backend) == 0 { diskInfo.Backend = self.getDefaultStorageType() } - disksConf.Set(fmt.Sprintf("disk.%d", diskIdx), jsonutils.Marshal(diskInfo)) + disksConf = append(disksConf, diskInfo) if _, ok := diskSizes[diskInfo.Backend]; !ok { diskSizes[diskInfo.Backend] = diskInfo.SizeMb } else { @@ -1163,7 +1169,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok pendingUsage := &SQuota{ Storage: diskSize, } - err := QuotaManager.CheckSetPendingQuota(ctx, userCred, self.ProjectId, pendingUsage) + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, self.ProjectId, pendingUsage) if err != nil { logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) return nil, httperrors.NewOutOfQuotaError(err.Error()) @@ -1172,7 +1178,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false) + err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false, nil) if err != nil { QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false) @@ -1182,7 +1188,9 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok return nil, err } -func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error { +func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, disks []*api.DiskConfig, parentTaskId string) error { + data := jsonutils.NewDict() + data.Add(jsonutils.Marshal(disks), "disks") task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateDiskTask", self, userCred, data, parentTaskId, "", nil) if err != nil { return err @@ -1394,7 +1402,8 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T log.Errorf("net_desc not found") return nil, httperrors.NewMissingParameterError("net_desc") } - conf, err := parseNetworkInfo(userCred, netDesc) + conf, err := cmdline.ParseNetworkConfigByJSON(netDesc, -1) + conf, err = parseNetworkInfo(userCred, conf) if err != nil { log.Errorf("parseNetworkInfo fail %s", err) return nil, err @@ -1523,7 +1532,12 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient. if err != nil { return nil, httperrors.NewBadRequestError(err.Error()) } - conf, err := parseNetworkInfo(userCred, netDesc) + conf, err := cmdline.ParseNetworkConfigByJSON(netDesc, -1) + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + + conf, err = parseNetworkInfo(userCred, conf) if err != nil { return nil, err } @@ -1706,15 +1720,16 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T var diskIdx = 1 var newDiskIdx = 0 var diskSizes = make(map[string]int, 0) - var newDisks = jsonutils.NewDict() + var newDisks = make([]*api.DiskConfig, 0) var resizeDisks = jsonutils.NewArray() - for { - diskNum := fmt.Sprintf("disk.%d", diskIdx) - diskDesc, err := data.Get(diskNum) - if err != nil || diskDesc == jsonutils.JSONNull { - break - } - diskConf, err := parseDiskInfo(ctx, userCred, diskDesc) + + inputDisks, err := cmdline.FetchDiskConfigsByJSON(data) + if err != nil { + return nil, err + } + + for _, diskConf := range inputDisks { + diskConf, err = parseDiskInfo(ctx, userCred, diskConf) if err != nil { return nil, httperrors.NewBadRequestError("Parse disk info error: %s", err) } @@ -1723,7 +1738,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } if diskConf.SizeMb > 0 { if diskIdx >= len(disks) { - newDisks.Add(jsonutils.Marshal(diskConf), fmt.Sprintf("disk.%d", newDiskIdx)) + newDisks = append(newDisks, diskConf) newDiskIdx += 1 addDisk += diskConf.SizeMb storage := host.GetLeastUsedStorage(diskConf.Backend) @@ -1772,8 +1787,8 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetId()) } - if newDisks.Length() > 0 { - confs.Add(newDisks, "create") + if len(newDisks) > 0 { + confs.Add(jsonutils.Marshal(newDisks), "create") } if resizeDisks.Length() > 0 { confs.Add(resizeDisks, "resize") @@ -1815,8 +1830,8 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } } - if newDisks.Length() > 0 { - err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false) + if len(newDisks) > 0 { + err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false, nil) if err != nil { QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err) @@ -1826,17 +1841,23 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T return nil, nil } -func (self *SGuest) confToSchedDesc(addCpu, addMem, addDisk int) *jsonutils.JSONDict { - desc := jsonutils.NewDict() - desc.Set("vmem_size", jsonutils.NewInt(int64(addMem))) - desc.Set("vcpu_count", jsonutils.NewInt(int64(addCpu))) +func (self *SGuest) confToSchedDesc(addCpu, addMem, addDisk int) *schedapi.ScheduleInput { guestDisks := self.GetDisks() - diskInfo := guestDisks[0].ToDiskInfo() - diskInfo.Size = int64(addDisk) - desc.Set("disk.0", jsonutils.Marshal(diskInfo)) - desc.Set("prefer_host_id", jsonutils.NewString(self.HostId)) - desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) - desc.Set("owner_tenant_id", jsonutils.NewString(self.ProjectId)) + diskInfo := guestDisks[0].ToDiskConfig() + diskInfo.SizeMb = addDisk + + desc := &schedapi.ScheduleInput{ + ServerConfig: schedapi.ServerConfig{ + ServerConfigs: &api.ServerConfigs{ + Hypervisor: self.Hypervisor, + Project: self.ProjectId, + PreferHost: self.HostId, + Disks: []*api.DiskConfig{diskInfo}, + }, + Memory: addMem, + Ncpu: addCpu, + }, + } return desc } @@ -2700,8 +2721,8 @@ type SImportNic struct { Ifname string `json:"ifname"` } -func (n SImportNic) ToNetConfig(net *SNetwork) *SNetworkConfig { - return &SNetworkConfig{ +func (n SImportNic) ToNetConfig(net *SNetwork) *api.NetworkConfig { + return &api.NetworkConfig{ Network: net.Id, Wire: net.WireId, Address: n.Ip, @@ -2725,8 +2746,8 @@ type SImportDisk struct { TemplateId string `json:"template_id"` } -func (d SImportDisk) ToDiskConfig() *SDiskConfig { - ret := &SDiskConfig{ +func (d SImportDisk) ToDiskConfig() *api.DiskConfig { + ret := &api.DiskConfig{ SizeMb: d.SizeMb, ImageId: d.TemplateId, Format: d.Format, @@ -2849,7 +2870,7 @@ func (self *SGuest) importDisks(ctx context.Context, userCred mcclient.TokenCred return httperrors.NewInputParameterError("Empty import disks") } for _, disk := range disks { - disk, err := self.createDiskOnHost(ctx, userCred, self.GetHost(), disk.ToDiskConfig(), nil, true, true) + disk, err := self.createDiskOnHost(ctx, userCred, self.GetHost(), disk.ToDiskConfig(), nil, true, true, nil) if err != nil { return err } diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index cde845baba..94815caf87 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -247,6 +248,7 @@ func (self *SGuestdisk) Detach(ctx context.Context, userCred mcclient.TokenCrede return db.DetachJoint(ctx, userCred, self) } +// DEPRECATE: will be remove in future, use ToDiskConfig func (self *SGuestdisk) ToDiskInfo() DiskInfo { disk := self.GetDisk() if disk == nil { @@ -257,3 +259,13 @@ func (self *SGuestdisk) ToDiskInfo() DiskInfo { info.Cache = self.CacheMode return info } + +func (self *SGuestdisk) ToDiskConfig() *api.DiskConfig { + disk := self.GetDisk() + if disk == nil { + return nil + } + conf := disk.ToDiskConfig() + conf.Index = int(self.Index) + return conf +} diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 34790a0d91..478fe10105 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -30,19 +31,20 @@ type IGuestDriver interface { GetJsonDescAtHost(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, host *SHost) jsonutils.JSONObject - ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) + ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *api.ServerCreateInput) (*api.ServerCreateInput, error) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) - ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *SHost, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) + ValidateCreateHostData(ctx context.Context, userCred mcclient.TokenCredential, bmName string, host *SHost, input *api.ServerCreateInput) (*api.ServerCreateInput, error) - PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *SHost, params *jsonutils.JSONDict) error + PrepareDiskRaidConfig(userCred mcclient.TokenCredential, host *SHost, params []*api.BaremetalDiskConfig) error - GetNamedNetworkConfiguration(guest *SGuest, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig) (*SNetwork, []SNicConfig, IPAddlocationDirection) + GetNamedNetworkConfiguration(guest *SGuest, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig) (*SNetwork, []SNicConfig, IPAddlocationDirection) - Attach2RandomNetwork(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) + Attach2RandomNetwork(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) GetRandomNetworkTypes() []string + GetStorageTypes() []string ChooseHostStorage(host *SHost, backend string) *SStorage StartGuestCreateTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 220f17acad..0b326e8427 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" @@ -629,3 +630,18 @@ func (self *SGuestnetwork) GetShortDesc(ctx context.Context) *jsonutils.JSONDict } return desc } + +func (self *SGuestnetwork) ToNetworkConfig() *api.NetworkConfig { + net := self.GetNetwork() + if net == nil { + return nil + } + ret := &api.NetworkConfig{ + Index: int(self.Index), + Network: net.Id, + Wire: net.GetWire().Id, + Address: self.IpAddr, + Project: net.ProjectId, + } + return ret +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0f2f09831c..fd50ef79ce 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -5,7 +5,6 @@ import ( "context" "database/sql" "fmt" - "strconv" "strings" "time" @@ -13,7 +12,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" @@ -22,6 +20,9 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -717,47 +718,30 @@ func (guest *SGuest) ValidateResizeDisk(disk *SDisk, storage *SStorage) error { return guest.GetDriver().ValidateResizeDisk(guest, disk, storage) } -func ValidateMemCpuData(data jsonutils.JSONObject) (int, int, error) { - vmemSize := 0 - vcpuCount := 0 - var err error - - hypervisor, _ := data.GetString("hypervisor") +func ValidateMemCpuData(vmemSize, vcpuCount int, hypervisor string) (int, int, error) { if len(hypervisor) == 0 { hypervisor = HYPERVISOR_DEFAULT } driver := GetDriver(hypervisor) - vmemStr, _ := data.GetString("vmem_size") - if len(vmemStr) > 0 { - if !regutils.MatchSize(vmemStr) { - return 0, 0, httperrors.NewInputParameterError("Memory size must be number[+unit], like 256M, 1G or 256") - } - vmemSize, err = fileutils.GetSizeMb(vmemStr, 'M', 1024) - if err != nil { - return 0, 0, err - } + if vmemSize > 0 { maxVmemGb := driver.GetMaxVMemSizeGB() if vmemSize < 8 || vmemSize > maxVmemGb*1024 { return 0, 0, httperrors.NewInputParameterError("Memory size must be 8MB ~ %d GB", maxVmemGb) } } - vcpuStr, _ := data.GetString("vcpu_count") - if len(vcpuStr) > 0 { - if !regutils.MatchInteger(vcpuStr) { - return 0, 0, httperrors.NewInputParameterError("CPU core count must be integer") - } - vcpuCount, _ = strconv.Atoi(vcpuStr) - maxVcpuCount := driver.GetMaxVCpuCount() - if vcpuCount < 1 || vcpuCount > maxVcpuCount { - return 0, 0, httperrors.NewInputParameterError("CPU core count must be 1 ~ %d", maxVcpuCount) - } + + maxVcpuCount := driver.GetMaxVCpuCount() + if vcpuCount < 1 || vcpuCount > maxVcpuCount { + return 0, 0, httperrors.NewInputParameterError("CPU core count must be 1 ~ %d", maxVcpuCount) } return vmemSize, vcpuCount, nil } func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - vmemSize, vcpuCount, err := ValidateMemCpuData(data) + memSize, _ := data.Int("vmem_size") + cpuCount, _ := data.Int("vcpu_count") + vmemSize, vcpuCount, err := ValidateMemCpuData(int(memSize), int(cpuCount), self.Hypervisor) if err != nil { return nil, err } @@ -797,26 +781,32 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To } func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - resetPassword := jsonutils.QueryBoolean(data, "reset_password", true) - passwd, _ := data.GetString("password") + input, err := cmdline.FetchServerCreateInputByJSON(data) + if err != nil { + return nil, err + } + resetPassword := true + if input.ResetPassword != nil { + resetPassword = *input.ResetPassword + } + + passwd := input.Password if resetPassword && len(passwd) > 0 { if !seclib2.MeetComplxity(passwd) { return nil, httperrors.NewWeakPasswordError() } } - var err error var hypervisor string // var rootStorageType string var osProf osprofile.SOSProfile - hypervisor, _ = data.GetString("hypervisor") + hypervisor = input.Hypervisor if hypervisor != HYPERVISOR_CONTAINER { - - disk0Json, _ := data.Get("disk.0") - if disk0Json == nil || disk0Json == jsonutils.JSONNull { + if len(input.Disks) == 0 { return nil, httperrors.NewInputParameterError("No disk information provided") } - diskConfig, err := parseDiskInfo(ctx, userCred, disk0Json) + diskConfig := input.Disks[0] + diskConfig, err = parseDiskInfo(ctx, userCred, diskConfig) if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) } @@ -833,19 +823,16 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m // } // rootStorageType = diskConfig.Backend - data.Add(jsonutils.Marshal(diskConfig), "disk.0") + input.Disks[0] = diskConfig imgProperties := diskConfig.ImageProperties - if data.Contains("cdrom") { - cdromStr, err := data.GetString("cdrom") - if err != nil { - return nil, httperrors.NewInputParameterError("invalid cdrom device description %s", err) - } + if input.Cdrom != "" { + cdromStr := input.Cdrom image, err := parseIsoInfo(ctx, userCred, cdromStr) if err != nil { return nil, httperrors.NewInputParameterError("parse cdrom device info error %s", err) } - data.Add(jsonutils.NewString(image.Id), "cdrom") + input.Cdrom = image.Id if len(imgProperties) == 0 { imgProperties = image.Properties } @@ -855,7 +842,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m imgProperties = map[string]string{"os_type": "Linux"} } - osType, _ := data.GetString("os_type") + osType := input.OsType osProf, err = osprofile.GetOSProfileFromImageProperties(imgProperties, hypervisor) if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) @@ -863,36 +850,36 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(osProf.Hypervisor) > 0 && len(hypervisor) == 0 { hypervisor = osProf.Hypervisor - data.Add(jsonutils.NewString(osProf.Hypervisor), "hypervisor") + input.Hypervisor = hypervisor } if len(osProf.OSType) > 0 && len(osType) == 0 { osType = osProf.OSType - data.Add(jsonutils.NewString(osProf.OSType), "os_type") + input.OsType = osType } - data.Add(jsonutils.Marshal(osProf), "__os_profile__") + input.OsProfile = jsonutils.Marshal(osProf) } - data, err = ValidateScheduleCreateData(ctx, userCred, data, hypervisor) + input, err = ValidateScheduleCreateData(ctx, userCred, input, hypervisor) if err != nil { return nil, err } - hypervisor, _ = data.GetString("hypervisor") + hypervisor = input.Hypervisor if hypervisor != HYPERVISOR_CONTAINER { // support sku here var sku *SServerSku - skuName := jsonutils.GetAnyString(data, []string{"sku", "flavor", "instance_type"}) + skuName := input.InstanceType if len(skuName) > 0 { sku, err := ServerSkuManager.FetchSkuByNameAndHypervisor(skuName, hypervisor, true) if err != nil { return nil, err } - data.Add(jsonutils.NewString(sku.Name), "instance_type") - data.Add(jsonutils.NewInt(int64(sku.MemorySizeMB)), "vmem_size") - data.Add(jsonutils.NewInt(int64(sku.CpuCoreCount)), "vcpu_count") + input.InstanceType = sku.Name + input.VmemSize = sku.MemorySizeMB + input.VcpuCount = sku.CpuCoreCount } else { - vmemSize, vcpuCount, err := ValidateMemCpuData(data) + vmemSize, vcpuCount, err := ValidateMemCpuData(input.VmemSize, input.VcpuCount, input.Hypervisor) if err != nil { return nil, err } @@ -903,28 +890,28 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if vcpuCount == 0 { vcpuCount = 1 } - data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") - data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") + input.VmemSize = vmemSize + input.VcpuCount = vcpuCount } - dataDiskDefs := []jsonutils.JSONObject{} + dataDiskDefs := []*api.DiskConfig{} if sku != nil && sku.AttachedDiskCount > 0 { for i := 0; i < sku.AttachedDiskCount; i += 1 { - dataDisk := jsonutils.Marshal(map[string]interface{}{ - "size_mb": sku.AttachedDiskSizeGB * 1024, - "backend": strings.ToLower(sku.AttachedDiskType), - }) + dataDisk := &api.DiskConfig{ + SizeMb: sku.AttachedDiskSizeGB * 1024, + Backend: strings.ToLower(sku.AttachedDiskType), + } dataDiskDefs = append(dataDiskDefs, dataDisk) } } // start from data disk - jsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 1; idx < len(jsonArray); idx += 1 { // data.Contains(fmt.Sprintf("disk.%d", idx)) - dataDiskDefs = append(dataDiskDefs, jsonArray[idx]) + disks := input.Disks + for idx := 1; idx < len(disks); idx += 1 { + dataDiskDefs = append(dataDiskDefs, disks[idx]) } - rootDiskConfig, err := parseDiskInfo(ctx, userCred, jsonArray[0]) + rootDiskConfig, err := parseDiskInfo(ctx, userCred, disks[0]) if err != nil { return nil, httperrors.NewGeneralError(err) // should no error } @@ -935,7 +922,8 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m rootDiskConfig.SizeMb = GetDriver(hypervisor).GetMinimalSysDiskSizeGb() * 1024 } log.Debugf("ROOT DISK: %#v", rootDiskConfig) - data.Set("disk.0", jsonutils.Marshal(rootDiskConfig)) + input.Disks[0] = rootDiskConfig + //data.Set("disk.0", jsonutils.Marshal(rootDiskConfig)) for i := 0; i < len(dataDiskDefs); i += 1 { diskConfig, err := parseDiskInfo(ctx, userCred, dataDiskDefs[i]) @@ -951,11 +939,11 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(diskConfig.Driver) == 0 { diskConfig.Driver = osProf.DiskDriver } - data.Set(fmt.Sprintf("disk.%d", i+1), jsonutils.Marshal(diskConfig)) + input.Disks[i+1] = diskConfig } - resourceTypeStr := jsonutils.GetAnyString(data, []string{"resource_type"}) - durationStr := jsonutils.GetAnyString(data, []string{"duration"}) + resourceTypeStr := input.ResourceType + durationStr := input.Duration if len(durationStr) > 0 { @@ -976,18 +964,18 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m return nil, httperrors.NewInputParameterError("unsupported duration %s", durationStr) } - data.Add(jsonutils.NewString(BILLING_TYPE_PREPAID), "billing_type") - data.Add(jsonutils.NewString(billingCycle.String()), "billing_cycle") + input.BillingType = BILLING_TYPE_PREPAID + input.BillingCycle = billingCycle.String() // expired_at will be set later by callback // data.Add(jsonutils.NewTimeString(billingCycle.EndAt(time.Time{})), "expired_at") - data.Set("duration", jsonutils.NewString(billingCycle.String())) + input.Duration = billingCycle.String() } } - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - for idx := 0; idx < len(netJsonArray); idx += 1 { // .Contains(fmt.Sprintf("net.%d", idx)); idx += 1 { - netConfig, err := parseNetworkInfo(userCred, netJsonArray[idx]) + netArray := input.Networks + for idx := 0; idx < len(netArray); idx += 1 { + netConfig, err := parseNetworkInfo(userCred, netArray[idx]) if err != nil { return nil, httperrors.NewInputParameterError("parse network description error %s", err) } @@ -998,12 +986,12 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(netConfig.Driver) == 0 { netConfig.Driver = osProf.NetDriver } - data.Set(fmt.Sprintf("net.%d", idx), jsonutils.Marshal(netConfig)) + input.Networks[idx] = netConfig } - isoDevArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") + isoDevArray := input.IsolatedDevices for idx := 0; idx < len(isoDevArray); idx += 1 { // .Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - if jsonutils.QueryBoolean(data, "backup", false) { + if input.Backup { return nil, httperrors.NewBadRequestError("Cannot create backup with isolated device") } devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, isoDevArray[idx]) @@ -1014,36 +1002,33 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, err } - data.Set(fmt.Sprintf("isolated_device.%d", idx), jsonutils.Marshal(devConfig)) + input.IsolatedDevices[idx] = devConfig } - keypairId, _ := data.GetString("keypair") - if len(keypairId) == 0 { - keypairId, _ = data.GetString("keypair_id") - } + keypairId := input.Keypair if len(keypairId) > 0 { keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Keypair %s not found", keypairId) } - data.Add(jsonutils.NewString(keypairObj.GetId()), "keypair_id") + input.Keypair = keypairObj.GetId() } else { - data.Add(jsonutils.NewString("None"), "keypair_id") + input.Keypair = "None" // TODO: ??? None? } - if data.Contains("secgroup") { - secGrpId, _ := data.GetString("secgroup") + if input.Secgroup != "" { + secGrpId := input.Secgroup secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred, secGrpId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Secgroup %s not found", secGrpId) } - data.Add(jsonutils.NewString(secGrpObj.GetId()), "secgrp_id") + input.Secgroup = secGrpObj.GetId() } else { - data.Add(jsonutils.NewString("default"), "secgrp_id") + input.Secgroup = "default" } - eipStr, _ := data.GetString("eip") - eipBw, _ := data.Int("eip_bw") + eipStr := input.Eip + eipBw := input.EipBw if len(eipStr) > 0 || eipBw > 0 { if !GetDriver(hypervisor).IsSupportEip() { return nil, httperrors.NewNotImplementedError("eip not supported for %s", hypervisor) @@ -1065,14 +1050,14 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if eip.IsAssociated() { return nil, httperrors.NewResourceBusyError("eip %s has been associated", eipStr) } - data.Set("eip_id", jsonutils.NewString(eipObj.GetId())) + input.Eip = eipObj.GetId() eipRegion := eip.GetRegion() preferRegionId, _ := data.GetString("prefer_region_id") if len(preferRegionId) > 0 && preferRegionId != eipRegion.Id { return nil, httperrors.NewConflictError("cannot assoicate with eip %s: different region", eipStr) } - data.Set("prefer_region_id", jsonutils.NewString(eipRegion.Id)) + input.PreferRegion = eipRegion.Id } else { // create new eip } @@ -1085,30 +1070,33 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m }*/ - data, err = GetDriver(hypervisor).ValidateCreateData(ctx, userCred, data) + input, err = GetDriver(hypervisor).ValidateCreateData(ctx, userCred, input) if err != nil { return nil, err } - data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) + data, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, input.JSON(input)) if err != nil { return nil, err } + if err := data.Unmarshal(input); err != nil { + return nil, err + } - if !jsonutils.QueryBoolean(data, "is_system", false) { - err = manager.checkCreateQuota(ctx, userCred, ownerProjId, data, - jsonutils.QueryBoolean(data, "backup", false)) + if !input.IsSystem { + err = manager.checkCreateQuota(ctx, userCred, ownerProjId, input, + input.Backup) if err != nil { return nil, err } } - data.Add(jsonutils.NewString(ownerProjId), "owner_tenant_id") - return data, nil + input.Project = ownerProjId + return input.JSON(input), nil } -func (manager *SGuestManager) checkCreateQuota(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, data *jsonutils.JSONDict, hasBackup bool) error { - req := getGuestResourceRequirements(ctx, userCred, data, 1, hasBackup) +func (manager *SGuestManager) checkCreateQuota(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, input *api.ServerCreateInput, hasBackup bool) error { + req := getGuestResourceRequirements(ctx, userCred, input, 1, hasBackup) err := QuotaManager.CheckSetPendingQuota(ctx, userCred, ownerProjId, &req) if err != nil { return httperrors.NewOutOfQuotaError(err.Error()) @@ -1133,35 +1121,27 @@ func (self *SGuest) checkUpdateQuota(ctx context.Context, userCred mcclient.Toke return err } -func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, count int, hasBackup bool) SQuota { - vcpuCount, _ := data.Int("vcpu_count") +func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, count int, hasBackup bool) SQuota { + vcpuCount := input.VcpuCount if vcpuCount == 0 { vcpuCount = 1 } - vmemSize, _ := data.Int("vmem_size") + vmemSize := input.VmemSize diskSize := 0 - diskJsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 0; idx < len(diskJsonArray); idx += 1 { // data.Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { - diskConfig, _ := parseDiskInfo(ctx, userCred, diskJsonArray[idx]) + for _, diskConfig := range input.Disks { diskSize += diskConfig.SizeMb } - isoDevArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") - devCount := len(isoDevArray) - // for idx := 0; data.Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - // devCount += 1 - //} + devCount := len(input.IsolatedDevices) eNicCnt := 0 iNicCnt := 0 eBw := 0 iBw := 0 - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - for idx := 0; idx < len(netJsonArray); idx += 1 { // .Contains(fmt.Sprintf("net.%d", idx)); idx += 1 { - netConfig, _ := parseNetworkInfo(userCred, netJsonArray[idx]) + for _, netConfig := range input.Networks { if isExitNetworkInfo(netConfig) { eNicCnt += 1 eBw += netConfig.BwLimit @@ -1177,7 +1157,7 @@ func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCr } eipCnt := 0 - eipBw, _ := data.Int("eip_bw") + eipBw := input.EipBw if eipBw > 0 { eipCnt = 1 } @@ -1233,8 +1213,9 @@ func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred } func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { - pendingUsage := getGuestResourceRequirements(ctx, userCred, data, len(items), - jsonutils.QueryBoolean(data, "backup", false)) + input := new(api.ServerCreateInput) + data.Unmarshal(input) + pendingUsage := getGuestResourceRequirements(ctx, userCred, input, len(items), input.Backup) RunBatchCreateTask(ctx, items, userCred, data, pendingUsage, "GuestBatchCreateTask") } @@ -2515,23 +2496,22 @@ func totalGuestResourceCount( return stat } -func (self *SGuest) getDefaultNetworkConfig() *SNetworkConfig { - netConf := SNetworkConfig{} +func (self *SGuest) getDefaultNetworkConfig() *api.NetworkConfig { + netConf := api.NetworkConfig{} netConf.BwLimit = options.Options.DefaultBandwidth osProf := self.getOSProfile() netConf.Driver = osProf.NetDriver return &netConf } -func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, data *jsonutils.JSONDict, pendingUsage quotas.IQuota) error { - netJsonArray := jsonutils.GetArrayOfPrefix(data, "net") - if len(netJsonArray) == 0 { +func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netArray []*api.NetworkConfig, pendingUsage quotas.IQuota) error { + if len(netArray) == 0 { netConfig := self.getDefaultNetworkConfig() _, err := self.attach2RandomNetwork(ctx, userCred, host, netConfig, pendingUsage) return err } - for idx := 0; idx < len(netJsonArray); idx += 1 { - netConfig, err := parseNetworkInfo(userCred, netJsonArray[idx]) + for _, netConfig := range netArray { + netConfig, err := parseNetworkInfo(userCred, netConfig) if err != nil { return err } @@ -2543,7 +2523,7 @@ func (self *SGuest) CreateNetworksOnHost(ctx context.Context, userCred mcclient. return nil } -func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { var gns []SGuestnetwork var err1, err2 error if len(netConfig.Network) > 0 { @@ -2563,7 +2543,7 @@ func (self *SGuest) attach2NetworkDesc(ctx context.Context, userCred mcclient.To } } -func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { driver := self.GetDriver() net, nicConfs, allocDir := driver.GetNamedNetworkConfiguration(self, userCred, host, netConfig) if net != nil { @@ -2582,33 +2562,38 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie } } -func (self *SGuest) attach2RandomNetwork(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *SNetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { +func (self *SGuest) attach2RandomNetwork(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) { driver := self.GetDriver() return driver.Attach2RandomNetwork(self, ctx, userCred, host, netConfig, pendingUsage) } -func (self *SGuest) CreateDisksOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, - data *jsonutils.JSONDict, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) error { - diskJsonArray := jsonutils.GetArrayOfPrefix(data, "disk") - for idx := 0; idx < len(diskJsonArray); idx += 1 { // .Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { - diskConfig, err := parseDiskInfo(ctx, userCred, diskJsonArray[idx]) +func (self *SGuest) CreateDisksOnHost( + ctx context.Context, + userCred mcclient.TokenCredential, + host *SHost, + disks []*api.DiskConfig, + pendingUsage quotas.IQuota, + inheritBilling bool, + isWithServerCreate bool, + candidate *schedapi.CandidateResource, +) error { + for idx := 0; idx < len(disks); idx += 1 { + diskConfig, err := parseDiskInfo(ctx, userCred, disks[idx]) if err != nil { return err } - disk, err := self.createDiskOnHost(ctx, userCred, host, diskConfig, pendingUsage, inheritBilling, isWithServerCreate) + disk, err := self.createDiskOnHost(ctx, userCred, host, diskConfig, pendingUsage, inheritBilling, isWithServerCreate, candidate.Disks[idx]) if err != nil { return err } - data.Add(jsonutils.NewString(disk.Id), fmt.Sprintf("disk.%d.id", idx)) - if len(diskConfig.SnapshotId) > 0 { - data.Add(jsonutils.NewString(diskConfig.SnapshotId), fmt.Sprintf("disk.%d.snapshot", idx)) - } + diskConfig.DiskId = disk.Id + disks[idx] = diskConfig } return nil } func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.TokenCredential, storage *SStorage, - diskConfig *SDiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { + diskConfig *api.DiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { lockman.LockObject(ctx, storage) defer lockman.ReleaseObject(ctx, storage) @@ -2642,9 +2627,26 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T return disk, nil } -func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, - diskConfig *SDiskConfig, pendingUsage quotas.IQuota, inheritBilling bool, isWithServerCreate bool) (*SDisk, error) { - storage := self.GetDriver().ChooseHostStorage(host, diskConfig.Backend) +func (self *SGuest) ChooseHostStorage(host *SHost, backend string, candidate *schedapi.CandidateDisk) *SStorage { + log.Errorf("==========candidate %#v", candidate) + if candidate == nil { + return self.GetDriver().ChooseHostStorage(host, backend) + } + return StorageManager.FetchStorageById(candidate.StorageId) +} + +func (self *SGuest) createDiskOnHost( + ctx context.Context, + userCred mcclient.TokenCredential, + host *SHost, + diskConfig *api.DiskConfig, + pendingUsage quotas.IQuota, + inheritBilling bool, + isWithServerCreate bool, + candidate *schedapi.CandidateDisk, +) (*SDisk, error) { + storage := self.ChooseHostStorage(host, diskConfig.Backend, candidate) + log.Debugf("Choose storage %s:%s for disk %#v", storage.Name, storage.Id, diskConfig) if storage == nil { return nil, fmt.Errorf("No storage on %s to create disk for %s", host.GetName(), diskConfig.Backend) } @@ -2652,6 +2654,7 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke if err != nil { return nil, err } + // TODO: use scheduler candidate storage if len(self.BackupHostId) > 0 { backupHost := HostManager.FetchHostById(self.BackupHostId) backupStorage := self.GetDriver().ChooseHostStorage(backupHost, diskConfig.Backend) @@ -2669,14 +2672,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke return disk, err } -func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, data *jsonutils.JSONDict, pendingUsage quotas.IQuota) error { - devJsonArray := jsonutils.GetArrayOfPrefix(data, "isolated_device") - for idx := 0; idx < len(devJsonArray); idx += 1 { // .Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { - devConfig, err := IsolatedDeviceManager.parseDeviceInfo(userCred, devJsonArray[idx]) - if err != nil { - return err - } - err = self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) +func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { + for idx := 0; idx < len(devs); idx += 1 { + devConfig := devs[idx] + err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage) if err != nil { return err } @@ -2684,7 +2683,7 @@ func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcc return nil } -func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *SIsolatedDeviceConfig, pendingUsage quotas.IQuota) error { +func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devConfig *api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error { lockman.LockClass(ctx, QuotaManager, self.ProjectId) defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId) @@ -2866,24 +2865,15 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To return nil } -type SDeployConfig struct { - Path string - Action string - Content string -} - func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, params *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { config := jsonutils.NewDict() desc := self.GetDriver().GetJsonDescAtHost(ctx, userCred, self, host) config.Add(desc, "desc") - deploys := make([]SDeployConfig, 0) - for idx := 0; params.Contains(fmt.Sprintf("deploy.%d.path", idx)); idx += 1 { - path, _ := params.GetString(fmt.Sprintf("deploy.%d.path", idx)) - action, _ := params.GetString(fmt.Sprintf("deploy.%d.action", idx)) - content, _ := params.GetString(fmt.Sprintf("deploy.%d.content", idx)) - deploys = append(deploys, SDeployConfig{Path: path, Action: action, Content: content}) + deploys, err := cmdline.FetchDeployConfigsByJSON(params) + if err != nil { + return nil, err } if len(deploys) > 0 { @@ -3856,34 +3846,39 @@ func (self *SGuest) getDefaultStorageType() string { return STORAGE_LOCAL } -func (self *SGuest) getSchedDesc() jsonutils.JSONObject { - desc := jsonutils.NewDict() +func (self *SGuest) getSchedDesc() *schedapi.ScheduleInput { + desc := new(schedapi.ScheduleInput) - desc.Add(jsonutils.NewString(self.Id), "id") - desc.Add(jsonutils.NewString(self.Name), "name") - desc.Add(jsonutils.NewInt(int64(self.VmemSize)), "vmem_size") - desc.Add(jsonutils.NewInt(int64(self.VcpuCount)), "vcpu_count") + desc.Id = self.Id + desc.Name = self.Name + desc.Memory = self.VmemSize + desc.Ncpu = int(self.VcpuCount) gds := self.GetDisks() if gds != nil { for i := 0; i < len(gds); i += 1 { - desc.Add(jsonutils.Marshal(gds[i].ToDiskInfo()), fmt.Sprintf("disk.%d", i)) + conf := new(api.DiskConfig) + jsonutils.Marshal(gds[i].ToDiskInfo()).Unmarshal(conf) + desc.Disks = append(desc.Disks, conf) } } gns, _ := self.GetNetworks("") if gns != nil { for i := 0; i < len(gns); i += 1 { - desc.Add(jsonutils.NewString(fmt.Sprintf("%s:%s", gns[i].NetworkId, gns[i].IpAddr)), fmt.Sprintf("net.%d", i)) + desc.Networks = append(desc.Networks, &api.NetworkConfig{ + Network: gns[i].NetworkId, + Address: gns[i].IpAddr, + }) } } if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { - desc.Add(jsonutils.NewString(self.HostId), "host_id") + desc.HostId = self.HostId } - desc.Add(jsonutils.NewString(self.ProjectId), "owner_tenant_id") - desc.Add(jsonutils.NewString(self.GetHypervisor()), "hypervisor") + desc.Project = self.ProjectId + desc.Hypervisor = self.GetHypervisor() return desc } @@ -3896,28 +3891,31 @@ func (self *SGuest) GetApptags() []string { return nil } -func (self *SGuest) ToSchedDesc() *jsonutils.JSONDict { - desc := jsonutils.NewDict() - desc.Set("id", jsonutils.NewString(self.Id)) - desc.Set("name", jsonutils.NewString(self.Name)) - desc.Set("vmem_size", jsonutils.NewInt(int64(self.VmemSize))) - desc.Set("vcpu_count", jsonutils.NewInt(int64(self.VcpuCount))) - self.FillGroupSchedDesc(desc) - self.FillDiskSchedDesc(desc) - self.FillNetSchedDesc(desc) - if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { - desc.Set("host_id", jsonutils.NewString(self.HostId)) +func (self *SGuest) ToSchedDesc() *schedapi.ScheduleInput { + desc := new(schedapi.ScheduleInput) + config := &schedapi.ServerConfig{ + Name: self.Name, + Memory: self.VmemSize, + Ncpu: int(self.VcpuCount), } - desc.Set("owner_tenant_id", jsonutils.NewString(self.ProjectId)) - tags := self.GetApptags() + //self.FillGroupSchedDesc(desc) + self.FillDiskSchedDesc(config) + self.FillNetSchedDesc(config) + if len(self.HostId) > 0 && regutils.MatchUUID(self.HostId) { + config.HostId = self.HostId + } + config.Project = self.ProjectId + /*tags := self.GetApptags() for i := 0; i < len(tags); i++ { desc.Set(tags[i], jsonutils.JSONTrue) - } - desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) + }*/ + + config.Hypervisor = self.GetHypervisor() + desc.ServerConfig = *config return desc } -func (self *SGuest) FillGroupSchedDesc(desc *jsonutils.JSONDict) { +/*func (self *SGuest) FillGroupSchedDesc(desc *schedapi.ServerConfig) { groups := make([]SGroupguest, 0) err := GroupguestManager.Query().Equals("guest_id", self.Id).All(&groups) if err != nil { @@ -3928,31 +3926,29 @@ func (self *SGuest) FillGroupSchedDesc(desc *jsonutils.JSONDict) { desc.Set(fmt.Sprintf("srvtag.%d", i), jsonutils.NewString(fmt.Sprintf("%s:%s", groups[i].SrvtagId, groups[i].Tag))) } -} +}*/ -func (self *SGuest) FillDiskSchedDesc(desc *jsonutils.JSONDict) { +func (self *SGuest) FillDiskSchedDesc(desc *schedapi.ServerConfig) { guestDisks := make([]SGuestdisk, 0) err := GuestdiskManager.Query().Equals("guest_id", self.Id).All(&guestDisks) if err != nil { - log.Errorln(err) + log.Errorln("FillDiskSchedDesc: %v", err) return } for i := 0; i < len(guestDisks); i++ { - desc.Set(fmt.Sprintf("disk.%d", i), jsonutils.Marshal(guestDisks[i].ToDiskInfo())) + desc.Disks = append(desc.Disks, guestDisks[i].ToDiskConfig()) } } -func (self *SGuest) FillNetSchedDesc(desc *jsonutils.JSONDict) { +func (self *SGuest) FillNetSchedDesc(desc *schedapi.ServerConfig) { guestNetworks := make([]SGuestnetwork, 0) err := GuestnetworkManager.Query().Equals("guest_id", self.Id).All(&guestNetworks) if err != nil { - log.Errorln(err) + log.Errorln("FillNetSchedDesc: %v", err) return } for i := 0; i < len(guestNetworks); i++ { - desc.Set(fmt.Sprintf("net.%d", i), - jsonutils.NewString(fmt.Sprintf("%s:%s", - guestNetworks[i].NetworkId, guestNetworks[i].IpAddr))) + desc.Networks = append(desc.Networks, guestNetworks[i].ToNetworkConfig()) } } diff --git a/pkg/compute/models/helper.go b/pkg/compute/models/helper.go index f573990cd4..6a1a628a9b 100644 --- a/pkg/compute/models/helper.go +++ b/pkg/compute/models/helper.go @@ -3,12 +3,12 @@ package models import ( "context" "database/sql" - "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -38,23 +38,20 @@ func RunBatchCreateTask( } } -func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, hypervisor string) (*jsonutils.JSONDict, error) { +func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput, hypervisor string) (*api.ServerCreateInput, error) { var err error - if jsonutils.QueryBoolean(data, "baremetal", false) { + if input.Baremetal { hypervisor = HYPERVISOR_BAREMETAL } // base validate_create_data - if (data.Contains("prefer_baremetal") || data.Contains("prefer_host")) && hypervisor != HYPERVISOR_CONTAINER { + if (input.PreferHost != "") && hypervisor != HYPERVISOR_CONTAINER { if !userCred.IsAdminAllow(consts.GetServiceType(), GuestManager.KeywordPlural(), policy.PolicyActionPerform, "assign-host") { return nil, httperrors.NewNotSufficientPrivilegeError("Only system admin can specify preferred host") } - bmName, _ := data.GetString("prefer_host") - if len(bmName) == 0 { - bmName, _ = data.GetString("prefer_baremetal") - } + bmName := input.PreferHost bmObj, err := HostManager.FetchByIdOrName(nil, bmName) if err != nil { if err == sql.ErrNoRows { @@ -80,45 +77,35 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred hypervisor = HYPERVISOR_DEFAULT } - _, err = GetDriver(hypervisor).ValidateCreateHostData(ctx, userCred, bmName, baremetal, data) + _, err = GetDriver(hypervisor).ValidateCreateHostData(ctx, userCred, bmName, baremetal, input) if err != nil { return nil, err } - data.Set("prefer_baremetal_id", jsonutils.NewString(baremetal.Id)) - data.Set("prefer_host_id", jsonutils.NewString(baremetal.Id)) + input.PreferHost = baremetal.Id zone := baremetal.GetZone() - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) + input.PreferRegion = region.Id } else { schedtags := make(map[string]string) - if data.Contains("aggregate_strategy") { - err = data.Unmarshal(&schedtags, "aggregate_strategy") - if err != nil { - return nil, httperrors.NewInputParameterError("invalid aggregate_strategy") - } - } - aggArray := jsonutils.GetArrayOfPrefix(data, "schedtag") - for idx := 0; idx < len(aggArray); idx += 1 { // data.Contains(fmt.Sprintf("schedtag.%d", idx)); idx += 1 { - aggStr, _ := aggArray[idx].GetString() // .GetString(fmt.Sprintf("schedtag.%d", idx)) - if len(aggStr) > 0 { - parts := strings.Split(aggStr, ":") - if len(parts) >= 2 && len(parts[0]) > 0 && len(parts[1]) > 0 { - schedtags[parts[0]] = parts[1] - } - } + for _, tag := range input.Schedtags { + schedtags[tag.Id] = tag.Strategy } if len(schedtags) > 0 { schedtags, err = SchedtagManager.ValidateSchedtags(userCred, schedtags) if err != nil { return nil, httperrors.NewInputParameterError("invalid aggregate_strategy: %s", err) } - data.Add(jsonutils.Marshal(schedtags), "aggregate_strategy") + tags := make([]*api.SchedtagConfig, 0) + for name, strategy := range schedtags { + tags = append(tags, &api.SchedtagConfig{Id: name, Strategy: strategy}) + } + input.Schedtags = tags } - if data.Contains("prefer_wire") { - wireStr, _ := data.GetString("prefer_wire") + if input.PreferWire != "" { + wireStr := input.PreferWire wireObj, err := WireManager.FetchById(wireStr) if err != nil { if err == sql.ErrNoRows { @@ -128,13 +115,13 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } wire := wireObj.(*SWire) - data.Set("prefer_wire_id", jsonutils.NewString(wire.Id)) + input.PreferWire = wire.Id zone := wire.GetZone() - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) - } else if data.Contains("prefer_zone") { - zoneStr, _ := data.GetString("prefer_zone") + input.PreferRegion = region.Id + } else if input.PreferZone != "" { + zoneStr := input.PreferZone zoneObj, err := ZoneManager.FetchById(zoneStr) if err != nil { if err == sql.ErrNoRows { @@ -144,11 +131,11 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } zone := zoneObj.(*SZone) - data.Set("prefer_zone_id", jsonutils.NewString(zone.Id)) + input.PreferZone = zone.Id region := zone.GetRegion() - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) - } else if data.Contains("prefer_region") { - regionStr, _ := data.GetString("prefer_region") + input.PreferRegion = region.Id + } else if input.PreferRegion != "" { + regionStr := input.PreferRegion regionObj, err := CloudregionManager.FetchById(regionStr) if err != nil { if err == sql.ErrNoRows { @@ -158,7 +145,7 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred } } region := regionObj.(*SCloudregion) - data.Set("prefer_region_id", jsonutils.NewString(region.Id)) + input.PreferRegion = region.Id } } @@ -171,6 +158,6 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred return nil, httperrors.NewInputParameterError("Hypervisor %s not supported", hypervisor) } - data.Add(jsonutils.NewString(hypervisor), "hypervisor") - return data, nil + input.Hypervisor = hypervisor + return input, nil } diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index c54e7a9381..83bc5867fe 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -13,6 +13,7 @@ import ( "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -534,7 +535,7 @@ func (self *SHost) BorrowIpAddrsFromGuest(ctx context.Context, userCred mcclient return nil } -func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { +func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { ihost, err := host.GetIHost() if err != nil { log.Errorf("host.GetIHost fail %s", err) @@ -555,6 +556,7 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC netifs := host.GetNetInterfaces() netIdx := 0 + input.Networks = make([]*api.NetworkConfig, 0) for i := 0; i < len(netifs); i += 1 { hn := netifs[i].GetBaremetalNetwork() if hn != nil { @@ -563,12 +565,15 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC return nil, err } packedMac := strings.Replace(netifs[i].Mac, ":", "", -1) - netDesc := fmt.Sprintf("%s:%s:%s:[reserved]", packedMac, hn.IpAddr, hn.NetworkId) - params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.NewString(netDesc)) + input.Networks = append(input.Networks, &api.NetworkConfig{ + Network: hn.NetworkId, + Mac: packedMac, + Address: hn.IpAddr, + }) netIdx += 1 } } - params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.JSONNull) + //params.Set(fmt.Sprintf("net.%d", netIdx), jsonutils.JSONNull) for i := 0; i < len(idisks); i += 1 { /*istorage, err := idisks[i].GetIStorage() @@ -577,31 +582,34 @@ func (host *SHost) SetGuestCreateNetworkAndDiskParams(ctx context.Context, userC return nil, err }*/ - key := fmt.Sprintf("disk.%d", i) - jsonConf, _ := params.Get(key) - if jsonConf != nil && jsonConf != jsonutils.JSONNull { - conf, err := parseDiskInfo(ctx, userCred, jsonConf) + var diskConfig *api.DiskConfig + if i < len(input.Disks) { + diskConfig = input.Disks[i] + diskConfig, err := parseDiskInfo(ctx, userCred, diskConfig) if err != nil { - log.Debugf("parseDiskInfo %s fail %s", jsonConf, err) + log.Debugf("parseDiskInfo %s fail %s", diskConfig, err) return nil, err } - conf.SizeMb = idisks[i].GetDiskSizeMB() - conf.Backend = STORAGE_LOCAL - params.Set(key, jsonutils.Marshal(conf)) + diskConfig.SizeMb = idisks[i].GetDiskSizeMB() + diskConfig.Backend = STORAGE_LOCAL + input.Disks[i] = diskConfig } else { - strConf := fmt.Sprintf("%s:%d", STORAGE_LOCAL, idisks[i].GetDiskSizeMB()) - conf, err := parseDiskInfo(ctx, userCred, jsonutils.NewString(strConf)) + conf := &api.DiskConfig{ + SizeMb: idisks[i].GetDiskSizeMB(), + Backend: STORAGE_LOCAL, + } + conf, err = parseDiskInfo(ctx, userCred, conf) if err != nil { return nil, err } - params.Set(key, jsonutils.Marshal(conf)) + input.Disks = append(input.Disks, conf) } } - params.Set(fmt.Sprintf("disk.%d", len(idisks)), jsonutils.JSONNull) + //params.Set(fmt.Sprintf("disk.%d", len(idisks)), jsonutils.JSONNull) // log.Debugf("params after rebuid: %s", params.String()) - return params, nil + return input, nil } func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 10bd0a64e5..5f88d90464 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3020,7 +3020,6 @@ func (self *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCrede if bn != nil { return nil } - log.Errorf("==========EnableNetif %#v, net: %s, ipAddr: %s, allocDir: %s, reserve: %v, requireDesignatedIp: %v", netif, network, ipAddr, allocDir, reserve, requireDesignatedIp) var net *SNetwork var err error if len(ipAddr) > 0 { diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 13221ef09d..c8358842d7 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -9,39 +9,33 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) const ( - DIRECT_PCI_TYPE = "PCI" - GPU_HPC_TYPE = "GPU-HPC" // # for compute - GPU_VGA_TYPE = "GPU-VGA" // # for display - USB_TYPE = "USB" - NIC_TYPE = "NIC" + DIRECT_PCI_TYPE = api.DIRECT_PCI_TYPE + GPU_HPC_TYPE = api.GPU_HPC_TYPE // # for compute + GPU_VGA_TYPE = api.GPU_VGA_TYPE // # for display + USB_TYPE = api.USB_TYPE + NIC_TYPE = api.NIC_TYPE - NVIDIA_VENDOR_ID = "10de" - AMD_VENDOR_ID = "1002" + NVIDIA_VENDOR_ID = api.NVIDIA_VENDOR_ID + AMD_VENDOR_ID = api.AMD_VENDOR_ID ) -var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} +var VALID_GPU_TYPES = api.VALID_GPU_TYPES -var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE} +var VALID_PASSTHROUGH_TYPES = api.VALID_PASSTHROUGH_TYPES -var ID_VENDOR_MAP = map[string]string{ - NVIDIA_VENDOR_ID: "NVIDIA", - AMD_VENDOR_ID: "AMD", -} +var ID_VENDOR_MAP = api.ID_VENDOR_MAP -var VENDOR_ID_MAP = map[string]string{ - "NVIDIA": NVIDIA_VENDOR_ID, - "AMD": AMD_VENDOR_ID, -} +var VENDOR_ID_MAP = api.VENDOR_ID_MAP type SIsolatedDeviceManager struct { db.SStandaloneResourceBaseManager @@ -228,43 +222,15 @@ func (self *SIsolatedDevice) isGpu() bool { return strings.HasPrefix(self.DevType, "GPU") } -type SIsolatedDeviceConfig struct { - Id string - DevType string - Model string - Vendor string -} - -func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SIsolatedDeviceConfig, error) { - devConfig := SIsolatedDeviceConfig{} - - devJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := devJson.Unmarshal(&devConfig) - if err != nil { - return nil, err - } - return &devConfig, nil - } - devStr, err := info.GetString() - if err != nil { - log.Errorf("invalid isolated device info format %s", err) - return nil, err - } +func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) { var devId, devType, devVendor string var matchDev *SIsolatedDevice - parts := strings.Split(devStr, ":") - for _, p := range parts { - if regutils.MatchUUIDExact(p) { - devId = p - } else if utils.IsInStringArray(p, VALID_PASSTHROUGH_TYPES) { - devType = p - } else if strings.HasPrefix(p, "vendor=") { - devVendor = p[len("vendor="):] - } else { - matchDev = manager.fuzzyMatchModel(p) - } - } + + devId = devConfig.Id + matchDev = manager.fuzzyMatchModel(devConfig.Model) + devVendor = devConfig.Vendor + devType = devConfig.DevType + if len(devId) == 0 { if matchDev == nil { return nil, fmt.Errorf("Isolated device info not contains either deviceID or model name") @@ -301,10 +267,10 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr if len(devType) > 0 { devConfig.DevType = devType } - return &devConfig, nil + return devConfig, nil } -func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *SIsolatedDeviceConfig) error { +func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *api.IsolatedDeviceConfig) error { if len(config.Id) > 0 { devObj, err := manager.FetchById(config.Id) if err != nil { @@ -318,7 +284,7 @@ func (manager *SIsolatedDeviceManager) isValidDeviceinfo(config *SIsolatedDevice return nil } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Id) > 0 { return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred) } else { @@ -326,7 +292,7 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context } } -func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.Context, guest *SGuest, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.Context, guest *SGuest, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { devObj, err := manager.FetchById(devConfig.Id) if devObj == nil { return fmt.Errorf("Device %s not found: %s", devConfig.Id, err) @@ -338,7 +304,7 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C return guest.attachIsolatedDevice(ctx, userCred, dev) } -func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *SIsolatedDeviceConfig, userCred mcclient.TokenCredential) error { +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Model) == 0 { return fmt.Errorf("Not found model from info: %s", devConfig) } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 01c79bd2ed..1236484d48 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -4,20 +4,18 @@ import ( "context" "database/sql" "fmt" - "strconv" - "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -31,39 +29,33 @@ import ( const ( // # DEFAULT_BANDWIDTH = options.default_bandwidth - MAX_BANDWIDTH = 100000 + MAX_BANDWIDTH = api.MAX_BANDWIDTH - NETWORK_TYPE_GUEST = "guest" - NETWORK_TYPE_BAREMETAL = "baremetal" - NETWORK_TYPE_CONTAINER = "container" - NETWORK_TYPE_PXE = "pxe" - NETWORK_TYPE_IPMI = "ipmi" + NETWORK_TYPE_GUEST = api.NETWORK_TYPE_GUEST + NETWORK_TYPE_BAREMETAL = api.NETWORK_TYPE_BAREMETAL + NETWORK_TYPE_CONTAINER = api.NETWORK_TYPE_CONTAINER + NETWORK_TYPE_PXE = api.NETWORK_TYPE_PXE + NETWORK_TYPE_IPMI = api.NETWORK_TYPE_IPMI - STATIC_ALLOC = "static" + STATIC_ALLOC = api.STATIC_ALLOC - MAX_NETWORK_NAME_LEN = 11 + MAX_NETWORK_NAME_LEN = api.MAX_NETWORK_NAME_LEN - EXTRA_DNS_UPDATE_TARGETS = "__extra_dns_update_targets" + EXTRA_DNS_UPDATE_TARGETS = api.EXTRA_DNS_UPDATE_TARGETS - NETWORK_STATUS_INIT = "init" - NETWORK_STATUS_PENDING = "pending" - NETWORK_STATUS_AVAILABLE = "available" - NETWORK_STATUS_FAILED = "failed" - NETWORK_STATUS_UNKNOWN = "unknown" - NETWORK_STATUS_START_DELETE = "start_delete" - NETWORK_STATUS_DELETING = "deleting" - NETWORK_STATUS_DELETED = "deleted" - NETWORK_STATUS_DELETE_FAILED = "delete_failed" + NETWORK_STATUS_INIT = api.NETWORK_STATUS_INIT + NETWORK_STATUS_PENDING = api.NETWORK_STATUS_PENDING + NETWORK_STATUS_AVAILABLE = api.NETWORK_STATUS_AVAILABLE + NETWORK_STATUS_FAILED = api.NETWORK_STATUS_FAILED + NETWORK_STATUS_UNKNOWN = api.NETWORK_STATUS_UNKNOWN + NETWORK_STATUS_START_DELETE = api.NETWORK_STATUS_START_DELETE + NETWORK_STATUS_DELETING = api.NETWORK_STATUS_DELETING + NETWORK_STATUS_DELETED = api.NETWORK_STATUS_DELETED + NETWORK_STATUS_DELETE_FAILED = api.NETWORK_STATUS_DELETE_FAILED ) var ( - ALL_NETWORK_TYPES = []string{ - NETWORK_TYPE_GUEST, - NETWORK_TYPE_BAREMETAL, - NETWORK_TYPE_CONTAINER, - NETWORK_TYPE_PXE, - NETWORK_TYPE_IPMI, - } + ALL_NETWORK_TYPES = api.ALL_NETWORK_TYPES ) type IPAddlocationDirection string @@ -706,119 +698,39 @@ func (manager *SNetworkManager) TotalPortCount( return NetworkPortStat{Count: ct, CountExt: ctExt} } -type SNetworkConfig struct { - Network string - Wire string - Exit bool - Private bool - Mac string - Address string - Address6 string - Driver string - BwLimit int - Vip bool - Reserved bool - Ifname string - NetType string - - RequireTeaming bool - TryTeaming bool - - StandbyPortCount int - StandbyAddrCount int -} - type SNicConfig struct { Mac string Index int8 Ifname string } -func parseNetworkInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SNetworkConfig, error) { - netConfig := SNetworkConfig{} - - netJson, ok := info.(*jsonutils.JSONDict) - if ok { - err := netJson.Unmarshal(&netConfig) - if err != nil { +func parseNetworkInfo(userCred mcclient.TokenCredential, info *api.NetworkConfig) (*api.NetworkConfig, error) { + netObj, err := NetworkManager.FetchByIdOrName(userCred, info.Network) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(NetworkManager.Keyword(), info.Network) + } else { return nil, err } - return &netConfig, nil } - netStr, err := info.GetString() - if err != nil { - log.Errorf("invalid networkinfo format %s", err) - return nil, err + net := netObj.(*SNetwork) + if net.IsOwner(userCred) || net.IsPublic || db.IsAdminAllowGet(userCred, net) { + info.Network = netObj.GetId() + } else { + return nil, httperrors.NewForbiddenError("no allow to access network %s", info.Network) } - parts := strings.Split(netStr, ":") - for _, p := range parts { - if len(p) == 0 { - continue - } - if regutils.MatchIP4Addr(p) { - netConfig.Address = p - } else if regutils.MatchIP6Addr(p) { - netConfig.Address6 = p - } else if regutils.MatchCompactMacAddr(p) { - netConfig.Mac = netutils.MacUnpackHex(p) - } else if strings.HasPrefix(p, "wire=") { - netConfig.Wire = p[len("wire="):] - } else if p == "[random_exit]" { - netConfig.Exit = true - } else if p == "[random]" { - netConfig.Exit = false - } else if p == "[private]" { - netConfig.Private = true - } else if p == "[reserved]" { - netConfig.Reserved = true - } else if p == "[teaming]" { - netConfig.RequireTeaming = true - } else if p == "[try-teaming]" { - netConfig.TryTeaming = true - } else if strings.HasPrefix(p, "standby-port=") { - netConfig.StandbyPortCount, _ = strconv.Atoi(p[len("standby-port="):]) - } else if strings.HasPrefix(p, "standby-addr=") { - netConfig.StandbyAddrCount, _ = strconv.Atoi(p[len("standby-addr="):]) - } else if utils.IsInStringArray(p, []string{"virtio", "e1000", "vmxnet3"}) { - netConfig.Driver = p - } else if regutils.MatchSize(p) { - bw, err := fileutils.GetSizeMb(p, 'M', 1000) - if err != nil { - return nil, err - } - netConfig.BwLimit = bw - } else if p == "[vip]" { - netConfig.Vip = true - } else if utils.IsInStringArray(p, ALL_NETWORK_TYPES) { - netConfig.NetType = p - } else { - netObj, err := NetworkManager.FetchByIdOrName(userCred, p) - if err != nil { - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(NetworkManager.Keyword(), p) - } else { - return nil, err - } - } - net := netObj.(*SNetwork) - if net.IsOwner(userCred) || net.IsPublic || db.IsAdminAllowGet(userCred, net) { - netConfig.Network = netObj.GetId() - } else { - return nil, httperrors.NewForbiddenError("no allow to access network %s", p) - } - } + + if info.BwLimit == 0 { + info.BwLimit = options.Options.DefaultBandwidth } - if netConfig.BwLimit == 0 { - netConfig.BwLimit = options.Options.DefaultBandwidth - } - return &netConfig, nil + return info, nil } func (self *SNetwork) getFreeAddressCount() int { return self.getIPRange().AddressCount() - self.GetTotalNicCount() } -func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkConfig) error { +func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *api.NetworkConfig) error { if len(netConfig.Network) > 0 { netObj, err := NetworkManager.FetchByIdOrName(userCred, netConfig.Network) if err != nil { @@ -862,7 +774,7 @@ func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkCo return nil } -func isExitNetworkInfo(netConfig *SNetworkConfig) bool { +func isExitNetworkInfo(netConfig *api.NetworkConfig) bool { if len(netConfig.Network) > 0 { netObj, _ := NetworkManager.FetchById(netConfig.Network) net := netObj.(*SNetwork) diff --git a/pkg/compute/models/schedpolicies.go b/pkg/compute/models/schedpolicies.go index 5f2c957342..a70b0bf1fc 100644 --- a/pkg/compute/models/schedpolicies.go +++ b/pkg/compute/models/schedpolicies.go @@ -8,6 +8,8 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -160,14 +162,14 @@ func (self *SSchedpolicy) PerformEvaluate(ctx context.Context, userCred mcclient desc := server.getSchedDesc() params := jsonutils.NewDict() - params.Add(desc, "server") + params.Add(desc.JSON(desc), "server") meet, err := conditionparser.Eval(self.Condition, params) if err != nil { return nil, err } result := jsonutils.NewDict() - result.Add(desc, "server") + result.Add(desc.JSON(desc), "server") if meet { result.Add(jsonutils.JSONTrue, "result") } else { @@ -176,26 +178,25 @@ func (self *SSchedpolicy) PerformEvaluate(ctx context.Context, userCred mcclient return result, nil } -func ApplySchedPolicies(params *jsonutils.JSONDict) *jsonutils.JSONDict { +func ApplySchedPolicies(params *schedapi.ScheduleInput) *schedapi.ScheduleInput { policies := SchedpolicyManager.getAllEnabledPolicies() if policies == nil { log.Errorf("getAllEnabledPolicies fail") + //return jsonutils.Marshal(params).(*jsonutils.JSONDict) return params } schedtags := make(map[string]string) - if params.Contains("aggregate_strategy") { - err := params.Unmarshal(&schedtags, "aggregate_strategy") - if err != nil { - log.Errorf("unmarshall aggregate_strategy fail %s", err) - return params + if len(params.ServerConfig.Schedtags) != 0 { + for _, tag := range params.ServerConfig.Schedtags { + schedtags[tag.Id] = tag.Strategy } log.Infof("original sched tag %#v", schedtags) } input := jsonutils.NewDict() - input.Add(params, "server") + input.Add(jsonutils.Marshal(params), "server") for i := 0; i < len(policies); i += 1 { meet, err := conditionparser.Eval(policies[i].Condition, input) @@ -207,11 +208,15 @@ func ApplySchedPolicies(params *jsonutils.JSONDict) *jsonutils.JSONDict { } } - newSchedtags := jsonutils.Marshal(schedtags) - log.Infof("updated sched tag %s", newSchedtags) + log.Infof("updated sched tag %s", schedtags) - ret := jsonutils.NewDict() - ret.Add(newSchedtags, "aggregate_strategy") + params.ServerConfig.Schedtags = make([]*api.SchedtagConfig, 0) + for name, strategy := range schedtags { + params.ServerConfig.Schedtags = append(params.ServerConfig.Schedtags, &api.SchedtagConfig{ + Id: name, + Strategy: strategy, + }) + } - return ret + return params } diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index a5a3c68024..692d33c7cd 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -19,16 +20,16 @@ import ( type SchedStrategyType string const ( - STRATEGY_REQUIRE = "require" - STRATEGY_EXCLUDE = "exclude" - STRATEGY_PREFER = "prefer" - STRATEGY_AVOID = "avoid" + STRATEGY_REQUIRE = api.STRATEGY_REQUIRE + STRATEGY_EXCLUDE = api.STRATEGY_EXCLUDE + STRATEGY_PREFER = api.STRATEGY_PREFER + STRATEGY_AVOID = api.STRATEGY_AVOID // # container used aggregate - CONTAINER_AGGREGATE = "container" + CONTAINER_AGGREGATE = api.CONTAINER_AGGREGATE ) -var STRATEGY_LIST = []string{STRATEGY_REQUIRE, STRATEGY_EXCLUDE, STRATEGY_PREFER, STRATEGY_AVOID} +var STRATEGY_LIST = api.STRATEGY_LIST type ISchedtagJointManager interface { db.IJointModelManager diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 6357910f10..263c203a70 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -21,80 +22,68 @@ import ( ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - STORAGE_NFS = "nfs" + STORAGE_LOCAL = api.STORAGE_LOCAL + STORAGE_BAREMETAL = api.STORAGE_BAREMETAL + STORAGE_SHEEPDOG = api.STORAGE_SHEEPDOG + STORAGE_RBD = api.STORAGE_RBD + STORAGE_DOCKER = api.STORAGE_DOCKER + STORAGE_NAS = api.STORAGE_NAS + STORAGE_VSAN = api.STORAGE_VSAN + STORAGE_NFS = api.STORAGE_NFS - STORAGE_PUBLIC_CLOUD = "cloud" - STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" - STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 - STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" //单块本地SSD盘, 容量最大不能超过800 GiB + STORAGE_PUBLIC_CLOUD = api.STORAGE_PUBLIC_CLOUD + STORAGE_CLOUD_EFFICIENCY = api.STORAGE_CLOUD_EFFICIENCY + STORAGE_CLOUD_SSD = api.STORAGE_CLOUD_SSD + STORAGE_CLOUD_ESSD = api.STORAGE_CLOUD_ESSD //增强型(Enhanced)SSD 云盘 + STORAGE_EPHEMERAL_SSD = api.STORAGE_EPHEMERAL_SSD //单块本地SSD盘, 容量最大不能超过800 GiB //Azure hdd and ssd storagetype - STORAGE_STANDARD_LRS = "standard_lrs" - STORAGE_STANDARDSSD_LRS = "standardssd_lrs" - STORAGE_PREMIUM_LRS = "premium_lrs" + STORAGE_STANDARD_LRS = api.STORAGE_STANDARD_LRS + STORAGE_STANDARDSSD_LRS = api.STORAGE_STANDARDSSD_LRS + STORAGE_PREMIUM_LRS = api.STORAGE_PREMIUM_LRS // aws storage type - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_HDD = "sc1" // aws Cold HDD - STORAGE_STANDARD_HDD = "standard" // aws Magnetic volumes + STORAGE_GP2_SSD = api.STORAGE_GP2_SSD // aws general purpose ssd + STORAGE_IO1_SSD = api.STORAGE_IO1_SSD // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = api.STORAGE_ST1_HDD // aws Throughput Optimized HDD + STORAGE_SC1_HDD = api.STORAGE_SC1_HDD // aws Cold HDD + STORAGE_STANDARD_HDD = api.STORAGE_STANDARD_HDD // aws Magnetic volumes // qcloud storage type // STORAGE_CLOUD_SSD ="cloud_ssd" - STORAGE_LOCAL_BASIC = "local_basic" - STORAGE_LOCAL_SSD = "local_ssd" - STORAGE_CLOUD_BASIC = "cloud_basic" - STORAGE_CLOUD_PREMIUM = "cloud_premium" + STORAGE_LOCAL_BASIC = api.STORAGE_LOCAL_BASIC + STORAGE_LOCAL_SSD = api.STORAGE_LOCAL_SSD + STORAGE_CLOUD_BASIC = api.STORAGE_CLOUD_BASIC + STORAGE_CLOUD_PREMIUM = api.STORAGE_CLOUD_PREMIUM // huawei storage type - STORAGE_HUAWEI_SSD = "SSD" // 超高IO云硬盘 - STORAGE_HUAWEI_SAS = "SAS" // 高IO云硬盘 - STORAGE_HUAWEI_SATA = "SATA" // 普通IO云硬盘 + STORAGE_HUAWEI_SSD = api.STORAGE_HUAWEI_SSD // 超高IO云硬盘 + STORAGE_HUAWEI_SAS = api.STORAGE_HUAWEI_SAS // 高IO云硬盘 + STORAGE_HUAWEI_SATA = api.STORAGE_HUAWEI_SATA // 普通IO云硬盘 // openstack - STORAGE_OPENSTACK_ISCSI = "iscsi" + STORAGE_OPENSTACK_ISCSI = api.STORAGE_OPENSTACK_ISCSI ) const ( - STORAGE_ENABLED = "enabled" + STORAGE_ENABLED = api.STORAGE_ENABLED // STORAGE_DISABLED = "disabled" - STORAGE_OFFLINE = "offline" - STORAGE_ONLINE = "online" + STORAGE_OFFLINE = api.STORAGE_OFFLINE + STORAGE_ONLINE = api.STORAGE_ONLINE - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" - DISK_TYPE_HYBRID = "hybrid" + DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE + DISK_TYPE_SSD = api.DISK_TYPE_SSD + DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID ) var ( - DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} - STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} + DISK_TYPES = api.DISK_TYPES + STORAGE_LOCAL_TYPES = api.STORAGE_LOCAL_TYPES STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES - STORAGE_ALL_TYPES = []string{ - STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, - STORAGE_NFS, - } - STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, - STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_EPHEMERAL_SSD, STORAGE_CLOUD_EFFICIENCY, - STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS, - STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, - STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, - STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, - STORAGE_OPENSTACK_ISCSI, - } + STORAGE_ALL_TYPES = api.STORAGE_ALL_TYPES + STORAGE_TYPES = api.STORAGE_TYPES - STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} + STORAGE_LIMITED_TYPES = api.STORAGE_LIMITED_TYPES ) type SStorageManager struct { @@ -847,7 +836,7 @@ func (manager *SStorageManager) TotalCapacity(rangeObj db.IStandaloneModel, host return res1 } -func (self *SStorage) createDisk(name string, diskConfig *SDiskConfig, userCred mcclient.TokenCredential, +func (self *SStorage) createDisk(name string, diskConfig *api.DiskConfig, userCred mcclient.TokenCredential, ownerProjId string, autoDelete bool, isSystem bool, billingType string, billingCycle string, ) (*SDisk, error) { diff --git a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go index e6314034c5..1d1e47b49e 100644 --- a/pkg/compute/tasks/baremetal_convert_hypervisor_task.go +++ b/pkg/compute/tasks/baremetal_convert_hypervisor_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -35,7 +36,12 @@ func (self *BaremetalConvertHypervisorTask) OnInit(ctx context.Context, obj db.I paramsDict := params.(*jsonutils.JSONDict) pendingUsage := models.SQuota{} self.GetPendingUsage(&pendingUsage) - guest.StartGuestCreateTask(ctx, self.UserCred, paramsDict, &pendingUsage, self.GetId()) + input, err := cmdline.FetchServerCreateInputByJSON(paramsDict) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + guest.StartGuestCreateTask(ctx, self.UserCred, input, &pendingUsage, self.GetId()) } func (self *BaremetalConvertHypervisorTask) OnGuestDeployComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { diff --git a/pkg/compute/tasks/disk_batch_create_task.go b/pkg/compute/tasks/disk_batch_create_task.go index 7948067510..af3ebc1750 100644 --- a/pkg/compute/tasks/disk_batch_create_task.go +++ b/pkg/compute/tasks/disk_batch_create_task.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -44,13 +46,30 @@ func (self *DiskBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandalo StartScheduleObjects(ctx, self, toSchedDisks) } +func (self *DiskBatchCreateTask) GetCreateInput() (*api.DiskCreateInput, error) { + input := new(api.DiskCreateInput) + err := self.GetParams().Unmarshal(input) + return input, err +} + +func (self *DiskBatchCreateTask) GetSchedParams() (*schedapi.ScheduleInput, error) { + input, err := self.GetCreateInput() + if err != nil { + return nil, err + } + ret := new(schedapi.ScheduleInput) + srvInput := input.ToServerCreateInput() + err = srvInput.JSON(srvInput).Unmarshal(ret) + return ret, err +} + func (self *DiskBatchCreateTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) { self.SSchedTask.OnScheduleFailCallback(ctx, obj, reason) disk := obj.(*models.SDisk) log.Errorf("Schedule disk %s failed", disk.Name) } -func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { var err error disk := obj.(*models.SDisk) pendingUsage := models.SQuota{} @@ -58,10 +77,25 @@ func (self *DiskBatchCreateTask) SaveScheduleResult(ctx context.Context, obj ISc if err != nil { log.Errorf("GetPendingUsage fail %s", err) } - diskConfig := models.SDiskConfig{} - self.GetParams().Unmarshal(&diskConfig, "disk.0") + quotaStorage := models.SQuota{Storage: disk.DiskSize} - err = disk.SetStorageByHost(hostId, &diskConfig) + + onError := func(err error) { + models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, disk.ProjectId, &pendingUsage, "aStorage) + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, err.Error()) + self.SetStageFailed(ctx, err.Error()) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE_FAIL, err, self.UserCred) + notifyclient.NotifySystemError(disk.Id, disk.Name, models.DISK_ALLOC_FAILED, err.Error()) + } + + diskConfig, err := self.GetFirstDisk() + if err != nil { + onError(err) + return + } + + //err = disk.SetStorageByHost(hostId, &diskConfig) + err = disk.SetStorage(candidate.Disks[0].StorageId, diskConfig) if err != nil { models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, disk.ProjectId, &pendingUsage, "aStorage) disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, err.Error()) diff --git a/pkg/compute/tasks/guest_backup_tasks.go b/pkg/compute/tasks/guest_backup_tasks.go index 3d88edbae7..1d90930cdc 100644 --- a/pkg/compute/tasks/guest_backup_tasks.go +++ b/pkg/compute/tasks/guest_backup_tasks.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -183,15 +184,15 @@ func (self *GuestCreateBackupTask) OnStartSchedule(obj IScheduleModel) { db.OpsLog.LogEvent(guest, db.ACT_START_CREATE_BACKUP, "", self.UserCred) } -func (self *GuestCreateBackupTask) GetSchedParams() *jsonutils.JSONDict { +func (self *GuestCreateBackupTask) GetSchedParams() (*schedapi.ScheduleInput, error) { obj := self.GetObject() guest := obj.(*models.SGuest) schedDesc := guest.ToSchedDesc() if self.Params.Contains("prefer_host_id") { - preferHostId, _ := self.Params.Get("prefer_host_id") - schedDesc.Set("prefer_host_id", preferHostId) + preferHostId, _ := self.Params.GetString("prefer_host_id") + schedDesc.ServerConfig.PreferHost = preferHostId } - return schedDesc + return schedDesc, nil } func (self *GuestCreateBackupTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) { @@ -204,9 +205,10 @@ func (self *GuestCreateBackupTask) OnScheduleFailed(ctx context.Context, reason self.TaskFailed(ctx, guest, reason) } -func (self *GuestCreateBackupTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, targetHostId string) { +func (self *GuestCreateBackupTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { guest := obj.(*models.SGuest) - targetHost := models.HostManager.FetchHostById(targetHostId) + targetHostId := candidate.HostId + targetHost := models.HostManager.FetchHostById(candidate.HostId) if targetHost == nil { self.TaskFailed(ctx, guest, "target host not found?") return diff --git a/pkg/compute/tasks/guest_batch_create_task.go b/pkg/compute/tasks/guest_batch_create_task.go index 398c751e31..54f3e2bb3f 100644 --- a/pkg/compute/tasks/guest_batch_create_task.go +++ b/pkg/compute/tasks/guest_batch_create_task.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -21,6 +23,12 @@ func init() { taskman.RegisterTask(GuestBatchCreateTask{}) } +func (self *GuestBatchCreateTask) GetCreateInput() (*api.ServerCreateInput, error) { + input := new(api.ServerCreateInput) + err := self.GetParams().Unmarshal(input) + return input, err +} + func (self *GuestBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandaloneModel, body jsonutils.JSONObject) { StartScheduleObjects(ctx, self, objs) } @@ -33,13 +41,13 @@ func (self *GuestBatchCreateTask) OnScheduleFailCallback(ctx context.Context, ob } } -func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) { +func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) { guest := obj.(*models.SGuest) - guest.SetHostIdWithBackup(self.UserCred, master, slave) + guest.SetHostIdWithBackup(self.UserCred, master.HostId, slave.HostId) self.SaveScheduleResult(ctx, obj, master) } -func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest *models.SGuest) error { +func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest *models.SGuest, candidate *schedapi.CandidateResource) error { pendingUsage := models.SQuota{} err := self.GetPendingUsage(&pendingUsage) if err != nil { @@ -51,18 +59,25 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest host := guest.GetHost() + input, err := self.GetCreateInput() + if host.IsPrepaidRecycle() { - params, err := host.SetGuestCreateNetworkAndDiskParams(ctx, self.UserCred, self.Params) + input, err = host.SetGuestCreateNetworkAndDiskParams(ctx, self.UserCred, input) if err != nil { log.Errorf("host.SetGuestCreateNetworkAndDiskParams fail %s", err) guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) return err } - self.Params = params + params := input.JSON(input) self.SaveParams(params) } - err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage) + input, err = self.GetCreateInput() + if err != nil { + guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) + return err + } + err = guest.CreateNetworksOnHost(ctx, self.UserCred, host, input.Networks, &pendingUsage) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -71,8 +86,8 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - guest.GetDriver().PrepareDiskRaidConfig(self.UserCred, host, self.Params) - err = guest.CreateDisksOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage, true, true) + guest.GetDriver().PrepareDiskRaidConfig(self.UserCred, host, input.BaremetalDiskConfigs) + err = guest.CreateDisksOnHost(ctx, self.UserCred, host, input.Disks, &pendingUsage, true, true, candidate) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -81,7 +96,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - err = guest.CreateIsolatedDeviceOnHost(ctx, self.UserCred, host, self.Params, &pendingUsage) + err = guest.CreateIsolatedDeviceOnHost(ctx, self.UserCred, host, input.IsolatedDevices, &pendingUsage) self.SetPendingUsage(&pendingUsage) if err != nil { @@ -100,9 +115,12 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return err } - autoStart := jsonutils.QueryBoolean(self.Params, "auto_start", false) - resetPassword := jsonutils.QueryBoolean(self.Params, "reset_password", true) - passwd, _ := self.Params.GetString("password") + autoStart := input.AutoStart + resetPassword := true + if input.ResetPassword != nil { + resetPassword = *input.ResetPassword + } + passwd := input.Password err = guest.StartRebuildRootTask(ctx, self.UserCred, "", false, autoStart, passwd, resetPassword, true) if err != nil { log.Errorf("start guest create task fail %s", err) @@ -112,7 +130,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return nil } - err = guest.StartGuestCreateTask(ctx, self.UserCred, self.Params, nil, self.GetId()) + err = guest.StartGuestCreateTask(ctx, self.UserCred, input, nil, self.GetId()) if err != nil { log.Errorf("start guest create task fail %s", err) guest.SetStatus(self.UserCred, models.VM_CREATE_FAILED, err.Error()) @@ -121,14 +139,15 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest return nil } -func (self *GuestBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *GuestBatchCreateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { var err error + hostId := candidate.HostId guest := obj.(*models.SGuest) if len(guest.HostId) == 0 { guest.OnScheduleToHost(ctx, self.UserCred, hostId) } - err = self.allocateGuestOnHost(ctx, guest) + err = self.allocateGuestOnHost(ctx, guest, candidate) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, err, self.UserCred) logclient.AddActionLogWithStartable(self, obj, logclient.ACT_ALLOCATE, err.Error(), self.GetUserCred(), false) diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go index d8b3446b9f..cb51f32eb5 100644 --- a/pkg/compute/tasks/guest_change_config_task.go +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -95,14 +96,14 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob } func (self *GuestChangeConfigTask) DoCreateDisksTask(ctx context.Context, guest *models.SGuest) { - iCreateData, err := self.Params.Get("create") - if err != nil || iCreateData == nil { + disks := make([]*api.DiskConfig, 0) + err := self.Params.Unmarshal(disks, "create") + if err != nil || len(disks) == 0 { self.OnCreateDisksComplete(ctx, guest, nil) return } - data := (iCreateData).(*jsonutils.JSONDict) self.SetStage("OnCreateDisksComplete", nil) - guest.StartGuestCreateDiskTask(ctx, self.UserCred, data, self.GetTaskId()) + guest.StartGuestCreateDiskTask(ctx, self.UserCred, disks, self.GetTaskId()) } func (self *GuestChangeConfigTask) OnCreateDisksCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 48f4b4a679..034af4e92a 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -39,7 +40,7 @@ func (self *GuestCreateDiskTask) OnDiskPreparedFailed(ctx context.Context, obj d /* --------------------------------------------- */ type KVMGuestCreateDiskTask struct { - SGuestBaseTask + SGuestCreateDiskBaseTask } func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -48,11 +49,16 @@ func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandalo } func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - var diskIndex = 0 var diskReady = true - for { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + + for _, d := range disks { + diskId := d.DiskId + if !diskReady { break } iDisk, err := models.DiskManager.FetchById(diskId) @@ -66,7 +72,7 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapshotId, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + snapshotId := d.SnapshotId err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshotId, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -75,14 +81,12 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d diskReady = false break } - diskIndex += 1 } - diskIndex = 0 - for { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { + for _, d := range disks { + if !diskReady { break } + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -97,7 +101,6 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d diskReady = false break } - diskIndex += 1 } if diskReady { guest := obj.(*models.SGuest) @@ -117,24 +120,34 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob self.SetStageComplete(ctx, nil) } -type ManagedGuestCreateDiskTask struct { +type SGuestCreateDiskBaseTask struct { SGuestBaseTask } +func (self *SGuestCreateDiskBaseTask) GetInputDisks() ([]api.DiskConfig, error) { + disks := make([]api.DiskConfig, 0) + err := self.GetParams().Unmarshal(&disks, "disks") + return disks, err +} + +type ManagedGuestCreateDiskTask struct { + SGuestCreateDiskBaseTask +} + func (self *ManagedGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { self.SetStage("on_managed_disk_prepared", nil) self.OnManagedDiskPrepared(ctx, obj, data) } func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - var diskIndex = 0 + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } - for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if err != nil { - self.SetStageFailed(ctx, err.Error()) - return - } + for _, d := range disks { + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -142,25 +155,20 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapInfo, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) - err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + snapshot := d.SnapshotId + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshot, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) return } return } - diskIndex += 1 } - diskIndex = 0 guest := obj.(*models.SGuest) - for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { - diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if err != nil { - return - } + for _, d := range disks { + diskId := d.DiskId iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) @@ -185,7 +193,6 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex return } time.Sleep(time.Second * 5) - diskIndex += 1 } self.SetStageComplete(ctx, nil) @@ -198,7 +205,7 @@ func (self *ManagedGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context */ type ESXiGuestCreateDiskTask struct { - SGuestBaseTask + SGuestCreateDiskBaseTask } func (self *ESXiGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -209,14 +216,13 @@ func (self *ESXiGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandal return } - diskIndex := 0 - for { - diskKey := fmt.Sprintf("disk.%d.id", diskIndex) - if !self.Params.Contains(diskKey) { - break - } - diskId, _ := self.Params.GetString(diskKey) - diskIndex += 1 + disks, err := self.GetInputDisks() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + for _, d := range disks { + diskId := d.DiskId guestDisk := guest.GetGuestDisk(diskId) if guestDisk == nil { self.SetStageFailed(ctx, "fail to find guestdisk") diff --git a/pkg/compute/tasks/guest_live_migrate_task.go b/pkg/compute/tasks/guest_live_migrate_task.go index 60b48a5724..bad4bf6e70 100644 --- a/pkg/compute/tasks/guest_live_migrate_task.go +++ b/pkg/compute/tasks/guest_live_migrate_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" @@ -31,15 +32,15 @@ func (self *GuestMigrateTask) OnInit(ctx context.Context, obj db.IStandaloneMode StartScheduleObjects(ctx, self, []db.IStandaloneModel{obj}) } -func (self *GuestMigrateTask) GetSchedParams() *jsonutils.JSONDict { +func (self *GuestMigrateTask) GetSchedParams() (*schedapi.ScheduleInput, error) { obj := self.GetObject() guest := obj.(*models.SGuest) schedDesc := guest.ToSchedDesc() if self.Params.Contains("prefer_host_id") { - preferHostId, _ := self.Params.Get("prefer_host_id") - schedDesc.Set("prefer_host_id", preferHostId) + preferHostId, _ := self.Params.GetString("prefer_host_id") + schedDesc.ServerConfig.PreferHost = preferHostId } - return schedDesc + return schedDesc, nil } func (self *GuestMigrateTask) OnStartSchedule(obj IScheduleModel) { @@ -58,7 +59,8 @@ func (self *GuestMigrateTask) OnScheduleFailed(ctx context.Context, reason strin self.TaskFailed(ctx, guest, reason) } -func (self *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, targetHostId string) { +func (self *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, target *schedapi.CandidateResource) { + targetHostId := target.HostId guest := obj.(*models.SGuest) targetHost := models.HostManager.FetchHostById(targetHostId) if targetHost == nil { diff --git a/pkg/compute/tasks/schedule.go b/pkg/compute/tasks/schedule.go index fc275ab941..f21d4a4b9c 100644 --- a/pkg/compute/tasks/schedule.go +++ b/pkg/compute/tasks/schedule.go @@ -7,6 +7,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" @@ -33,7 +36,7 @@ type IScheduleModel interface { type IScheduleTask interface { GetUserCred() mcclient.TokenCredential - GetSchedParams() *jsonutils.JSONDict + GetSchedParams() (*schedapi.ScheduleInput, error) GetPendingUsage(quota quotas.IQuota) error SetStage(stageName string, data *jsonutils.JSONDict) error SetStageFailed(ctx context.Context, reason string) @@ -41,17 +44,42 @@ type IScheduleTask interface { OnStartSchedule(obj IScheduleModel) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) OnScheduleComplete(ctx context.Context, items []db.IStandaloneModel, data *jsonutils.JSONDict) - SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) - SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) + SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) + SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) OnScheduleFailed(ctx context.Context, reason string) } type SSchedTask struct { taskman.STask + input *schedapi.ScheduleInput } -func (self *SSchedTask) GetSchedParams() *jsonutils.JSONDict { - return self.GetParams() +func (self *SSchedTask) GetSchedParams() (*schedapi.ScheduleInput, error) { + params := self.GetParams() + input, err := cmdline.FetchScheduleInputByJSON(params) + if err != nil { + return nil, fmt.Errorf("Unmarsh to schedule input: %v", err) + } + return input, err +} + +func (self *SSchedTask) GetDisks() ([]*api.DiskConfig, error) { + input, err := self.GetSchedParams() + if err != nil { + return nil, err + } + return input.Disks, nil +} + +func (self *SSchedTask) GetFirstDisk() (*api.DiskConfig, error) { + disks, err := self.GetDisks() + if err != nil { + return nil, err + } + if len(disks) == 0 { + return nil, fmt.Errorf("Empty disks to schedule") + } + return disks[0], nil } func (self *SSchedTask) OnStartSchedule(obj IScheduleModel) { @@ -70,11 +98,11 @@ func (self *SSchedTask) OnScheduleComplete(ctx context.Context, items []db.IStan self.SetStageComplete(ctx, nil) } -func (self *SSchedTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, hostId string) { +func (self *SSchedTask) SaveScheduleResult(ctx context.Context, obj IScheduleModel, candidate *schedapi.CandidateResource) { // ... } -func (self *SSchedTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave string) { +func (self *SSchedTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) { // ... } @@ -101,18 +129,23 @@ func doScheduleObjects( task IScheduleTask, objs []IScheduleModel, ) { - parmas := task.GetSchedParams() - schedtags := models.ApplySchedPolicies(parmas) - - task.SetStage("OnScheduleComplete", schedtags) - - s := auth.GetAdminSession(ctx, options.Options.Region, "") - results, err := modules.SchedManager.DoSchedule(s, parmas, len(objs)) + schedInput, err := task.GetSchedParams() if err != nil { onSchedulerRequestFail(ctx, task, objs, fmt.Sprintf("Scheduler fail: %s", err)) return } - onSchedulerResults(ctx, task, objs, results) + schedInput = models.ApplySchedPolicies(schedInput) + + params := jsonutils.Marshal(schedInput).(*jsonutils.JSONDict) + task.SetStage("OnScheduleComplete", params) + + s := auth.GetAdminSession(ctx, options.Options.Region, "") + output, err := modules.SchedManager.DoSchedule(s, schedInput, len(objs)) + if err != nil { + onSchedulerRequestFail(ctx, task, objs, fmt.Sprintf("Scheduler fail: %s", err)) + return + } + onSchedulerResults(ctx, task, objs, output.Candidates) } func cancelPendingUsage(ctx context.Context, task IScheduleTask) { @@ -122,7 +155,12 @@ func cancelPendingUsage(ctx context.Context, task IScheduleTask) { log.Errorf("Taks GetPendingUsage fail %s", err) return } - ownerProjectId, _ := task.GetSchedParams().GetString("owner_tenant_id") + schedInput, err := task.GetSchedParams() + if err != nil { + log.Errorf("cancelPendingUsage GetSchedParams fail: %s", err) + return + } + ownerProjectId := schedInput.ServerConfig.Project err = models.QuotaManager.CancelPendingUsage(ctx, task.GetUserCred(), ownerProjectId, &pendingUsage, &pendingUsage) if err != nil { log.Errorf("cancelpendingusage error %s", err) @@ -162,31 +200,24 @@ func onSchedulerResults( ctx context.Context, task IScheduleTask, objs []IScheduleModel, - results []jsonutils.JSONObject, + results []*schedapi.CandidateResource, ) { succCount := 0 for idx := 0; idx < len(objs); idx += 1 { obj := objs[idx] result := results[idx] - if result.Contains("candidate", "id") { - hostId, _ := result.GetString("candidate", "id") - onScheduleSucc(ctx, task, obj, hostId) - succCount += 1 - } else if result.Contains("candidate", "master_id") { - master, _ := result.GetString("candidate", "master_id") - slave, _ := result.GetString("candidate", "slave_id") - if len(master) == 0 || len(slave) == 0 { - onObjScheduleFail(ctx, task, obj, "Scheduler candidates not match") - } else { - onMasterSlaveScheduleSucc(ctx, task, obj, master, slave) - } - } else if result.Contains("error") { - msg, _ := result.Get("error") - onObjScheduleFail(ctx, task, obj, fmt.Sprintf("%s", msg)) + + if len(result.Error) != 0 { + onObjScheduleFail(ctx, task, obj, fmt.Sprintf("%s", result.Error)) + continue + } + + if result.BackupCandidate == nil { + // normal schedule + onScheduleSucc(ctx, task, obj, result) } else { - msg := fmt.Sprintf("Unknown scheduler result %s", result) - onObjScheduleFail(ctx, task, obj, msg) - return + // backup schedule + onMasterSlaveScheduleSucc(ctx, task, obj, result, result.BackupCandidate) } } if succCount == 0 { @@ -199,24 +230,25 @@ func onMasterSlaveScheduleSucc( ctx context.Context, task IScheduleTask, obj IScheduleModel, - master, slave string, + master, slave *schedapi.CandidateResource, ) { lockman.LockObject(ctx, obj) defer lockman.ReleaseObject(ctx, obj) task.SaveScheduleResultWithBackup(ctx, obj, master, slave) - models.HostManager.ClearSchedDescCache(master) - models.HostManager.ClearSchedDescCache(slave) + models.HostManager.ClearSchedDescCache(master.HostId) + models.HostManager.ClearSchedDescCache(slave.HostId) } func onScheduleSucc( ctx context.Context, task IScheduleTask, obj IScheduleModel, - hostId string, + candidate *schedapi.CandidateResource, ) { + hostId := candidate.HostId lockman.LockRawObject(ctx, models.HostManager.KeywordPlural(), hostId) defer lockman.ReleaseRawObject(ctx, models.HostManager.KeywordPlural(), hostId) - task.SaveScheduleResult(ctx, obj, hostId) - models.HostManager.ClearSchedDescCache(hostId) + task.SaveScheduleResult(ctx, obj, candidate) + models.HostManager.ClearSchedDescCache(candidate.HostId) } diff --git a/pkg/mcclient/modules/mod_scheduler.go b/pkg/mcclient/modules/mod_scheduler.go index aa54d66164..6561215861 100644 --- a/pkg/mcclient/modules/mod_scheduler.go +++ b/pkg/mcclient/modules/mod_scheduler.go @@ -6,6 +6,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" + + api "yunion.io/x/onecloud/pkg/apis/scheduler" ) var ( @@ -22,50 +24,31 @@ type SchedulerManager struct { ResourceManager } -func (this *SchedulerManager) DoScheduleListResult(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) (*ListResult, error) { - candidates, err := this.DoSchedule(s, params, count) - if err != nil { - return nil, err - } - ret := ListResult{Data: make([]jsonutils.JSONObject, len(candidates))} - for i, candidate := range candidates { - host, err := candidate.Get("candidate") - if err == nil { - ret.Data[i] = host - } else { - ret.Data[i] = candidate - } - } - return &ret, nil -} - -func (this *SchedulerManager) DoSchedule(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) ([]jsonutils.JSONObject, error) { +func (this *SchedulerManager) DoSchedule(s *mcclient.ClientSession, input *api.ScheduleInput, count int) (*api.ScheduleOutput, error) { url := fmt.Sprintf("/%s", this.Keyword) - body := jsonutils.NewDict() - body.Add(params, this.Keyword) if count <= 0 { count = 1 } - body.Add(jsonutils.NewInt(int64(count)), "count") - cands, err := this._post(s, url, body, this.Keyword) + input.Count = count + body := input.JSON(input) + ret, err := this._post(s, url, body, "") if err != nil { return nil, err } - candidates, err := cands.GetArray() + output := new(api.ScheduleOutput) + err = ret.Unmarshal(output) if err != nil { - return nil, fmt.Errorf("Not a valid response") + return nil, fmt.Errorf("Not a valid response: %v", err) } - return candidates, nil + return output, nil } -func (this *SchedulerManager) DoScheduleForecast(s *mcclient.ClientSession, params jsonutils.JSONObject, count int) (bool, error) { - body := jsonutils.NewDict() - body.Add(params, this.Keyword) +func (this *SchedulerManager) DoScheduleForecast(s *mcclient.ClientSession, params *api.ScheduleInput, count int) (bool, error) { if count <= 0 { count = 1 } - body.Add(jsonutils.NewInt(int64(count)), "count") - res, err := this.DoForecast(s, body) + params.Count = count + res, err := this.DoForecast(s, params.JSON(params)) if err != nil { return false, err } @@ -81,9 +64,9 @@ func newSchedIdentURL(action, ident string) string { return fmt.Sprintf("%s/%s", newSchedURL(action), ident) } -func (this *SchedulerManager) Test(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (this *SchedulerManager) Test(s *mcclient.ClientSession, params *api.ScheduleInput) (jsonutils.JSONObject, error) { url := newSchedURL("test") - _, obj, err := this.jsonRequest(s, "POST", url, nil, params) + _, obj, err := this.jsonRequest(s, "POST", url, nil, params.JSON(params)) if err != nil { return nil, err } diff --git a/pkg/mcclient/options/disks.go b/pkg/mcclient/options/disks.go new file mode 100644 index 0000000000..9a937ffa50 --- /dev/null +++ b/pkg/mcclient/options/disks.go @@ -0,0 +1,49 @@ +package options + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" +) + +type DiskCreateOptions struct { + Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` + Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` + Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` + Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` + + NAME string `help:"Name of the disk"` + DISKDESC string `help:"Image size or size of virtual disk"` + Desc string `help:"Description" metavar:"Description"` + Storage string `help:"ID or name of storage where the disk is created"` + Hypervisor string `help:"Hypervisor of this disk, used by schedule"` + Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` + TaskNotify bool `help:"Setup task notify"` +} + +func (o DiskCreateOptions) Params() (*api.DiskCreateInput, error) { + config, err := cmdline.ParseDiskConfig(o.DISKDESC, 0) + if err != nil { + return nil, err + } + for _, desc := range o.Schedtag { + tag, err := cmdline.ParseSchedtagConfig(desc) + if err != nil { + return nil, err + } + config.Schedtags = append(config.Schedtags, tag) + } + params := &api.DiskCreateInput{ + PreferRegion: o.Region, + PreferZone: o.Zone, + PreferWire: o.Wire, + PreferHost: o.Host, + Description: o.Desc, + Name: o.NAME, + DiskConfig: config, + Hypervisor: o.Hypervisor, + } + if o.Storage != "" { + params.Storage = o.Storage + } + return params, nil +} diff --git a/pkg/mcclient/options/schedule.go b/pkg/mcclient/options/schedule.go deleted file mode 100644 index 599708a82e..0000000000 --- a/pkg/mcclient/options/schedule.go +++ /dev/null @@ -1,17 +0,0 @@ -package options - -import "yunion.io/x/jsonutils" - -type ScheduleOptions struct { - Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` - Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` - Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` - Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` - Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei"` - ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` -} - -func (opts *ScheduleOptions) Params() (*jsonutils.JSONDict, error) { - return optionsStructToParams(opts) -} diff --git a/pkg/mcclient/options/schedulers.go b/pkg/mcclient/options/schedulers.go index 85832ef394..f22a37f18c 100644 --- a/pkg/mcclient/options/schedulers.go +++ b/pkg/mcclient/options/schedulers.go @@ -1,90 +1,39 @@ package options import ( - "fmt" - - "yunion.io/x/jsonutils" - + "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/modules" ) type SchedulerTestBaseOptions struct { - Mem int64 `help:"Memory size (MB), default 512" metavar:"MEMORY" default:"512"` - Ncpu int64 `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` - Disk []string `help:"Disk descriptions" nargs:"+"` - DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` - BaremetalDiskConfig []string `help:"Baremetal disk layout configuration"` - Net []string `help:"Network descriptions" metavar:"NETWORK"` - IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` - Schedtag []string `help:"Schedule policy, key = SchedTag name, value = require|exclude|prefer|avoid" metavar:""` - Zone string `help:"Preferred zone where virtual server should be created"` - Host string `help:"Preferred host where virtual server should be created"` - Project string `help:"Owner project ID or Name"` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun"` - Count int64 `help:"Create multiple simultaneously, default 1" default:"1"` - Log bool `help:"Record to schedule history"` + *ServerConfigs + + Mem int `help:"Memory size (MB), default 512" metavar:"MEMORY" default:"512"` + Ncpu int `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` + Log bool `help:"Record to schedule history"` } -func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { - data := jsonutils.NewDict() +func (o SchedulerTestBaseOptions) data(s *mcclient.ClientSession) (*scheduler.ServerConfig, error) { + config, err := o.ServerConfigs.Data() + if err != nil { + return nil, err + } + + data := new(scheduler.ServerConfig) + data.ServerConfigs = config + if o.Mem > 0 { - data.Add(jsonutils.NewInt(o.Mem), "vmem_size") + data.Memory = o.Mem } if o.Ncpu > 0 { - data.Add(jsonutils.NewInt(o.Ncpu), "vcpu_count") - } - for i, d := range o.Disk { - data.Add(jsonutils.NewString(d), fmt.Sprintf("disk.%d", i)) - } - for i, n := range o.Net { - data.Add(jsonutils.NewString(n), fmt.Sprintf("net.%d", i)) - } - for i, g := range o.IsolatedDevice { - data.Add(jsonutils.NewString(g), fmt.Sprintf("isolated_device.%d", i)) - } - if len(o.Host) > 0 { - data.Add(jsonutils.NewString(o.Host), "prefer_host") - } else { - if len(o.Zone) > 0 { - data.Add(jsonutils.NewString(o.Zone), "prefer_zone") - } - if len(o.Schedtag) > 0 { - for i, aggr := range o.Schedtag { - data.Add(jsonutils.NewString(aggr), fmt.Sprintf("aggregate.%d", i)) - } - } - } - if len(o.Project) > 0 { - ret, err := modules.Projects.Get(s, o.Project, nil) - if err != nil { - return nil, err - } - projectId, err := ret.GetString("id") - if err != nil { - return nil, err - } - data.Add(jsonutils.NewString(projectId), "owner_tenant_id") - } - if len(o.Hypervisor) > 0 { - data.Add(jsonutils.NewString(o.Hypervisor), "hypervisor") - if o.Hypervisor == "baremetal" { - for i, c := range o.BaremetalDiskConfig { - data.Add(jsonutils.NewString(c), fmt.Sprintf("baremetal_disk_config.%d", i)) - } - } + data.Ncpu = o.Ncpu } return data, nil } -func (o SchedulerTestBaseOptions) options() *jsonutils.JSONDict { - opt := jsonutils.NewDict() - opt.Add(jsonutils.NewInt(o.Count), "count") - if o.Log { - opt.Add(jsonutils.JSONTrue, "record_to_history") - } else { - opt.Add(jsonutils.JSONFalse, "record_to_history") - } +func (o SchedulerTestBaseOptions) options() *scheduler.ScheduleBaseConfig { + opt := new(scheduler.ScheduleBaseConfig) + opt.RecordLog = o.Log return opt } @@ -95,38 +44,34 @@ type SchedulerTestOptions struct { Details bool `help:"Show suggestion details"` } -func (o SchedulerTestOptions) Params(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { +func (o *SchedulerTestOptions) Params(s *mcclient.ClientSession) (*scheduler.ScheduleInput, error) { data, err := o.data(s) if err != nil { - return data, err - } - params := o.options() - params.Add(jsonutils.NewInt(o.SuggestionLimit), "suggestion_limit") - if o.SuggestionAll { - params.Add(jsonutils.JSONTrue, "suggestion_all") - } else { - params.Add(jsonutils.JSONFalse, "suggestion_all") + return nil, err } + opts := o.options() + input := new(scheduler.ScheduleInput) + input.ServerConfig = *data + input.ScheduleBaseConfig = *opts + input.SuggestionLimit = o.SuggestionLimit + input.SuggestionAll = o.SuggestionAll + input.Details = o.Details - if o.Details { - params.Add(jsonutils.JSONTrue, "suggestion_details") - } else { - params.Add(jsonutils.JSONFalse, "suggestion_details") - } - params.Add(data, "scheduler") - return params, nil + return input, nil } type SchedulerForecastOptions struct { SchedulerTestBaseOptions } -func (o SchedulerForecastOptions) Params(s *mcclient.ClientSession) (*jsonutils.JSONDict, error) { +func (o SchedulerForecastOptions) Params(s *mcclient.ClientSession) (*scheduler.ScheduleInput, error) { data, err := o.data(s) if err != nil { - return data, err + return nil, err } - params := o.options() - params.Add(data, "scheduler") - return params, nil + opts := o.options() + input := new(scheduler.ScheduleInput) + input.ServerConfig = *data + input.ScheduleBaseConfig = *opts + return input, nil } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 179304db20..0c138cf9d7 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -3,9 +3,16 @@ package options import ( "fmt" "io/ioutil" + "strconv" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" + "yunion.io/x/pkg/util/regutils" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" ) type ServerListOptions struct { @@ -61,14 +68,8 @@ type ServerShowOptions struct { WithMeta *bool `help:"With meta data"` } -type ServerDeployInfo struct { - Action string - Path string - Content string -} - -func ParseServerDeployInfo(info string) (*ServerDeployInfo, error) { - sdi := &ServerDeployInfo{} +func ParseServerDeployInfo(info string) (*computeapi.DeployConfig, error) { + sdi := &computeapi.DeployConfig{} colon := strings.IndexByte(info, ':') if colon <= 0 { return nil, fmt.Errorf("malformed deploy info: %s", info) @@ -90,33 +91,117 @@ func ParseServerDeployInfo(info string) (*ServerDeployInfo, error) { return sdi, nil } -func ParseServerDeployInfoList(list []string) (*jsonutils.JSONDict, error) { - params := jsonutils.NewDict() - for i, info := range list { - ret, err := ParseServerDeployInfo(info) +func ParseServerDeployInfoList(list []string) ([]*computeapi.DeployConfig, error) { + ret := make([]*computeapi.DeployConfig, 0) + for _, info := range list { + deployInfo, err := ParseServerDeployInfo(info) if err != nil { return nil, err } - params.Set(fmt.Sprintf("deploy.%d.action", i), jsonutils.NewString(ret.Action)) - params.Set(fmt.Sprintf("deploy.%d.path", i), jsonutils.NewString(ret.Path)) - params.Set(fmt.Sprintf("deploy.%d.content", i), jsonutils.NewString(ret.Content)) + ret = append(ret, deployInfo) } - return params, nil + return ret, nil +} + +type ServerConfigs struct { + Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` + Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` + Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` + Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` + BackupHost string `help:"Perfered host where virtual backup server should be created"` + + Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei"` + ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` + Backup bool `help:"Create server with backup server"` + + Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` + Disk []string `help:"Disk descriptions" nargs:"+"` + DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` + Net []string `help:"Network descriptions" metavar:"NETWORK"` + IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` + RaidConfig []string `help:"Baremetal raid config" json:"-"` + Project string `help:"'Owner project ID or Name" json:"tenant"` + User string `help:"Owner user ID or Name"` + Count int `help:"Create multiple simultaneously" default:"1"` +} + +func (o ServerConfigs) Data() (*computeapi.ServerConfigs, error) { + data := &computeapi.ServerConfigs{ + PreferRegion: o.Region, + PreferZone: o.Zone, + PreferWire: o.Wire, + PreferHost: o.Host, + PreferBackupHost: o.BackupHost, + Hypervisor: o.Hypervisor, + ResourceType: o.ResourceType, + Project: o.Project, + Backup: o.Backup, + Count: o.Count, + } + for i, d := range o.Disk { + disk, err := cmdline.ParseDiskConfig(d, i) + if err != nil { + return nil, err + } + data.Disks = append(data.Disks, disk) + } + for _, dtag := range o.DiskSchedtag { + idx, tag, err := cmdline.ParseResourceSchedtagConfig(dtag) + if err != nil { + return nil, fmt.Errorf("ParseDiskSchedtag: %v", err) + } + if idx >= len(data.Disks) { + return nil, fmt.Errorf("Invalid disk index: %d", idx) + } + d := data.Disks[idx] + d.Schedtags = append(d.Schedtags, tag) + } + for i, n := range o.Net { + net, err := cmdline.ParseNetworkConfig(n, i) + if err != nil { + return nil, err + } + data.Networks = append(data.Networks, net) + } + for i, g := range o.IsolatedDevice { + dev, err := cmdline.ParseIsolatedDevice(g, i) + if err != nil { + return nil, err + } + data.IsolatedDevices = append(data.IsolatedDevices, dev) + } + if len(o.RaidConfig) > 0 { + if data.Hypervisor != "baremetal" { + return nil, fmt.Errorf("RaidConfig is applicable to baremetal ONLY") + } + for _, conf := range o.RaidConfig { + raidConf, err := cmdline.ParseBaremetalDiskConfig(conf) + if err != nil { + return nil, err + } + data.BaremetalDiskConfigs = append(data.BaremetalDiskConfigs, raidConf) + } + } + for _, tag := range o.Schedtag { + schedtag, err := cmdline.ParseSchedtagConfig(tag) + if err != nil { + return nil, err + } + data.Schedtags = append(data.Schedtags, schedtag) + } + return data, nil } type ServerCreateOptions struct { - ScheduleOptions + ServerConfigs NAME string `help:"Name of server" json:"-"` MEMSPEC string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"` - Disk []string `help:"Disk descriptions" nargs:"+"` - Net []string `help:"Network descriptions" metavar:"NETWORK"` - IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` Keypair string `help:"SSH Keypair"` Password string `help:"Default user password"` Iso string `help:"ISO image ID" metavar:"IMAGE_ID" json:"cdrom"` - VcpuCount *int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` + VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl"` Vdi string `help:"VDI protocool" choices:"vnc|spice"` Bios string `help:"BIOS" choices:"BIOS|UEFI"` @@ -125,18 +210,12 @@ type ServerCreateOptions struct { NoAccountInit *bool `help:"Not reset account password"` AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` ShutdownBehavior string `help:"Behavior after VM server shutdown, stop or terminate server" metavar:"" choices:"stop|terminate"` - AutoStart *bool `help:"Auto start server after it is created"` - Backup *bool `help:"Create server with backup server" json:"backup"` - BackupHost string `help:"Perfered host where virtual backup server should be created" json:"prefer_backup_host"` + AutoStart bool `help:"Auto start server after it is created"` Deploy []string `help:"Specify deploy files in virtual server file system" json:"-"` Group []string `help:"Group of virtual server"` - Project string `help:"'Owner project ID or Name" json:"tenant"` - User string `help:"Owner user ID or Name"` - System *bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` + System bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` TaskNotify *bool `help:"Setup task notify" json:"-"` - Count *int `help:"Create multiple simultaneously" default:"1" json:"-"` DryRun *bool `help:"Dry run to test scheduler" json:"-"` - RaidConfig []string `help:"Baremetal raid config" json:"-"` UserDataFile string `help:"user_data file path" json:"-"` Duration string `help:"valid duration of the server, e.g. 1H, 1D, 1W, 1M, 1Y, ADMIN ONLY option"` @@ -150,50 +229,130 @@ type ServerCreateOptions struct { Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` } -func (opts *ServerCreateOptions) Params() (*jsonutils.JSONDict, error) { - schedParams, err := opts.ScheduleOptions.Params() +func (o *ServerCreateOptions) ToScheduleInput() (*schedapi.ScheduleInput, error) { + data := new(schedapi.ServerConfig) + + // only support digit number as for now + memSize, err := strconv.Atoi(o.MEMSPEC) if err != nil { return nil, err } - - params, err := optionsStructToParams(opts) - if err != nil { - return nil, err + data.Memory = memSize + if o.VcpuCount > 0 { + data.Ncpu = o.VcpuCount } - params.Update(schedParams) - - { - deployParams, err := ParseServerDeployInfoList(opts.Deploy) + for i, d := range o.Disk { + disk, err := cmdline.ParseDiskConfig(d, i) if err != nil { return nil, err } - params.Update(deployParams) + data.Disks = append(data.Disks, disk) } + for i, n := range o.Net { + net, err := cmdline.ParseNetworkConfig(n, i) + if err != nil { + return nil, err + } + data.Networks = append(data.Networks, net) + } + for i, g := range o.IsolatedDevice { + dev, err := cmdline.ParseIsolatedDevice(g, i) + if err != nil { + return nil, err + } + data.IsolatedDevices = append(data.IsolatedDevices, dev) + } + count := 1 + if o.Count > 1 { + count = o.Count + } + input := new(schedapi.ScheduleInput) + input.Count = count + input.ServerConfig = *data + if o.DryRun != nil && *o.DryRun { + input.Details = true + } + return input, nil +} + +func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error) { + config, err := opts.ServerConfigs.Data() + if err != nil { + return nil, err + } + + params := &computeapi.ServerCreateInput{ + ServerConfigs: config, + VcpuCount: opts.VcpuCount, + Keypair: opts.Keypair, + Password: opts.Password, + Cdrom: opts.Iso, + Vga: opts.Vga, + Vdi: opts.Vdi, + Bios: opts.Bios, + Description: opts.Desc, + ShutdownBehavior: opts.ShutdownBehavior, + AutoStart: opts.AutoStart, + IsSystem: opts.System, + Duration: opts.Duration, + AutoPrepaidRecycle: opts.AutoPrepaidRecycle, + EipBw: opts.EipBw, + EipChargeType: opts.EipChargeType, + Eip: opts.Eip, + } + + if opts.GenerateName { + params.GenerateName = opts.NAME + } else { + params.Name = opts.NAME + } + if regutils.MatchSize(opts.MEMSPEC) { + memSize, err := fileutils.GetSizeMb(opts.MEMSPEC, 'M', 1024) + if err != nil { + return nil, err + } + params.VmemSize = memSize + } else { + params.InstanceType = opts.MEMSPEC + } + + deployInfos, err := ParseServerDeployInfoList(opts.Deploy) + if err != nil { + return nil, err + } + params.DeployConfigs = deployInfos if len(opts.Boot) > 0 { if opts.Boot == "disk" { - params.Set("boot_order", jsonutils.NewString("cdn")) + params.BootOrder = "cdn" } else { - params.Set("boot_order", jsonutils.NewString("dcn")) + params.BootOrder = "dcn" } } + resetPasswd := false + if opts.NoAccountInit != nil && *opts.NoAccountInit { + params.ResetPassword = &resetPasswd + } else { + params.ResetPassword = nil + } + + if len(opts.UserDataFile) > 0 { + userdata, err := ioutil.ReadFile(opts.UserDataFile) + if err != nil { + return nil, err + } + params.UserData = string(userdata) + } + if BoolV(opts.AllowDelete) { - params.Set("disable_delete", jsonutils.JSONFalse) - } - - if len(opts.RaidConfig) > 0 { - if opts.Hypervisor != "baremetal" { - return nil, fmt.Errorf("RaidConfig is applicable to baremetal ONLY") - } - for i, conf := range opts.RaidConfig { - params.Set(fmt.Sprintf("baremetal_disk_config.%d", i), jsonutils.NewString(conf)) - } + params.DisableDelete = false } if BoolV(opts.DryRun) { - params.Set("suggestion", jsonutils.JSONTrue) + params.Suggestion = true } + return params, nil } @@ -254,25 +413,22 @@ type ServerDeployOptions struct { AutoStart *bool `help:"Auto start server after deployed"` } -func (opts *ServerDeployOptions) Params() (*jsonutils.JSONDict, error) { - params, err := optionsStructToParams(opts) - if err != nil { - return nil, err - } +func (opts *ServerDeployOptions) Params() (*computeapi.ServerDeployInput, error) { + params := new(computeapi.ServerDeployInput) { deleteKeyPair := BoolV(opts.DeleteKeypair) if deleteKeyPair { - params.Add(jsonutils.JSONTrue, "__delete_keypair__") + params.DeleteKeypair = true } else if len(opts.Keypair) > 0 { - params.Add(jsonutils.NewString(opts.Keypair), "keypair") + params.Keypair = opts.Keypair } } { - deployParams, err := ParseServerDeployInfoList(opts.Deploy) + deployInfos, err := ParseServerDeployInfoList(opts.Deploy) if err != nil { return nil, err } - params.Update(deployParams) + params.DeployConfigs = deployInfos } return params, nil } diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go index bcd477b839..6c4b796fd0 100644 --- a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -63,7 +63,7 @@ func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []c type schedtagCandidateW struct { core.Candidater - schedData *api.SchedData + schedData *api.SchedInfo } func (w schedtagCandidateW) GetDynamicSchedDesc() *jsonutils.JSONDict { @@ -96,25 +96,27 @@ func (p *AggregatePredicate) exec(h *PredicateHelper) string { return "" } +func SetCandidateScoreBySchedtag(u *core.Unit, c core.Candidater, aggCountMap map[string]int, postiveScore bool) { + stepScore := core.PriorityStep + if !postiveScore { + stepScore = -stepScore + } + for n, count := range aggCountMap { + u.SetFrontScore( + c.IndexKey(), + score.NewScore(score.TScore(count*stepScore), n), + ) + } +} + func (p *AggregatePredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { hostAggs := c.Getter().HostSchedtags() avoidCountMap := GetSchedtagCount(p.SchedtagPredicate.GetAvoidTags(), hostAggs, api.AggregateStrategyAvoid) preferCountMap := GetSchedtagCount(p.SchedtagPredicate.GetPreferTags(), hostAggs, api.AggregateStrategyPrefer) - setScore := func(aggCountMap map[string]int, postiveScore bool) { - stepScore := core.PriorityStep - if !postiveScore { - stepScore = -stepScore - } - for n, count := range aggCountMap { - u.SetFrontScore( - c.IndexKey(), - score.NewScore(score.TScore(count*stepScore), n), - ) - } - } + setScore := SetCandidateScoreBySchedtag - setScore(preferCountMap, true) - setScore(avoidCountMap, false) + setScore(u, c, preferCountMap, true) + setScore(u, c, avoidCountMap, false) } diff --git a/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go index f1346eefd6..493b3cf45c 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go @@ -22,8 +22,8 @@ func (p *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.Pr d := u.SchedData() freeCPUCount := h.GetInt64("FreeCPUCount", 0) - reqCPUCount := d.VCPUCount - if freeCPUCount < d.VCPUCount { + reqCPUCount := int64(d.Ncpu) + if freeCPUCount < reqCPUCount { totalCPUCount := h.GetInt64("CPUCount", 0) h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) h.SetCapacity(0) diff --git a/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go index feb9ca7d5d..e9696e0f5c 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go @@ -22,7 +22,7 @@ func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core d := u.SchedData() freeMemSize := h.GetInt64("FreeMemSize", 0) - reqMemSize := d.VMEMSize + reqMemSize := int64(d.Memory) if freeMemSize < reqMemSize { totalMemSize := h.GetInt64("MemSize", 0) h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) diff --git a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go index 62191c39ae..8aec50c6a5 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go @@ -7,9 +7,9 @@ import ( "yunion.io/x/pkg/utils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -35,7 +35,7 @@ func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, u.AppendSelectPlugin(p) d := u.SchedData() - if len(d.HostID) > 0 && len(d.Networks) == 0 { + if len(d.HostId) > 0 && len(d.Networks) == 0 { return false, nil } @@ -54,7 +54,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor counters := core.NewCounters() isMigrate := func() bool { - return len(schedData.HostID) > 0 + return len(schedData.HostId) > 0 } isRandomNetworkAvailable := func(private bool, exit bool, wire string) string { @@ -69,7 +69,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor if wire != "" && !utils.HasPrefix(wire, network.WireId) && !utils.HasPrefix(wire, network.GetWire().GetName()) { // re appendError(predicates.ErrWireIsNotMatch) } - if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.OwnerTenantID) { + if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.Project) { // TODO: support reservedNetworks reservedNetworks := 0 restPort := int64(network.GetPorts() - reservedNetworks) @@ -100,12 +100,12 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor h.SetCapacityCounter(counters) } - isNetworkAvaliable := func(network *api.Network) string { - if network.Idx == "" { + isNetworkAvaliable := func(network *computeapi.NetworkConfig) string { + if network.Network == "" { return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire) } for _, net := range candidate.Networks { - if (network.Idx == net.Id || network.Idx == net.Name) && (net.IsPublic || net.ProjectId == schedData.OwnerTenantID) && (net.GetPorts() > 0 || isMigrate()) { + if (network.Network == net.Id || network.Network == net.Name) && (net.IsPublic || net.ProjectId == schedData.Project) && (net.GetPorts() > 0 || isMigrate()) { h.SetCapacity(1) return "" } diff --git a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go index 2abf7694c4..8e170b6c3b 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go @@ -3,9 +3,9 @@ package baremetal import ( "fmt" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -21,22 +21,19 @@ func (p *StoragePredicate) Clone() core.FitPredicate { return &StoragePredicate{} } -func toBaremetalDisks(disks []*api.Disk) []*baremetal.Disk { +func toBaremetalDisks(disks []*computeapi.DiskConfig) []*baremetal.Disk { ret := make([]*baremetal.Disk, len(disks)) for i, disk := range disks { ret[i] = &baremetal.Disk{ - Backend: disk.Backend, - ImageID: disk.ImageID, - Fs: disk.Fs, - Os: disk.Os, - OSDistribution: disk.OSDistribution, - Format: disk.Format, - MountPoint: disk.MountPoint, - Driver: disk.Driver, - Cache: disk.Cache, - ImageDiskFormat: disk.ImageDiskFormat, - Size: disk.Size, - Storage: disk.Storage, + Backend: disk.Backend, + ImageID: disk.ImageId, + Fs: &disk.Fs, + Format: disk.Format, + MountPoint: &disk.Mountpoint, + Driver: &disk.Driver, + Cache: &disk.Cache, + Size: int64(disk.SizeMb), + Storage: &disk.Storage, } } return ret diff --git a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go index e6ba918a9e..ae8c26f893 100644 --- a/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go +++ b/pkg/scheduler/algorithm/predicates/disk_schedtag_predicate.go @@ -6,17 +6,70 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/util/errors" + "yunion.io/x/pkg/utils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) -type DiskStoragesMap map[int][]*api.CandidateStorage +type DiskStoragesMap map[int][]*PredicatedStorage + +func (m DiskStoragesMap) getAllTags(isPrefer bool) []computeapi.SchedtagConfig { + ret := make([]computeapi.SchedtagConfig, 0) + for _, ss := range m { + for _, s := range ss { + var tags []computeapi.SchedtagConfig + if isPrefer { + tags = s.PreferTags + } else { + tags = s.AvoidTags + } + ret = append(ret, tags...) + } + } + return ret +} + +func (m DiskStoragesMap) GetPreferTags() []computeapi.SchedtagConfig { + return m.getAllTags(true) +} + +func (m DiskStoragesMap) GetAvoidTags() []computeapi.SchedtagConfig { + return m.getAllTags(false) +} type CandidateDiskStoragesMap map[string]DiskStoragesMap +type PredicatedStorage struct { + *api.CandidateStorage + PreferTags []computeapi.SchedtagConfig + AvoidTags []computeapi.SchedtagConfig +} + +func newPredicatedStorage(s *api.CandidateStorage, preferTags, avoidTags []computeapi.SchedtagConfig) *PredicatedStorage { + return &PredicatedStorage{ + CandidateStorage: s, + PreferTags: preferTags, + AvoidTags: avoidTags, + } +} + +func (s *PredicatedStorage) isNoTag() bool { + return len(s.PreferTags) == 0 && len(s.AvoidTags) == 0 +} + +func (s *PredicatedStorage) hasPreferTags() bool { + return len(s.PreferTags) != 0 +} + +func (s *PredicatedStorage) hasAvoidTags() bool { + return len(s.AvoidTags) != 0 +} + type DiskSchedtagPredicate struct { BasePredicate plugin.BasePlugin @@ -24,6 +77,8 @@ type DiskSchedtagPredicate struct { SchedtagPredicate *SchedtagPredicate CandidateDiskStoragesMap CandidateDiskStoragesMap + + Hypervisor string } func (p *DiskSchedtagPredicate) Name() string { @@ -36,9 +91,9 @@ func (p *DiskSchedtagPredicate) Clone() core.FitPredicate { } } -func (p *DiskSchedtagPredicate) getSchedtagDisks(disks []*api.Disk) ([]*api.Disk, []*api.Disk) { - noTagDisk := make([]*api.Disk, 0) - tagDisk := make([]*api.Disk, 0) +func (p *DiskSchedtagPredicate) getSchedtagDisks(disks []*computeapi.DiskConfig) ([]*computeapi.DiskConfig, []*computeapi.DiskConfig) { + noTagDisk := make([]*computeapi.DiskConfig, 0) + tagDisk := make([]*computeapi.DiskConfig, 0) for _, d := range disks { if len(d.Schedtags) != 0 { tagDisk = append(tagDisk, d) @@ -55,6 +110,8 @@ func (p *DiskSchedtagPredicate) PreExecute(u *core.Unit, cs []core.Candidater) ( return false, nil } + p.Hypervisor = u.SchedData().Hypervisor + // always select each storages to disks u.AppendSelectPlugin(p) @@ -63,11 +120,11 @@ func (p *DiskSchedtagPredicate) PreExecute(u *core.Unit, cs []core.Candidater) ( type schedtagStorageW struct { candidater *api.CandidateStorage - disk *api.Disk + disk *computeapi.DiskConfig } func (w schedtagStorageW) IndexKey() string { - return fmt.Sprintf("%d:%s", w.disk.Size, w.disk.Backend) + return fmt.Sprintf("%s:%s", w.candidater.GetName(), w.candidater.StorageType) } func (w schedtagStorageW) GetDynamicSchedDesc() *jsonutils.JSONDict { @@ -82,10 +139,10 @@ func (w schedtagStorageW) ResourceType() string { return models.StorageManager.KeywordPlural() } -func (p *DiskSchedtagPredicate) check(d *api.Disk, s *api.CandidateStorage) (bool, error) { +func (p *DiskSchedtagPredicate) check(d *computeapi.DiskConfig, s *api.CandidateStorage) (*PredicatedStorage, error) { allTags, err := GetAllSchedtags(models.StorageManager.KeywordPlural()) if err != nil { - return false, err + return nil, err } tagPredicate := NewSchedtagPredicate(d.Schedtags, allTags) if err := tagPredicate.Check( @@ -94,22 +151,24 @@ func (p *DiskSchedtagPredicate) check(d *api.Disk, s *api.CandidateStorage) (boo disk: d, }, ); err != nil { - return false, err + return nil, err } - return true, nil + avoidTags := tagPredicate.GetAvoidTags() + preferTags := tagPredicate.GetPreferTags() + return newPredicatedStorage(s, preferTags, avoidTags), nil } -func (p *DiskSchedtagPredicate) checkStorages(d *api.Disk, storages []*api.CandidateStorage) ([]*api.CandidateStorage, error) { +func (p *DiskSchedtagPredicate) checkStorages(d *computeapi.DiskConfig, storages []*api.CandidateStorage) ([]*PredicatedStorage, error) { errs := make([]error, 0) - ret := make([]*api.CandidateStorage, 0) + ret := make([]*PredicatedStorage, 0) for _, s := range storages { - _, err := p.check(d, s) + ps, err := p.check(d, s) if err != nil { // append err, storage not suit disk errs = append(errs, err) continue } - ret = append(ret, s) + ret = append(ret, ps) } if len(ret) == 0 { return nil, errors.NewAggregate(errs) @@ -120,7 +179,7 @@ func (p *DiskSchedtagPredicate) checkStorages(d *api.Disk, storages []*api.Candi func (p *DiskSchedtagPredicate) GetDiskStoragesMap(candidateId string) DiskStoragesMap { ret, ok := p.CandidateDiskStoragesMap[candidateId] if !ok { - ret = make(map[int][]*api.CandidateStorage) + ret = make(map[int][]*PredicatedStorage) p.CandidateDiskStoragesMap[candidateId] = ret } return ret @@ -129,40 +188,132 @@ func (p *DiskSchedtagPredicate) GetDiskStoragesMap(candidateId string) DiskStora func (p *DiskSchedtagPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(p, u, c) - //noTagDisks, tagDisks := p.getSchedtagDisks(u.SchedData().Disks) storages := c.Getter().Storages() ds := p.GetDiskStoragesMap(c.IndexKey()) disks := u.SchedData().Disks - for _, d := range disks { + for idx, d := range disks { matchedStorages, err := p.checkStorages(d, storages) if err != nil { h.Exclude(err.Error()) } - ds[d.Index] = matchedStorages + ds[idx] = matchedStorages } return h.GetResult() } +func (p *DiskSchedtagPredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { + storageTags := []models.SSchedtag{} + for _, s := range c.Getter().Storages() { + storageTags = append(storageTags, s.Schedtags...) + } + + ds := p.GetDiskStoragesMap(c.IndexKey()) + avoidTags := ds.GetAvoidTags() + preferTags := ds.GetPreferTags() + + avoidCountMap := GetSchedtagCount(avoidTags, storageTags, api.AggregateStrategyAvoid) + preferCountMap := GetSchedtagCount(preferTags, storageTags, api.AggregateStrategyPrefer) + + setScore := SetCandidateScoreBySchedtag + + setScore(u, c, preferCountMap, true) + setScore(u, c, avoidCountMap, false) +} + func (p *DiskSchedtagPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { res := u.GetAllocatedResource(c.IndexKey()) diskStorages := p.GetDiskStoragesMap(c.IndexKey()) - res.Disks = make([]*core.DiskAllocatedResource, len(diskStorages)) + res.Disks = make([]*schedapi.CandidateDisk, len(diskStorages)) disks := u.SchedData().Disks for idx, ds := range diskStorages { res.Disks[idx] = p.allocatedDiskResource(c, disks[idx], ds) } - log.Errorf("============OnSelectEnd %s called: %#v", c.Getter().Name(), jsonutils.Marshal(res.Disks).String()) } -func (p *DiskSchedtagPredicate) allocatedDiskResource(c core.Candidater, disk *api.Disk, storages []*api.CandidateStorage) *core.DiskAllocatedResource { +func (p *DiskSchedtagPredicate) allocatedDiskResource(c core.Candidater, disk *computeapi.DiskConfig, storages []*PredicatedStorage) *schedapi.CandidateDisk { storage := p.selectStorage(disk, storages) - return &core.DiskAllocatedResource{ + log.Debugf("Select storage %s:%s for disk: %s", storage.Id, storage.Name, disk.Index) + return &schedapi.CandidateDisk{ Index: disk.Index, StorageId: storage.Id, } } -func (p *DiskSchedtagPredicate) selectStorage(d *api.Disk, storages []*api.CandidateStorage) *api.CandidateStorage { - return storages[0] +func (p *DiskSchedtagPredicate) selectStorage(d *computeapi.DiskConfig, storages []*PredicatedStorage) *api.CandidateStorage { + preferStorages := []*api.CandidateStorage{} + noTagStorages := []*api.CandidateStorage{} + avoidStorages := []*api.CandidateStorage{} + for _, storage := range storages { + if p.isStorageFitDisk(storage, d) { + candi := storage.CandidateStorage + if storage.isNoTag() { + noTagStorages = append(noTagStorages, candi) + } else if storage.hasPreferTags() { + preferStorages = append(preferStorages, candi) + } else if storage.hasAvoidTags() { + avoidStorages = append(avoidStorages, candi) + } + } + } + sortStorages := []*api.CandidateStorage{} + sortStorages = append(sortStorages, preferStorages...) + sortStorages = append(sortStorages, noTagStorages...) + sortStorages = append(sortStorages, avoidStorages...) + return p.GetLeastUsedStorage(sortStorages, d.Backend) +} + +func (p *DiskSchedtagPredicate) GetLeastUsedStorage(storages []*api.CandidateStorage, backend string) *api.CandidateStorage { + var backends []string + if backend == computeapi.STORAGE_LOCAL { + backends = []string{computeapi.STORAGE_NAS, computeapi.STORAGE_LOCAL} + } else if len(backend) > 0 { + backends = []string{backend} + } else { + backends = []string{} + } + return p.getLeastUsedStorage(storages, backends) +} + +func (p *DiskSchedtagPredicate) getLeastUsedStorage(storages []*api.CandidateStorage, backends []string) *api.CandidateStorage { + var best *api.CandidateStorage + var bestCap int + for i := 0; i < len(storages); i++ { + s := storages[i] + if len(backends) > 0 { + in, _ := utils.InStringArray(s.StorageType, backends) + if !in { + continue + } + } + capa := s.GetFreeCapacity() + if best == nil || bestCap < capa { + bestCap = capa + best = s + } + } + return best +} + +func (p *DiskSchedtagPredicate) GetHypervisorDriver() models.IGuestDriver { + return models.GetDriver(p.Hypervisor) +} + +func (p *DiskSchedtagPredicate) isStorageFitDisk(storage *PredicatedStorage, d *computeapi.DiskConfig) bool { + if d.Storage != "" { + if storage.Id == d.Storage || storage.Name == d.Storage { + return true + } + return false + } + if storage.StorageType == d.Backend { + return true + } + + for _, stype := range p.GetHypervisorDriver().GetStorageTypes() { + if storage.StorageType == stype { + return true + } + } + return false } diff --git a/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go index f11296603c..6c544ebaa2 100644 --- a/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go @@ -26,7 +26,7 @@ func (f *CPUPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, err data := u.SchedData() - if data.VCPUCount <= 0 { + if data.Ncpu <= 0 { return false, nil } @@ -43,7 +43,7 @@ func (f *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.Pr useRsvd := h.UseReserved() freeCPUCount := hc.GetFreeCPUCount(useRsvd) - reqCPUCount := d.VCPUCount + reqCPUCount := int64(d.Ncpu) if freeCPUCount < reqCPUCount { totalCPUCount := hc.GetTotalCPUCount(useRsvd) h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) diff --git a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go index 9c9610c73c..02420ed12f 100644 --- a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go @@ -37,13 +37,13 @@ func (p *GroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, e for _, r := range d.GroupRelations { if r.Strategy == "exclude" { - p.ExcludeGroups = append(p.ExcludeGroups, r.GroupID) + p.ExcludeGroups = append(p.ExcludeGroups, r.GroupId) } else if r.Strategy == "require" { - p.RequireGroups = append(p.RequireGroups, r.GroupID) + p.RequireGroups = append(p.RequireGroups, r.GroupId) } else if r.Strategy == "avoid" { - p.AvoidGroups = append(p.AvoidGroups, r.GroupID) + p.AvoidGroups = append(p.AvoidGroups, r.GroupId) } else if r.Strategy == "prefer" { - p.PreferGroups = append(p.PreferGroups, r.GroupID) + p.PreferGroups = append(p.PreferGroups, r.GroupId) } } u.AppendSelectPlugin(p) diff --git a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go index 98d12fdaa2..7b1fcad6e9 100644 --- a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go @@ -41,16 +41,16 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool // check by specify device id for _, dev := range reqIsoDevs { - if len(dev.ID) == 0 { + if len(dev.Id) == 0 { continue } - if fDev := hc.GetIsolatedDevice(dev.ID); fDev != nil { + if fDev := hc.GetIsolatedDevice(dev.Id); fDev != nil { if len(fDev.GuestID) != 0 { - h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.ID, fDev.GuestID)) + h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.Id, fDev.GuestID)) return h.GetResult() } } else { - h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.ID)) + h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.Id)) return h.GetResult() } minCapacity = 1 @@ -72,8 +72,8 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool // check host device by type devTypeRequest := make(map[string]int, 0) for _, dev := range reqIsoDevs { - if len(dev.Type) != 0 { - devTypeRequest[dev.Type] += 1 + if len(dev.DevType) != 0 { + devTypeRequest[dev.DevType] += 1 } } for devType, reqCount := range devTypeRequest { diff --git a/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go index 4b505a8bb4..eb51cb41a9 100644 --- a/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go @@ -27,7 +27,7 @@ func (p *MemoryPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, data := u.SchedData() - if data.VMEMSize <= 0 { + if data.Memory <= 0 { return false, nil } @@ -44,7 +44,7 @@ func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core useRsvd := h.UseReserved() freeMemSize := hc.GetFreeMemSize(useRsvd) - reqMemSize := d.VMEMSize + reqMemSize := int64(d.Memory) if freeMemSize < reqMemSize { totalMemSize := hc.GetTotalMemSize(useRsvd) h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) diff --git a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go index 10c53736ce..ed50fe59c8 100644 --- a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go @@ -19,13 +19,13 @@ func (p *MigratePredicate) Clone() core.FitPredicate { } func (p *MigratePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { - return len(u.SchedData().HostID) > 0, nil + return len(u.SchedData().HostId) > 0, nil } func (p *MigratePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := predicates.NewPredicateHelper(p, u, c) - if u.SchedData().HostID == c.IndexKey() { + if u.SchedData().HostId == c.IndexKey() { h.Exclude(predicates.ErrHostIsSpecifiedForMigration) } diff --git a/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go index ec75e9a2de..69d8a41ddb 100644 --- a/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go @@ -30,7 +30,7 @@ func (p *NestPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.P d := u.SchedData() - if d.Meta["kvm"] == "enabled" { + if d.Metadata["kvm"] == "enabled" { if hc.Metadata["nest"] != "enabled" { h.Exclude(predicates.ErrNotSupportNest) } diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index cd42c648f1..32e3948a28 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -10,9 +10,9 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" ) @@ -35,7 +35,7 @@ func (p *NetworkPredicate) Clone() core.FitPredicate { func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { data := u.SchedData() - if len(data.HostID) > 0 && len(data.Networks) == 0 { + if len(data.HostId) > 0 && len(data.Networks) == 0 { return false, nil } @@ -53,7 +53,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor d := u.SchedData() isMigrate := func() bool { - return len(d.HostID) > 0 + return len(d.HostId) > 0 } // ServerType's value is 'guest', 'container' or ''(support all type) will return true. @@ -98,7 +98,7 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor appendError(predicates.ErrWireIsNotMatch) } - if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.OwnerTenantID)) { + if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.Project)) { appendError(predicates.ErrNotOwner) } @@ -131,9 +131,9 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor h.SetCapacityCounter(counters) } - isNetworkAvaliable := func(n *api.Network, counters *core.MinCounters, + isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters, networks []models.SNetwork) string { - if n.Idx == "" { + if n.Network == "" { counters0 := core.NewCounters() ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0) counters.Add(counters0) @@ -150,17 +150,17 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID)) continue }*/ - if !(n.Idx == net.GetId() || n.Idx == net.GetName()) { + if !(n.Network == net.GetId() || n.Network == net.GetName()) { errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id)) - } else if !(net.IsPublic || net.ProjectId == d.OwnerTenantID) { - errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.OwnerTenantID)) + } else if !(net.IsPublic || net.ProjectId == d.Project) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.Project)) } else if !(net.GetPorts() > 0 || isMigrate()) { errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id)) } else { // add resource reservedNetworks := 0 counter := counterOfNetwork(u, &net, reservedNetworks) - if counter.GetCount() < d.Count { + if counter.GetCount() < int64(d.Count) { errMsgs = append(errMsgs, fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count)) continue } diff --git a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go index 30a84ea995..0566a41854 100644 --- a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go @@ -44,7 +44,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor d := u.SchedData() isMigrate := func() bool { - return len(d.HostID) > 0 + return len(d.HostId) > 0 } isLocalhostBackend := func(backend string) bool { @@ -72,7 +72,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor ss := make([]string, 0, len(d.Disks)) for _, disk := range d.Disks { if disk.Backend == backend { - ss = append(ss, fmt.Sprintf("%v", disk.Size)) + ss = append(ss, fmt.Sprintf("%v", disk.SizeMb)) } } @@ -97,16 +97,16 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor storeRequest := make(map[string]int64, 0) for _, disk := range d.Disks { if isMigrate() && !isLocalhostBackend(disk.Backend) { - storeRequest[*disk.Storage] = 1 + storeRequest[disk.Storage] = 1 } else { if _, ok := sizeRequest[disk.Backend]; !ok { sizeRequest[disk.Backend] = map[string]int64{"max": -1, "total": 0} } max := sizeRequest[disk.Backend]["max"] - if max < disk.Size { - sizeRequest[disk.Backend]["max"] = disk.Size + if max < int64(disk.SizeMb) { + sizeRequest[disk.Backend]["max"] = int64(disk.SizeMb) } - sizeRequest[disk.Backend]["total"] += disk.Size + sizeRequest[disk.Backend]["total"] += int64(disk.SizeMb) } } diff --git a/pkg/scheduler/algorithm/predicates/schedtag_helper.go b/pkg/scheduler/algorithm/predicates/schedtag_helper.go index 868677205d..c394914073 100644 --- a/pkg/scheduler/algorithm/predicates/schedtag_helper.go +++ b/pkg/scheduler/algorithm/predicates/schedtag_helper.go @@ -6,16 +6,16 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/util/conditionparser" ) type ISchedtagPredicate interface { - GetExcludeTags() []api.Schedtag - GetRequireTags() []api.Schedtag - GetAvoidTags() []api.Schedtag - GetPreferTags() []api.Schedtag + GetExcludeTags() []computeapi.SchedtagConfig + GetRequireTags() []computeapi.SchedtagConfig + GetAvoidTags() []computeapi.SchedtagConfig + GetPreferTags() []computeapi.SchedtagConfig } type ISchedtagCandidate interface { @@ -28,14 +28,14 @@ type ISchedtagCandidate interface { } type SchedtagPredicate struct { - requireTags []api.Schedtag - execludeTags []api.Schedtag - preferTags []api.Schedtag - avoidTags []api.Schedtag + requireTags []computeapi.SchedtagConfig + execludeTags []computeapi.SchedtagConfig + preferTags []computeapi.SchedtagConfig + avoidTags []computeapi.SchedtagConfig checker *SchedtagChecker } -func NewSchedtagPredicate(reqTags []api.Schedtag, allTags []models.SSchedtag) *SchedtagPredicate { +func NewSchedtagPredicate(reqTags []*computeapi.SchedtagConfig, allTags []models.SSchedtag) *SchedtagPredicate { p := new(SchedtagPredicate) requireTags, execludeTags, preferTags, avoidTags := GetRequestSchedtags(reqTags, allTags) p.requireTags = requireTags @@ -46,19 +46,19 @@ func NewSchedtagPredicate(reqTags []api.Schedtag, allTags []models.SSchedtag) *S return p } -func (p *SchedtagPredicate) GetExcludeTags() []api.Schedtag { +func (p *SchedtagPredicate) GetExcludeTags() []computeapi.SchedtagConfig { return p.execludeTags } -func (p *SchedtagPredicate) GetRequireTags() []api.Schedtag { +func (p *SchedtagPredicate) GetRequireTags() []computeapi.SchedtagConfig { return p.requireTags } -func (p *SchedtagPredicate) GetAvoidTags() []api.Schedtag { +func (p *SchedtagPredicate) GetAvoidTags() []computeapi.SchedtagConfig { return p.avoidTags } -func (p *SchedtagPredicate) GetPreferTags() []api.Schedtag { +func (p *SchedtagPredicate) GetPreferTags() []computeapi.SchedtagConfig { return p.preferTags } @@ -66,12 +66,12 @@ func (p *SchedtagPredicate) Check(candidate ISchedtagCandidate) error { return p.checker.Check(p, candidate) } -func GetSchedtagCount(inTags []api.Schedtag, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { +func GetSchedtagCount(inTags []computeapi.SchedtagConfig, objTags []models.SSchedtag, strategy string) (countMap map[string]int) { countMap = make(map[string]int) - in := func(objTag models.SSchedtag, inTags []api.Schedtag) bool { + in := func(objTag models.SSchedtag, inTags []computeapi.SchedtagConfig) bool { for _, tag := range inTags { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } @@ -94,31 +94,31 @@ func GetAllSchedtags(resType string) ([]models.SSchedtag, error) { return tags, nil } -func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (requireTags, execludeTags, preferTags, avoidTags []api.Schedtag) { - requireTags = make([]api.Schedtag, 0) - execludeTags = make([]api.Schedtag, 0) - preferTags = make([]api.Schedtag, 0) - avoidTags = make([]api.Schedtag, 0) +func GetRequestSchedtags(reqTags []*computeapi.SchedtagConfig, allTags []models.SSchedtag) (requireTags, execludeTags, preferTags, avoidTags []computeapi.SchedtagConfig) { + requireTags = make([]computeapi.SchedtagConfig, 0) + execludeTags = make([]computeapi.SchedtagConfig, 0) + preferTags = make([]computeapi.SchedtagConfig, 0) + avoidTags = make([]computeapi.SchedtagConfig, 0) appendedTagIds := make(map[string]int) - appendTagByStrategy := func(tag api.Schedtag) { + appendTagByStrategy := func(tag *computeapi.SchedtagConfig) { switch tag.Strategy { case models.STRATEGY_REQUIRE: - requireTags = append(requireTags, tag) + requireTags = append(requireTags, *tag) case models.STRATEGY_EXCLUDE: - execludeTags = append(execludeTags, tag) + execludeTags = append(execludeTags, *tag) case models.STRATEGY_PREFER: - preferTags = append(preferTags, tag) + preferTags = append(preferTags, *tag) case models.STRATEGY_AVOID: - avoidTags = append(avoidTags, tag) + avoidTags = append(avoidTags, *tag) } } for _, tag := range reqTags { appendTagByStrategy(tag) - appendedTagIds[tag.Idx] = 1 + appendedTagIds[tag.Id] = 1 } for _, tag := range allTags { @@ -126,7 +126,7 @@ func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (re _, idOk := appendedTagIds[tag.Id] if !(nameOk || idOk) { - apiTag := api.Schedtag{Idx: tag.Id, Strategy: tag.DefaultStrategy} + apiTag := &computeapi.SchedtagConfig{Id: tag.Id, Strategy: tag.DefaultStrategy} appendTagByStrategy(apiTag) } } @@ -136,11 +136,11 @@ func GetRequestSchedtags(reqTags []api.Schedtag, allTags []models.SSchedtag) (re type SchedtagChecker struct{} -type apiTags []api.Schedtag +type apiTags []computeapi.SchedtagConfig func (t apiTags) contains(objTag models.SSchedtag) bool { for _, tag := range t { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } @@ -149,25 +149,25 @@ func (t apiTags) contains(objTag models.SSchedtag) bool { type objTags []models.SSchedtag -func (t objTags) contains(atag api.Schedtag) bool { +func (t objTags) contains(atag computeapi.SchedtagConfig) bool { for _, tag := range t { - if tag.Id == atag.Idx || tag.Name == atag.Idx { + if tag.Id == atag.Id || tag.Name == atag.Id { return true } } return false } -func (c *SchedtagChecker) contains(tags []api.Schedtag, objTag models.SSchedtag) bool { +func (c *SchedtagChecker) contains(tags []computeapi.SchedtagConfig, objTag models.SSchedtag) bool { for _, tag := range tags { - if tag.Idx == objTag.Id || tag.Idx == objTag.Name { + if tag.Id == objTag.Id || tag.Id == objTag.Name { return true } } return false } -func (c *SchedtagChecker) HasIntersection(tags []api.Schedtag, objTags []models.SSchedtag) (bool, *models.SSchedtag) { +func (c *SchedtagChecker) HasIntersection(tags []computeapi.SchedtagConfig, objTags []models.SSchedtag) (bool, *models.SSchedtag) { var atags apiTags = tags for _, objTag := range objTags { if atags.contains(objTag) { @@ -177,7 +177,7 @@ func (c *SchedtagChecker) HasIntersection(tags []api.Schedtag, objTags []models. return false, nil } -func (c *SchedtagChecker) Contains(objectTags []models.SSchedtag, tags []api.Schedtag) (bool, *api.Schedtag) { +func (c *SchedtagChecker) Contains(objectTags []models.SSchedtag, tags []computeapi.SchedtagConfig) (bool, *computeapi.SchedtagConfig) { var otags objTags = objectTags for _, tag := range tags { if !otags.contains(tag) { @@ -245,17 +245,21 @@ func (c *SchedtagChecker) Check(p ISchedtagPredicate, candidate ISchedtagCandida if err != nil { return err } + execludeTags := p.GetExcludeTags() + requireTags := p.GetRequireTags() + + log.V(10).Debugf("[SchedtagChecker] check candidate: %s requireTags: %#v, execludeTags: %#v, candidateTags: %#v", candidate.IndexKey(), requireTags, execludeTags, candidateTags) + if len(execludeTags) > 0 { if ok, tag := c.HasIntersection(execludeTags, candidateTags); ok { return fmt.Errorf("Execlude by schedtag: '%s:%s'", tag.Name, tag.Id) } } - requireTags := p.GetRequireTags() if len(requireTags) > 0 { if ok, tag := c.Contains(candidateTags, requireTags); !ok { - return fmt.Errorf("Need schedtag: '%s'", tag.Idx) + return fmt.Errorf("Need schedtag: '%s'", tag.Id) } } diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go index 46698dd13d..b1b2e7da59 100644 --- a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go @@ -25,7 +25,7 @@ func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostP return core.HostPriority{}, err } - ownerTenantID := u.SchedData().OwnerTenantID + ownerTenantID := u.SchedData().Project if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { h.SetFrontRawScore(-1 * int(count)) } diff --git a/pkg/scheduler/api/history.go b/pkg/scheduler/api/history.go index 7f79a252ee..16823ccb0b 100644 --- a/pkg/scheduler/api/history.go +++ b/pkg/scheduler/api/history.go @@ -67,7 +67,7 @@ type HistoryDetailArgs struct { type HistoryTask struct { Type string `json:"type"` Status string `json:"status"` - Data *SchedData `json:"data"` + Data *SchedInfo `json:"data"` Time string `json:"time"` Consuming string `json:"consuming"` //Result []SchedResultItem `json:"result"` diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index eca23da386..9d3056555c 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -1,321 +1,144 @@ package api import ( - "encoding/json" - "fmt" - "strconv" - "strings" - - simplejson "github.com/bitly/go-simplejson" + "net/http" + //"yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/compute/baremetal" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + api "yunion.io/x/onecloud/pkg/apis/scheduler" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/compute/models" o "yunion.io/x/onecloud/pkg/scheduler/options" ) -type Meta map[string]string +type SchedInfo struct { + *api.ScheduleInput -type Disk struct { - Backend string `json:"backend"` - ImageID string `json:"image_id"` - Fs *string `json:"fs"` - Os string `json:"os"` - OSDistribution string `json:"os_distribution"` - OsVersion string `json:"os_version"` - Format string `json:"format"` - MountPoint *string `json:"mountpoint"` - Driver *string `json:"driver"` - Cache *string `json:"cache"` - ImageDiskFormat string `json:"image_disk_format"` - Size int64 `json:"size"` - Storage *string `json:"storage"` - Schedtags []Schedtag `json:"schedtags"` - Index int `json:"index"` + Tag string `json:"tag"` + Type string `json:"type"` + IsContainer bool `json:"is_container"` + Candidates []string `json:"candidates"` + + IgnoreFilters map[string]bool `json:"ignore_filters"` + IsSuggestion bool `json:"suggestion"` + ShowSuggestionDetails bool `json:"suggestion_details"` + Raw string } -type Network struct { - Idx string `json:"idx"` - TenantId string `json:"tenant_id"` - Private bool `json:"private"` - Ports int64 `json:"ports"` - Exit bool `json:"exit"` - Wire string `json:"wire"` - Mac string `json:"mac"` - Address string `json:"address"` - Address6 string `json:"address6"` - Driver string `json:"driver"` - BwLimit int64 `json:"bw_limit"` - Vip bool `json:"vip"` - Reserved bool `json:"reserved"` -} - -type IsolatedDevice struct { - ID string `json:"id"` - Type string `json:"dev_type"` - Model string `json:"model"` - Vendor string `json:"vendor"` -} - -type ForGuest struct { - ID string `json:"id"` - Name string `json:"name"` -} - -type Schedtag struct { - Idx string `json:"idx"` - Strategy string `json:"strategy"` -} - -type GroupRelation struct { - GroupID string `json:"group_id"` - Strategy string `json:"strategy"` - Scope string `json:"scope"` -} - -type SchedData struct { - Tag string `json:"tag"` - Type string `json:"type"` - IsContainer bool `json:"is_container"` - Count int64 `json:"count"` - RegionID string `json:"region_id"` - ZoneID string `json:"zone_id"` - HostID string `json:"host_id"` - Candidates []string `json:"candidates"` - OwnerTenantID string `json:"owner_tenant_id"` - OwnerUserID string `json:"owner_user_id"` - VMEMSize int64 `json:"vmem_size"` - VCPUCount int64 `json:"vcpu_count"` - Disks []*Disk `json:"disks"` - Name string `json:"name"` - Networks []*Network `json:"networks"` - IsolatedDevices []*IsolatedDevice `json:"isolated_devices"` - Schedtags []Schedtag `json:"aggregate_stategy"` - Meta Meta `json:"__meta__"` - ForGuests []*ForGuest `json:"for_guests"` - GuestStatus string `json:"guest_status"` - Hypervisor string `json:"hypervisor"` - ResourceType string `json:"resource_type"` - - // VM - Groups []string `json:"group"` - GroupRelations []GroupRelation `json:"group_relations"` - - // baremental - BaremetalDiskConfigs []*baremetal.BaremetalDiskConfig `json:"baremetal_disk_config"` - - // vm backup schedule - // Schedule server with backup server - Backup bool `json:"backup"` - // Backup master host - PreferHostId string `json:"prefer_host_id"` - // Backup host should be scheduled - BackupHostID string `json:"backup_host_id"` -} - -func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, error) { - data := new(SchedData) - - if tag, ok := sjson.CheckGet("tag"); ok { - if str, err := tag.String(); err == nil { - data.Tag = str - } - } - - if zoneID, ok := sjson.CheckGet("prefer_zone_id"); ok { - if str, err := zoneID.String(); err == nil { - data.ZoneID = str - } - } else if zoneID, ok := sjson.CheckGet("prefer_zone"); ok { - if str, err := zoneID.String(); err == nil { - data.ZoneID = str - } - } - - if regionID, ok := sjson.CheckGet("prefer_region_id"); ok { - if str, err := regionID.String(); err == nil { - data.RegionID = str - } - } - - candidates := make([]string, 0) - - if backupObj, ok := sjson.CheckGet("backup"); ok { - if backup, err := backupObj.Bool(); err == nil && backup { - data.Backup = true - } - } - - if hostID, ok := sjson.CheckGet("prefer_host_id"); ok { - if str, err := hostID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } else { - data.PreferHostId = str - } - } - } else if hostID, ok := sjson.CheckGet("prefer_host"); ok { - if str, err := hostID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } else { - data.PreferHostId = str - } - } - } - - if baremetalID, ok := sjson.CheckGet("prefer_baremetal_id"); ok { - if str, err := baremetalID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } - } - } else if baremetalID, ok := sjson.CheckGet("prefer_baremetal"); ok { - if str, err := baremetalID.String(); err == nil { - if !data.Backup { - candidates = append(candidates, str) - } - } - } - - if backupHostID, ok := sjson.CheckGet("prefer_backup_host_id"); ok { - if backHost, err := backupHostID.String(); err == nil { - data.BackupHostID = backHost - } - } else if backupHostID, ok := sjson.CheckGet("prefer_backup_host"); ok { - if backHost, err := backupHostID.String(); err == nil { - data.BackupHostID = backHost - } - } - - if resourceType, ok := sjson.CheckGet("resource_type"); ok { - data.ResourceType = resourceType.MustString() - } else { - data.ResourceType = models.HostResourceTypeShared - } - - data.Candidates = candidates - - err := data.reviseSchedType(sjson) +func FetchSchedInfo(req *http.Request) (*SchedInfo, error) { + body, err := appsrv.FetchJSON(req) if err != nil { return nil, err } - data.reviseSchedHypervisor(sjson) - - data.VMEMSize = sjson.Get("vmem_size").MustInt64() - data.OwnerTenantID = sjson.Get("owner_tenant_id").MustString() - data.OwnerUserID = sjson.Get("owner_user_id").MustString() - data.VCPUCount = sjson.Get("vcpu_count").MustInt64() - data.Name = sjson.Get("name").MustString() - data.Count = count - data.GuestStatus = sjson.Get("guest_status").MustString() - data.Meta = map[string]string{} - data.HostID = sjson.Get("host_id").MustString() - - for key, value := range sjson.Get("__meta__").MustMap() { - data.Meta[key] = fmt.Sprintf("%v", value) - } - - if err := data.fillNetworksInfo(sjson); err != nil { + input, err := cmdline.FetchScheduleInputByJSON(body) + if err != nil { return nil, err } - if err := data.fillIsolatedDeviceInfo(sjson); err != nil { - return nil, err - } - - if err := data.fillDisksInfo(sjson, byTest); err != nil { - return nil, err - } - - data.fillForGuests(sjson) - - if err := data.fillSchedtags(sjson, byTest); err != nil { - return nil, err - } - - if err := data.fillGroupRelations(sjson); err != nil { - return nil, err - } - - if err := data.fillBaremetalDiskConfig(sjson); err != nil { - return nil, err - } - - if err := data.fillGroups(sjson); err != nil { - return nil, err - } - - data.reviseSchedData() + data := NewSchedInfo(input) return data, nil } -func (s *SchedData) reviseSchedHypervisor(sjson *simplejson.Json) { - var finalHypervisor string - switch hypervisor := sjson.Get("hypervisor").MustString(); hypervisor { - case "", SchedTypeKvm: - finalHypervisor = HostHypervisorForKvm - default: - finalHypervisor = hypervisor +func NewSchedInfo(input *api.ScheduleInput) *SchedInfo { + data := new(SchedInfo) + data.ScheduleInput = input + + if data.Hypervisor == "" || data.Hypervisor == SchedTypeKvm { + data.Hypervisor = HostHypervisorForKvm } - s.Hypervisor = finalHypervisor + + candidates := make([]string, 0) + if !data.Backup && data.PreferHost != "" { + candidates = append(candidates, data.PreferHost) + } + + //if !data.Backup && data.PreferBaremetal != "" { + //candidates = append(candidates, data.PreferBaremetal) + //} + + if data.ResourceType == "" { + data.ResourceType = models.HostResourceTypeShared + } + + if len(data.BaremetalDiskConfigs) == 0 { + defaultConfs := []*computeapi.BaremetalDiskConfig{&computeapi.BaremetalDefaultDiskConfig} + data.BaremetalDiskConfigs = defaultConfs + log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) + } + + data.Candidates = candidates + + data.reviseData() + + return data } -func (s *SchedData) reviseSchedType(sjson *simplejson.Json) error { - var bodyType string - if reqType, ok := sjson.CheckGet("type"); ok { - bodyType = reqType.MustString() - } else { - bodyType = HostTypeHost +func (data *SchedInfo) reviseData() { + for _, d := range data.Disks { + if len(d.Backend) == 0 { + d.Backend = computeapi.STORAGE_LOCAL + } } - switch bodyType { - case "", HostTypeHost, SchedTypeGuest, SchedTypeEsxi, SchedTypeKvm, SchedTypeHyperV: - s.Type = SchedTypeGuest + input := data.ScheduleInput - case SchedTypeContainer: - s.Type = SchedTypeGuest - s.IsContainer = true - - case HostTypeBaremetal: - s.Type = SchedTypeBaremetal - - default: - return fmt.Errorf("Sched current type=%s not support", bodyType) + if input.Count <= 0 { + data.Count = 1 } - return nil + + ignoreFilters := make(map[string]bool, len(input.IgnoreFilters)) + for _, filter := range input.IgnoreFilters { + ignoreFilters[filter] = true + } + data.IgnoreFilters = ignoreFilters + + if data.SuggestionLimit == 0 { + data.SuggestionLimit = int64(o.GetOptions().SchedulerTestLimit) + } + + data.Raw = input.JSON(input).String() } -func (d *SchedData) SkipDirtyMarkHost() bool { +func (d *SchedInfo) SkipDirtyMarkHost() bool { isSharePublicCloudProvider := d.IsPublicCloudProvider() && (d.ResourceType == "" || d.ResourceType == models.HostResourceTypeShared) skipByHypervisor := isSharePublicCloudProvider || d.IsContainer || d.Hypervisor == SchedTypeContainer skipByBackup := d.Backup return skipByHypervisor || skipByBackup } -func (d *SchedData) IsPublicCloudProvider() bool { +func (d *SchedInfo) GetCandidateHostTypes() []string { + switch d.Hypervisor { + case SchedTypeContainer: + return []string{HostTypeKubelet, HostHypervisorForKvm} + default: + return []string{d.Hypervisor} + } +} + +func (d *SchedInfo) IsPublicCloudProvider() bool { return PublicCloudProviders.Has(d.Hypervisor) } -func (d *SchedData) getDiskSize(backend string) int64 { +func (d *SchedInfo) getDiskSize(backend string) int64 { total := int64(0) for _, disk := range d.Disks { if disk.Backend == backend { - total += disk.Size + total += int64(disk.SizeMb) } } return total } -func (d *SchedData) AllDiskBackendSize() map[string]int64 { +func (d *SchedInfo) AllDiskBackendSize() map[string]int64 { backendSizeMap := make(map[string]int64, len(d.Disks)) for _, disk := range d.Disks { - newSize := disk.Size + newSize := int64(disk.SizeMb) if size, ok := backendSizeMap[disk.Backend]; ok { newSize += size } @@ -326,552 +149,6 @@ func (d *SchedData) AllDiskBackendSize() map[string]int64 { return backendSizeMap } -func parseSize(s string, defaultUnit string, base int) (int64, error) { - if utils.IsMatch("^\\d+$", s) { - s = s + defaultUnit - } - unit := s[len(s)-1:] - size, err := strconv.ParseFloat(s[0:len(s)-1], 64) - if err != nil { - return 0, err - } - - if unit == "g" || unit == "G" { - size *= float64(base * base * base) - } else if unit == "m" || unit == "M" { - size *= float64(base * base) - } else if unit == "k" || unit == "K" { - size *= float64(base) - } - - return int64(size), nil -} - -func macUnpackHex(mac string) (string, error) { - d := strings.Split(mac, ":") - if len(d) < 6 { - d = strings.Split(mac, "-") - } - if len(d) == 6 { - return strings.Join(d, ""), nil - } - - return "", fmt.Errorf("Mac unpack hex: format error: %s", mac) -} - -func newNetworkFromDesc(desc string) (*Network, error) { - network := new(Network) - network.Driver = "virtio" - network.Idx = "" - network.Wire = "" - network.Exit = false - network.Private = false - network.Mac = "" - network.Address = "" - network.Address6 = "" - - for _, it := range strings.Split(desc, ":") { - if utils.IsMatchIP4(it) { - network.Address = it - } else if utils.IsMatchIP6(it) { - network.Address6 = it - } else if utils.IsMatchCompactMacAddr(it) { - mac, err := macUnpackHex(it) - if err != nil { - return nil, err - } - network.Mac = mac - } else if strings.HasPrefix(it, "wire=") { - network.Wire = it[len("wire="):] - } else if it == "[random_exit]" { - network.Exit = true - } else if it == "[random]" { - network.Exit = false - } else if it == "[private]" { - network.Private = true - } else if it == "[reserved]" { - network.Reserved = true - } else if it == "virtio" || it == "e1000" || it == "sriov" { - network.Driver = it - } else if utils.IsMatchSize(it) { - limit, err := parseSize(it, "M", 1000) - if err != nil { - return nil, err - } - network.BwLimit = limit / 1000 / 1000 - } else if it == "[vip]" { - network.Vip = true - } else { - network.Idx = it - } - } - - return network, nil -} - -func (d *SchedData) fillNetworksInfo(sjson *simplejson.Json) error { - networks := []*Network{} - for index := 0; ; index++ { - netIndex := fmt.Sprintf("net.%d", index) - s, ok := sjson.CheckGet(netIndex) - if !ok { - break - } - net, err := newNetworkFromDesc(s.MustString()) - if err != nil || net.Idx == "" { - net = new(Network) - net.Idx = s.Get("network").MustString() - net.Wire = s.Get("wire").MustString() - net.Driver = s.Get("driver").MustString() - net.Exit = s.Get("exit").MustBool() - net.BwLimit = s.Get("bw_limit").MustInt64() - net.Private = s.Get("private").MustBool() - net.Reserved = s.Get("reserved").MustBool() - net.Vip = s.Get("vip").MustBool() - } - networks = append(networks, net) - } - d.Networks = networks - return nil -} - -func newIsolatedDeviceFromDesc(desc string) (dev *IsolatedDevice, err error) { - dev = new(IsolatedDevice) - for _, it := range strings.Split(desc, ":") { - if utils.IsMatchUUID(it) { - dev.ID = it - } else if ValidPassthroughTypes.Has(it) { - dev.Type = it - } else if strings.HasPrefix(it, "vendor=") { - vendor := it[len("vendor="):] - vid, ok := IsolatedVendorIDMap[strings.ToUpper(vendor)] - if !ok { - vid = vendor - } - dev.Vendor = vid - } else { - dev.Model = it - } - } - if len(dev.ID) == 0 && len(dev.Model) == 0 { - dev = nil - err = fmt.Errorf("Invalid isolated device description: %s", desc) - return - } - return -} - -func (d *SchedData) fillIsolatedDeviceInfo(sjson *simplejson.Json) error { - devs := []*IsolatedDevice{} - for idx := 0; ; idx++ { - devIdx := fmt.Sprintf("isolated_device.%d", idx) - s, ok := sjson.CheckGet(devIdx) - if !ok { - break - } - dev, err := newIsolatedDeviceFromDesc(s.MustString()) - if err != nil { - dev = new(IsolatedDevice) - dev.ID = s.Get("id").MustString() - dev.Model = s.Get("model").MustString() - dev.Vendor = s.Get("vendor").MustString() - dev.Type = s.Get("dev_type").MustString() - if len(dev.ID) == 0 && len(dev.Model) == 0 { - return fmt.Errorf("Invalid isolated device description") - } - } - devs = append(devs, dev) - } - d.IsolatedDevices = devs - return nil -} - -func newDiskFromCmdline(dstr string) (*Disk, error) { - matchFs := func(fs string) bool { - switch fs { - case "swap", "ext2", "ext3", "ext4", "xfs", "ntfs", "fat", "hfsplus": - return true - default: - return false - } - } - - matchFormat := func(f string) bool { - switch f { - case "qcow2", "raw", "docker", "iso", "vmdk": - return true - default: - return false - } - } - - matchDriver := func(d string) bool { - switch d { - case "virtio", "ide", "scsi", "sata", "pvscsi": - return true - default: - return false - } - } - - matchCache := func(c string) bool { - switch c { - case "writeback", "none", "writethrough": - return true - default: - return false - } - } - - matchBackend := func(b string) bool { - switch b { - case "local", "baremetal", "docker": - return true - default: - return false - } - } - disk := &Disk{Backend: "local"} - for _, nd := range strings.Split(dstr, ":") { - d := nd - if utils.IsMatchSize(d) { - size, err := parseSize(d, "G", 1024) - if err != nil { - return nil, err - } - disk.Size = size / 1024 / 1024 - } - if matchFs(d) { - disk.Fs = &d - } - if matchFormat(d) { - disk.Format = d - } - if matchDriver(d) { - disk.Driver = &d - } - if matchCache(d) { - disk.Cache = &d - } - if d == "autoextend" { - disk.Size = -1 - } - if matchBackend(d) { - disk.Backend = d - } - if strings.HasPrefix(d, "/") { - disk.MountPoint = &d - } - } - return disk, nil -} - -func newDiskFromSimpleJson(sjson *simplejson.Json, byTest bool) (*Disk, error) { - disk := new(Disk) - if byTest { - return newDiskFromCmdline(sjson.MustString()) - } - disk.Backend = sjson.Get("backend").MustString() - disk.ImageID = sjson.Get("image_id").MustString() - if fs, ok := sjson.CheckGet("fs"); ok { - if str, err := fs.String(); err == nil { - disk.Fs = &str - } - } - disk.Os = sjson.Get("os").MustString() - disk.OSDistribution = sjson.Get("os_distribution").MustString() - disk.OsVersion = sjson.Get("os_version").MustString() - - disk.Format = sjson.Get("format").MustString() - if mountPoint, ok := sjson.CheckGet("mountpoint"); ok { - if str, err := mountPoint.String(); err == nil { - disk.MountPoint = &str - } - } - if driver, ok := sjson.CheckGet("driver"); ok { - if str, err := driver.String(); err == nil { - disk.Driver = &str - } - } - if cache, ok := sjson.CheckGet("cache"); ok { - if str, err := cache.String(); err == nil { - disk.Cache = &str - } - } - disk.ImageDiskFormat = sjson.Get("image_disk_format").MustString() - disk.Size = sjson.Get("size").MustInt64() - - if storage, ok := sjson.CheckGet("storage"); ok { - if str, err := storage.String(); err == nil { - disk.Storage = &str - } - } - return disk, nil -} - -func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*baremetal.BaremetalDiskConfig, error) { - baremetalDiskConfig := new(baremetal.BaremetalDiskConfig) - baremetalDiskConfig.Count = sjson.Get("count").MustInt64() - baremetalDiskConfig.Conf = sjson.Get("conf").MustString() - - rangeArray := make([]int64, 0) - ranges := sjson.Get("range").MustArray() - for _, size := range ranges { - sizeNum, ok := size.(json.Number) - if !ok { - return nil, fmt.Errorf("Invalid range size, not json.Number: %v", size) - } - rangeSize, err := sizeNum.Int64() - if err != nil { - return nil, err - } - rangeArray = append(rangeArray, rangeSize) - } - baremetalDiskConfig.Range = rangeArray - - baremetalDiskConfig.Splits = sjson.Get("splits").MustString() - baremetalDiskConfig.Type = sjson.Get("type").MustString() - ada := sjson.Get("adapter").MustInt() - baremetalDiskConfig.Adapter = &ada - if val, ok := sjson.CheckGet("strip"); ok { - strip := val.MustInt64() - baremetalDiskConfig.Strip = &strip - } - if val, ok := sjson.CheckGet("cachedbadbbu"); ok { - cachedbadbbu := val.MustBool() - baremetalDiskConfig.Cachedbadbbu = &cachedbadbbu - } - - return baremetalDiskConfig, nil -} - -func (d *SchedData) fillDisksInfo(sjson *simplejson.Json, byTest bool) error { - disks := make([]*Disk, 0) - index := 0 - for { - diskIndex := fmt.Sprintf("disk.%d", index) - d, ok := sjson.CheckGet(diskIndex) - if !ok { - break - } - disk, err := newDiskFromSimpleJson(d, byTest) - if err != nil { - return err - } - disk.Index = index - index++ - disks = append(disks, disk) - } - if index == 0 && d.Hypervisor != SchedTypeContainer { - return fmt.Errorf("No disk info found in json") - } - d.Disks = disks - return nil -} - -func (d *SchedData) fillBaremetalDiskConfig(sjson *simplejson.Json) error { - baremetalDiskConfigs := []*baremetal.BaremetalDiskConfig{} - if d.Hypervisor != HostTypeBaremetal { - return nil - } - config, ok := sjson.CheckGet("baremetal_disk_config") - if !ok { - defaultConfs := []*baremetal.BaremetalDiskConfig{&baremetal.BaremetalDefaultDiskConfig} - d.BaremetalDiskConfigs = defaultConfs - log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) - return nil - } - for index := range config.MustArray() { - baremetalDiskConfig, err := newBaremetalDiskConfigFromSimpleJson(config.GetIndex(index)) - if err != nil { - return err - } - baremetalDiskConfigs = append(baremetalDiskConfigs, baremetalDiskConfig) - } - - d.BaremetalDiskConfigs = baremetalDiskConfigs - return nil -} - -func (d *SchedData) fillForGuests(sjson *simplejson.Json) { - newGuest := func(sjson *simplejson.Json) *ForGuest { - gst := new(ForGuest) - gst.ID = sjson.Get("id").MustString() - gst.Name = sjson.Get("name").MustString() - return gst - } - for i := range sjson.Get("for_guests").MustArray() { - gst := sjson.Get("for_guests").GetIndex(i) - d.ForGuests = append(d.ForGuests, newGuest(gst)) - } -} - -func NewSchedTagFromCmdline(str string) (agg Schedtag, err error) { - rs := strings.Split(str, ":") - if len(rs) == 1 || rs[1] == "" { - err = fmt.Errorf("SchedTag %q no strategy.", str) - return - } - name, strategy := rs[0], rs[1] - - err = SchedtagStrategyCheck(strategy) - if err != nil { - return - } - - agg = Schedtag{name, strategy} - return -} - -func (d *SchedData) fillSchedtags(sjson *simplejson.Json, byTest bool) error { - d.Schedtags = []Schedtag{} - - if !byTest { - if aggNode, ok := sjson.CheckGet("aggregate_strategy"); ok { - for name, strategy := range aggNode.MustMap() { - d.Schedtags = append(d.Schedtags, Schedtag{ - Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), - }) - } - } - } else { - index := 0 - for { - aggIndex := fmt.Sprintf("aggregate.%d", index) - a, ok := sjson.CheckGet(aggIndex) - if !ok { - break - } - agg, err := NewSchedTagFromCmdline(a.MustString()) - if err != nil { - return err - } - d.Schedtags = append(d.Schedtags, agg) - index++ - } - } - - return nil -} - -func (d *SchedData) fillGroupRelations(sjson *simplejson.Json) error { - d.GroupRelations = []GroupRelation{} - - if aggNode, ok := sjson.CheckGet("group_relations"); ok { - for index := range aggNode.MustArray() { - it := aggNode.GetIndex(index) - - r := GroupRelation{} - - if v, ok := it.CheckGet("id"); ok { - r.GroupID = v.MustString() - } else { - return fmt.Errorf("id missing in group relation") - } - - if v, ok := it.CheckGet("scope"); ok { - r.Scope = v.MustString() - } else { - r.Scope = "host" - } - - if v, ok := it.CheckGet("strategy"); ok { - r.Strategy = v.MustString() - } else { - r.Strategy = "" - } - - d.GroupRelations = append(d.GroupRelations, r) - } - } - - return nil -} - -func (d *SchedData) fillGroups(sjson *simplejson.Json) error { - d.Groups = []string{} - if groupsNode, ok := sjson.CheckGet("groups"); ok { - for index := range groupsNode.MustArray() { - it := groupsNode.GetIndex(index) - d.Groups = append(d.Groups, it.MustString()) - } - } - - return nil -} - -// TODO -func (d *SchedData) reviseSchedData() { - /*if strategy, ok := data.Meta["aggregate_strategy"]; ok { - d.Aggregates = append(d.Aggregates, Aggregate{ - Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), - }) - }*/ -} - -type SchedInfo struct { - Data *SchedData `json:"scheduler"` - IgnoreFilters map[string]bool `json:"ignore_filters"` - SessionID string `json:"session_id"` - IsSuggestion bool `json:"suggestion"` - ShowSuggestionDetails bool `json:"suggestion_details"` - SuggestionLimit int64 `json:"suggestion_limit"` - SuggestionAll bool `json:"suggestion_all"` - Raw string `json:"raw"` - BestEffort bool `json:"best_effort"` -} - -func NewSchedInfo(sjson *simplejson.Json, byTest bool) (*SchedInfo, error) { - info := new(SchedInfo) - - schedData, ok := sjson.CheckGet("scheduler") - if !ok { - return nil, fmt.Errorf("Not found 'scheduler' from request body") - - } - - count := sjson.Get("count").MustInt64(1) - - if data, err := NewSchedData(schedData, count, byTest); err == nil { - info.Data = data - } else { - return nil, err - } - - info.SessionID = sjson.Get("session_id").MustString() - - ignoreFiltersString := sjson.Get("ignore_filters").MustString() - filters := strings.Split(ignoreFiltersString, ",") - ignoreFilters := make(map[string]bool, len(filters)) - for _, filter := range filters { - ignoreFilters[filter] = true - - } - info.IgnoreFilters = ignoreFilters - - info.SuggestionLimit = sjson.Get("suggestion_limit").MustInt64() - if info.SuggestionLimit == 0 { - info.SuggestionLimit = int64(o.GetOptions().SchedulerTestLimit) - } - info.SuggestionAll = sjson.Get("suggestion_all").MustBool() - info.ShowSuggestionDetails = sjson.Get("suggestion_details").MustBool() - info.BestEffort = sjson.Get("best_effort").MustBool() - - if bytes, err := sjson.EncodePretty(); err == nil { - info.Raw = string(bytes) - - } - return info, nil -} - -func (i *SchedInfo) String() (string, error) { - bytes, err := json.Marshal(i) - if err != nil { - return "", err - } - return string(bytes), nil -} - type SchedResultItem interface{} type SchedResult struct { diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go index 51dfc2b831..454f2b362c 100644 --- a/pkg/scheduler/cache/candidate/baremetals.go +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -189,7 +189,6 @@ func (bb *BaremetalBuilder) Type() string { } func (bb *BaremetalBuilder) AllIDs() ([]string, error) { - log.Errorf("=======get baremetal host ids") return models.AllBaremetalIDs() } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index fc7fc7aab2..b55a85aa3e 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -172,12 +172,14 @@ func (b *BaseHostDesc) fillNetworks(hostID string) error { return nil } -func (b *BaseHostDesc) fillStorages(storages []*computemodels.SStorage) error { +func (b *BaseHostDesc) fillStorages(host *computemodels.SHost) error { ss := make([]*api.CandidateStorage, 0) - for _, s := range storages { + //log.Errorf("====host %s append storages: %#v", b.Name, storages) + for _, s := range host.GetHoststorages() { + storage := s.GetStorage() ss = append(ss, &api.CandidateStorage{ - SStorage: s, - Schedtags: s.GetSchedtags(), + SStorage: storage, + Schedtags: storage.GetSchedtags(), }) } b.Storages = ss diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 32b9399530..0cad7c9ce1 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -182,8 +182,8 @@ type HostBuilder struct { guestDict map[string]interface{} guestIDs []string - hostStorages []computemodels.SHoststorage - hostStoragesDict map[string][]*computemodels.SStorage + hostStorages []computemodels.SHoststorage + //hostStoragesDict map[string][]*computemodels.SStorage storages []interface{} storageStatesSizeDict map[string]map[string]interface{} @@ -540,7 +540,7 @@ func (b *HostBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGr b.setMetadataInfo(ids, errMessageChannel) }, func() { - b.setStorages(ids, errMessageChannel) + //b.setStorages(ids, errMessageChannel) b.setDiskStats(errMessageChannel) }, func() { b.setMetadataInfo(ids, errMessageChannel) }, @@ -605,32 +605,32 @@ func (b *HostBuilder) setSchedtags(ids []string, errMessageChannel chan error) { b.schedtags = tags } -func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { - q := computemodels.HoststorageManager.Query().In("host_id", ids) - hostStorages := make([]computemodels.SHoststorage, 0) - err := computedb.FetchModelObjects(computemodels.HoststorageManager, q, &hostStorages) - if err != nil { - errMessageChannel <- err - return - } +//func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { +//q := computemodels.HoststorageManager.Query().In("host_id", ids) +//hostStorages := make([]computemodels.SHoststorage, 0) +//err := computedb.FetchModelObjects(computemodels.HoststorageManager, q, &hostStorages) +//if err != nil { +//errMessageChannel <- err +//return +//} - hostStoragesDict := make(map[string][]*computemodels.SStorage) +////hostStoragesDict := make(map[string][]*computemodels.SStorage) - for _, s := range hostStorages { - if ss, ok := hostStoragesDict[s.HostId]; !ok { - storage := s.GetStorage() - ss = make([]*computemodels.SStorage, 0) - ss = append(ss, storage) - hostStoragesDict[s.HostId] = ss - } else { - ss = append(ss, s.GetStorage()) - } - } +//for _, s := range hostStorages { +//if ss, ok := hostStoragesDict[s.HostId]; !ok { +//storage := s.GetStorage() +//ss = make([]*computemodels.SStorage, 0) +//ss = append(ss, storage) +//hostStoragesDict[s.HostId] = ss +//} else { +//ss = append(ss, s.GetStorage()) +//} +//} - b.hostStorages = hostStorages - b.hostStoragesDict = hostStoragesDict - return -} +//b.hostStorages = hostStorages +//b.hostStoragesDict = hostStoragesDict +//return +//} func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { guests, err := FetchGuestByHostIDs(ids) @@ -898,7 +898,6 @@ func (b *HostBuilder) AllIDs() ([]string, error) { } ret = append(ret, id) } - log.Errorf("==============================\n ============================get hostsIDs: %v", ret) return ret, nil } @@ -1126,7 +1125,7 @@ func (b *HostBuilder) guestAppTags(guest computemodels.SGuest) []string { } func (b *HostBuilder) fillStorages(desc *HostDesc, host *computemodels.SHost) error { - return desc.fillStorages(b.hostStoragesDict[host.GetId()]) + return desc.fillStorages(host) } func (b *HostBuilder) fillSchedtags(desc *HostDesc, host *computemodels.SHost) error { diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index 053d0e4852..de22bb09ef 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -394,11 +394,11 @@ func (u *Unit) Info() string { } func (u *Unit) SessionID() string { - return u.SchedInfo.SessionID + return u.SchedInfo.SessionId } -func (u *Unit) SchedData() *api.SchedData { - return u.SchedInfo.Data +func (u *Unit) SchedData() *api.SchedInfo { + return u.SchedInfo } func (u *Unit) IsPublicCloudProvider() bool { diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 0dc59410c3..2ee0941464 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -15,6 +15,7 @@ import ( utiltrace "yunion.io/x/pkg/util/trace" "yunion.io/x/pkg/util/workqueue" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" o "yunion.io/x/onecloud/pkg/scheduler/options" ) @@ -99,12 +100,12 @@ func (g *GenericScheduler) Schedule(unit *Unit, candidates []Candidater) (*Sched // new trace follow all steps trace := utiltrace.New(fmt.Sprintf("SessionID: %s, schedule info: %s", - schedInfo.SessionID, unit.Info())) + schedInfo.SessionId, unit.Info())) defer trace.LogIfLong(100 * time.Millisecond) if len(candidates) == 0 { return nil, &NoResourceError{ - sessionID: schedInfo.SessionID, + sessionID: schedInfo.SessionId, info: unit.Info(), } } @@ -242,6 +243,14 @@ type SchedResultItem struct { *AllocatedResource } +func (item *SchedResultItem) ToCandidateResource() *schedapi.CandidateResource { + return &schedapi.CandidateResource{ + HostId: item.ID, + Name: item.Name, + Disks: item.Disks, + } +} + func GetCapacities(u *Unit, id string) (res map[string]int64) { res = make(map[string]int64) capacities := u.GetCapacities(id) diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index ab420bcaef..f267d52683 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -3,6 +3,7 @@ package core import ( "yunion.io/x/jsonutils" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core/score" @@ -129,17 +130,12 @@ type Priority interface { ScoreIntervals() score.Intervals } -type DiskAllocatedResource struct { - Index int `json:"index"` - StorageId string `json:"storage_id"` -} - type AllocatedResource struct { - Disks []*DiskAllocatedResource `json:"disks"` + Disks []*schedapi.CandidateDisk `json:"disks"` } func NewAllocatedResource() *AllocatedResource { return &AllocatedResource{ - Disks: make([]*DiskAllocatedResource, 0), + Disks: make([]*schedapi.CandidateDisk, 0), } } diff --git a/pkg/scheduler/data_manager/candidate_manager.go b/pkg/scheduler/data_manager/candidate_manager.go index 36dd18637d..3071b2fc76 100644 --- a/pkg/scheduler/data_manager/candidate_manager.go +++ b/pkg/scheduler/data_manager/candidate_manager.go @@ -5,6 +5,7 @@ import ( "time" "yunion.io/x/pkg/util/ttlpool" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/scheduler/cache" candidatecache "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" @@ -15,9 +16,10 @@ import ( type CandidateGetArgs struct { // ResType is candidate host_type - ResType string - RegionID string - ZoneID string + ResType string + RegionID string + ZoneID string + HostTypes []string } type DataManager struct { @@ -222,6 +224,13 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida return true } + matchHostTypes := func(c core.Candidater, hostTypes []string) bool { + if len(hostTypes) == 0 { + return true + } + return utils.IsInStringArray(c.Getter().HostType(), hostTypes) + } + for _, c := range candidates { r := c.(core.Candidater) @@ -237,6 +246,10 @@ func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candida continue } + if !matchHostTypes(r, args.HostTypes) { + continue + } + result = append(result, r) } diff --git a/pkg/scheduler/handler/backup_helper.go b/pkg/scheduler/handler/backup_helper.go index da5012df92..ef90c016da 100644 --- a/pkg/scheduler/handler/backup_helper.go +++ b/pkg/scheduler/handler/backup_helper.go @@ -6,7 +6,7 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/scheduler/api" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/scheduler/core" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" ) @@ -18,24 +18,27 @@ func transToBackupSchedResult(result *core.SchedResultItemList, preferMasterHost } apiResults := newBackupSchedResult(result, preferMasterHost, preferBackupHost, count) - return regionResponse(apiResults) + return apiResults } -func newBackupSchedResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string, count int64) []api.SchedResultItem { - apiResults := make([]api.SchedResultItem, 0) +func newBackupSchedResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string, count int64) *schedapi.ScheduleOutput { + ret := new(schedapi.ScheduleOutput) + apiResults := make([]*schedapi.CandidateResource, 0) for i := 0; i < int(count); i++ { log.V(10).Debugf("Select backup host from result: %s", result) target, err := getSchedBackupResult(result, preferMasterHost, preferBackupHost) if err != nil { - apiResults = append(apiResults, api.SchedErrItem{Error: err.Error()}) + er := &schedapi.CandidateResource{Error: err.Error()} + apiResults = append(apiResults, er) continue } - apiResults = append(apiResults, api.SchedSuccItem{Candidate: target}) + apiResults = append(apiResults, target) } - return apiResults + ret.Candidates = apiResults + return ret } -func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string) (*api.SchedBackupResultItem, error) { +func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, preferBackupHost string) (*schedapi.CandidateResource, error) { masterHost := selectMasterHost(result.Data, preferMasterHost, preferBackupHost) if masterHost == nil { return nil, fmt.Errorf("Can't find master host") @@ -49,10 +52,8 @@ func getSchedBackupResult(result *core.SchedResultItemList, preferMasterHost, pr markHostUsed(backupHost) sort.Sort(sort.Reverse(result)) - ret := &api.SchedBackupResultItem{ - MasterID: masterHost.ID, - SlaveID: backupHost.ID, - } + ret := masterHost.ToCandidateResource() + ret.BackupCandidate = backupHost.ToCandidateResource() return ret, nil } diff --git a/pkg/scheduler/handler/forecast_helper.go b/pkg/scheduler/handler/forecast_helper.go index 18556bb1b1..42ceee3ee7 100644 --- a/pkg/scheduler/handler/forecast_helper.go +++ b/pkg/scheduler/handler/forecast_helper.go @@ -9,7 +9,7 @@ import ( func transToSchedForecastResult(result *core.SchedResultItemList) interface{} { unit := result.Unit - reqCount := unit.SchedData().Count + reqCount := int64(unit.SchedData().Count) var readyCount int64 filters := make([]*api.ForecastFilter, 0) diff --git a/pkg/scheduler/handler/handler.go b/pkg/scheduler/handler/handler.go index 4c6e8b6394..83e14be5be 100644 --- a/pkg/scheduler/handler/handler.go +++ b/pkg/scheduler/handler/handler.go @@ -11,12 +11,12 @@ import ( "yunion.io/x/log" + schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" + computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" "yunion.io/x/onecloud/pkg/scheduler/db/models" schedman "yunion.io/x/onecloud/pkg/scheduler/manager" - - computemodels "yunion.io/x/onecloud/pkg/compute/models" ) // InstallHandler is an interface that registes route and @@ -93,12 +93,7 @@ func doSchedulerTest(c *gin.Context) { return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, true) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -128,12 +123,7 @@ func doSchedulerForecast(c *gin.Context) { return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, true) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -268,12 +258,7 @@ func doSyncSchedule(c *gin.Context) { c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) return } - sjson, err := simplejson.NewFromReader(c.Request.Body) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - schedInfo, err := api.NewSchedInfo(sjson, false) + schedInfo, err := api.FetchSchedInfo(c.Request) if err != nil { c.AbortWithError(http.StatusBadRequest, err) return @@ -285,10 +270,10 @@ func doSyncSchedule(c *gin.Context) { return } - count := schedInfo.Data.Count + count := int64(schedInfo.Count) var resp interface{} - if schedInfo.Data.Backup { - resp = transToBackupSchedResult(result, schedInfo.Data.HostID, schedInfo.Data.BackupHostID, count) + if schedInfo.Backup { + resp = transToBackupSchedResult(result, schedInfo.HostId, schedInfo.PreferBackupHost, count) } else { resp = transToRegionSchedResult(result.Data, count) } @@ -296,21 +281,15 @@ func doSyncSchedule(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func transToRegionSchedResult(result []*core.SchedResultItem, count int64) interface{} { - apiResults := make([]api.SchedResultItem, 0) +func transToRegionSchedResult(result []*core.SchedResultItem, count int64) *schedapi.ScheduleOutput { + apiResults := make([]*schedapi.CandidateResource, 0) succCount := 0 for _, nr := range result { for { if nr.Count <= 0 { break } - tr := api.SchedSuccItem{ - Candidate: api.SchedNormalResultItem{ - ID: nr.ID, - Name: nr.Name, - Data: nr.Data, - }, - } + tr := nr.ToCandidateResource() apiResults = append(apiResults, tr) nr.Count-- succCount++ @@ -321,12 +300,14 @@ func transToRegionSchedResult(result []*core.SchedResultItem, count int64) inter if int64(succCount) >= count { break } - er := api.SchedErrItem{Error: "Out of resource"} + er := &schedapi.CandidateResource{Error: "Out of resource"} apiResults = append(apiResults, er) succCount++ } - return regionResponse(apiResults) + return &schedapi.ScheduleOutput{ + Candidates: apiResults, + } } func regionResponse(v interface{}) interface{} { diff --git a/pkg/scheduler/manager/manager.go b/pkg/scheduler/manager/manager.go index 0b43409cf3..4ba896812a 100644 --- a/pkg/scheduler/manager/manager.go +++ b/pkg/scheduler/manager/manager.go @@ -113,8 +113,8 @@ func NewSessionID() string { // Schedule process the request data that is scheduled for dispatch and complements // the session information. func Schedule(info *api.SchedInfo) (*core.SchedResultItemList, error) { - if len(info.SessionID) == 0 { - info.SessionID = NewSessionID() + if len(info.SessionId) == 0 { + info.SessionId = NewSessionID() } return schedManager.schedule(info) } @@ -364,18 +364,19 @@ func newHistoryItem(historyItem *HistoryItem) *api.HistoryItem { forGuests := []string{} countDict := make(map[string]int64) - data := schedInfo.Data - tenants = append(tenants, data.OwnerTenantID) + data := schedInfo + tenants = append(tenants, data.Project) for _, forGuest := range data.ForGuests { - forGuests = append(forGuests, fmt.Sprintf("%v(%v)", forGuest.ID, forGuest.Name)) + //forGuests = append(forGuests, fmt.Sprintf("%v(%v)", forGuest.ID, forGuest.Name)) + forGuests = append(forGuests, fmt.Sprintf("%v", forGuest)) } guestType := data.Hypervisor if c, ok := countDict[guestType]; !ok { - countDict[guestType] = data.Count + countDict[guestType] = int64(data.Count) } else { - countDict[guestType] = c + data.Count + countDict[guestType] = c + int64(data.Count) } counts := []string{} @@ -412,7 +413,7 @@ func GetHistoryDetail(historyDetailArgs *api.HistoryDetailArgs) (*api.HistoryDet schedInfo := task.SchedInfo historyTasks := []api.HistoryTask{} - data := schedInfo.Data + data := schedInfo taskExecutor := task.GetTaskExecutor(data.Tag) historyTask := api.HistoryTask{ Type: data.Hypervisor, diff --git a/pkg/scheduler/manager/scheduler.go b/pkg/scheduler/manager/scheduler.go index 35ac8903a6..a344e11f28 100644 --- a/pkg/scheduler/manager/scheduler.go +++ b/pkg/scheduler/manager/scheduler.go @@ -14,7 +14,7 @@ type CandidatesProvider interface { CandidateManager() *data_manager.CandidateManager } -func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) ([]core.Candidater, error) { +func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedInfo) ([]core.Candidater, error) { var hosts []core.Candidater var err error @@ -23,8 +23,10 @@ func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) hosts, err = candidateManager.GetCandidatesByIds(provider.CandidateType(), schedData.Candidates) } else { args := data_manager.CandidateGetArgs{ - ResType: provider.CandidateType(), - ZoneID: schedData.ZoneID, + ResType: provider.CandidateType(), + ZoneID: schedData.PreferZone, + RegionID: schedData.PreferRegion, + HostTypes: schedData.GetCandidateHostTypes(), } hosts, err = candidateManager.GetCandidates(args) } @@ -87,7 +89,7 @@ func (b *BaremetalCandidatesProvider) Candidates() ([]core.Candidater, error) { } type Scheduler interface { - SchedData() *api.SchedData + SchedData() *api.SchedInfo CandidateManager() *data_manager.CandidateManager // Schedule process @@ -123,8 +125,8 @@ func (s *BaseScheduler) CandidateManager() *data_manager.CandidateManager { return s.schedManager.CandidateManager } -func (s *BaseScheduler) SchedData() *api.SchedData { - return s.schedInfo.Data +func (s *BaseScheduler) SchedData() *api.SchedInfo { + return s.schedInfo } func (s *BaseScheduler) Unit() *core.Unit { diff --git a/pkg/scheduler/manager/task_queue.go b/pkg/scheduler/manager/task_queue.go index 6fa0e167d0..b341291b22 100644 --- a/pkg/scheduler/manager/task_queue.go +++ b/pkg/scheduler/manager/task_queue.go @@ -222,7 +222,7 @@ func (tm *TaskManager) AddTask(schedulerManager *SchedulerManager, schedInfo *ap task := NewTask(schedulerManager, schedInfo) // Split into multiple scheduling tasks by host specification type. - if schedInfo.Data.Hypervisor == api.SchedTypeBaremetal { + if schedInfo.Hypervisor == api.SchedTypeBaremetal { scheduler, err = newBaremetalScheduler(schedulerManager, schedInfo) } else { scheduler, err = newGuestScheduler(schedulerManager, schedInfo) @@ -282,7 +282,7 @@ func (t *Task) GetTaskExecutor(tag string) *TaskExecutor { } func (t *Task) GetSessionID() string { - return t.SchedInfo.SessionID + return t.SchedInfo.SessionId } func (t *Task) GetStatus() string { @@ -338,7 +338,7 @@ func (t *Task) onError() { taskExecutor.Kill() } - log.Errorf("Remove Session on error: %v\n", t.SchedInfo.SessionID) + log.Errorf("Remove Session on error: %v\n", t.SchedInfo.SessionId) //t.manager.ReservedPoolManager.RemoveSession(t.SchedInfo.SessionID) close(t.waitCh) diff --git a/pkg/scheduler/options/doc.go b/pkg/scheduler/options/doc.go new file mode 100644 index 0000000000..583155b3fc --- /dev/null +++ b/pkg/scheduler/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/scheduler/options" diff --git a/pkg/scheduler/options/options.go b/pkg/scheduler/options/options.go index e8b335a274..ef57948781 100644 --- a/pkg/scheduler/options/options.go +++ b/pkg/scheduler/options/options.go @@ -3,7 +3,7 @@ package options import ( "os" - "yunion.io/x/onecloud/pkg/cloudcommon" + common_optoins "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/compute/options" ) @@ -91,6 +91,6 @@ func GetOptions() *SchedulerOptions { } func Init() { - cloudcommon.ParseOptions(&opt, os.Args, "region.conf", "scheduler") + common_optoins.ParseOptions(&opt, os.Args, "region.conf", "scheduler") options.Options = opt.ComputeOptions } diff --git a/pkg/scheduler/service/doc.go b/pkg/scheduler/service/doc.go new file mode 100644 index 0000000000..9893e063d1 --- /dev/null +++ b/pkg/scheduler/service/doc.go @@ -0,0 +1 @@ +package service // import "yunion.io/x/onecloud/pkg/scheduler/service" diff --git a/pkg/scheduler/service/service.go b/pkg/scheduler/service/service.go index 6e744e0ae6..75d06c2bc2 100644 --- a/pkg/scheduler/service/service.go +++ b/pkg/scheduler/service/service.go @@ -13,6 +13,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" "yunion.io/x/onecloud/pkg/cloudcommon/db" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/db/models" @@ -57,7 +58,7 @@ func StartService() error { } commonOpts := &opts.CommonOptions - cloudcommon.InitAuth(commonOpts, func() { + app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") startSched() })