diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go new file mode 100644 index 0000000000..4e36e11d17 --- /dev/null +++ b/cmd/scheduler/app/server.go @@ -0,0 +1,100 @@ +// Package app implements a Server object for running the scheduler. +package app + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "strconv" + + "gopkg.in/gin-gonic/gin.v1" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/mcclient/auth" + _ "github.com/yunionio/onecloud/pkg/scheduler/algorithmprovider" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + schedhandler "github.com/yunionio/onecloud/pkg/scheduler/handler" + schedman "github.com/yunionio/onecloud/pkg/scheduler/manager" + "github.com/yunionio/pkg/util/gin/middleware" + "github.com/yunionio/pkg/util/prometheus" + "github.com/yunionio/pkg/utils" +) + +type SchedulerServer struct { + Address string + Port int32 + SQLConn string + DBDialect string + AuthInfo *auth.AuthInfo +} + +func NewServerFromConfig() *SchedulerServer { + authURL := o.GetOptions().AuthURL + adminUser := o.GetOptions().AdminUser + adminPasswd := o.GetOptions().AdminPasswd + adminTenant := o.GetOptions().AdminTenant + a := auth.NewV2AuthInfo(authURL, adminUser, adminPasswd, adminTenant) + + s := &SchedulerServer{} + s.Address = o.GetOptions().Address + s.Port = int32(o.GetOptions().Port) + sqlDialect, sqlConn, err := utils.TransSQLAchemyURL(o.GetOptions().SqlConnection) + if err != nil { + log.Fatalf("Backup python sql_connection config err: %v", err) + } + s.SQLConn = sqlConn + s.DBDialect = sqlDialect + s.AuthInfo = a + + return s +} + +func Run(s *SchedulerServer) error { + startSched := func() { + err := models.Init(s.DBDialect, s.SQLConn) + if err != nil { + log.Fatalf("DB init error: %v, dialect: %s, url: %s", err, s.DBDialect, s.SQLConn) + } + + stopEverything := make(chan struct{}) + schedman.InitAndStart(stopEverything) + } + + debug := o.GetOptions().LogLevel == "debug" + + auth.AsyncInit(s.AuthInfo, debug, true, startSched) + + return startHTTP(s) +} + +func startHTTP(s *SchedulerServer) error { + gin.DefaultWriter = ioutil.Discard + + router := gin.Default() + router.Use(middleware.Logger()) + router.Use(middleware.ErrorHandler) + router.Use(middleware.KeystoneTokenVerifyMiddleware()) + + prometheus.InstallHandler(router) + schedhandler.InstallHandler(router) + + server := &http.Server{ + Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), + Handler: router, + } + + log.Infof("Start server on: %s:%d", s.Address, s.Port) + return server.ListenAndServe() +} + +func Execute() error { + o.Parse() + + err := Run(NewServerFromConfig()) + if err != nil { + err = fmt.Errorf("scheduler app failed to run: %v", err) + } + return err +} diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go new file mode 100644 index 0000000000..250156018f --- /dev/null +++ b/cmd/scheduler/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "os" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/cmd/scheduler/app" +) + +func main() { + if err := app.Execute(); err != nil { + log.Errorln(err) + os.Exit(-1) + } + os.Exit(0) +} diff --git a/cmd/scheduler/options/options.go b/cmd/scheduler/options/options.go new file mode 100644 index 0000000000..498c9342a2 --- /dev/null +++ b/cmd/scheduler/options/options.go @@ -0,0 +1,159 @@ +package options + +import ( + "fmt" + "os" + + "gopkg.in/gin-gonic/gin.v1" + + "github.com/yunionio/log" + "github.com/yunionio/structarg" + "github.com/yunionio/pkg/util/version" +) + +type SchedulerOptions struct { + IgnoreNonRunningGuests bool `help:"Ignore non running guests when build host memory and cpu size" default:"false" alias:"ignore-nonrunning-guests"` + IgnoreFakeDeletedGuests bool `help:"Ignore fake deleted guests when build host memory and cpu size" default:"false"` + AlwaysCheckAllPredicates bool `help:"Excute all predicates when scheduling" default:"false"` + DisableBaremetalPredicates bool `help:"Switch to trigger baremetal related predicates" default:"false"` + SchedulerTestLimit int `help:"Scheduler test items' limitations" default:"100"` + SchedulerHistoryLimit int `help:"Scheduler history items' limitations" default:"1000"` + SchedulerHistoryCleanPeriod string `help:"Scheduler history cleanup period" default:"60s"` + + // per isolated device default reserverd resource + MemoryReservedPerIsolatedDevice int64 `help:"Per isolated device default reserverd memory size in MB" default:"8192"` // 8G + CpuReservedPerIsolatedDevice int64 `help:"Per isolated device default reserverd CPU count" default:"8"` // 8 core + StorageReservedPerIsolatedDevice int64 `help:"Per isolated device default reserverd storage size in MB" default:"102400"` // 100G + + // parallelization options + HostBuildParallelizeSize int `help:"Number of host description build parallelization" default:"14"` + PredicateParallelizeSize int `help:"Number of execute predicates parallelization" default:"14"` + PriorityParallelizeSize int `help:"Number of execute priority parallelization" default:"14"` + + // overcommit bound options + DefaultStorageOvercommitBound int `help:"Default storage overcommit bound" default:"1"` + DefaultCpuOvercommitBound int `help:"Default cpu overcommit bound" default:"8"` + DefaultMemoryOvercommitBound int `help:"Default memory overcommit bound" default:"1"` + + // expire queue options + ExpireQueueConsumptionPeriod string `help:"Expire queue consumption period" default:"3s"` + ExpireQueueConsumptionTimeout string `help:"Expire queue consumption timeout" default:"10s"` + ExpireQueueMaxLength int `help:"Expire queue max length" default:"1000"` + ExpireQueueDealLength int `help:"Expire queue deal length" default:"100"` + + // completed queue options + CompletedQueueConsumptionPeriod string `help:"Completed queue consumption period" default:"30s"` + CompletedQueueConsumptionTimeout string `help:"Completed queue consumption timeout" default:"30s"` + CompletedQueueMaxLength int `help:"Completed queue max length" default:"100"` + CompletedQueueDealLength int `help:"Completed queue deal length" default:"10"` + + // cache options + HostCandidateCacheTTL string `help:"Build host description candidate cache TTL" default:"0s"` + HostCandidateCacheReloadCount int `help:"Build host description candidate cache reload times count" default:"20"` + HostCandidateCachePeriod string `help:"Build host description candidate cache period" default:"30s"` + + BaremetalCandidateCacheTTL string `help:"Build Baremetal description candidate cache TTL" default:"0s"` + BaremetalCandidateCacheReloadCount int `help:"Build Baremetal description candidate cache reload times count" default:"20"` + BaremetalCandidateCachePeriod string `help:"Build Baremetal description candidate cache period" default:"30s"` + + NetworkCacheTTL string `help:"Build network info from database to cache TTL" default:"0s"` + NetworkCachePeriod string `help:"Build network info from database to cache TTL" default:"1m"` + + ClusterDBCacheTTL string `help:"Cluster database cache TTL" default:"0s"` + ClusterDBCachePeriod string `help:"Cluster database cache period" default:"5m"` + + BaremetalAgentDBCacheTTL string `help:"BaremetalAgent database cache TTL" default:"0s"` + BaremetalAgentDBCachePeriod string `help:"BaremetalAgent database cache period" default:"5m"` + + AggregateDBCacheTTL string `help:"Aggregate database cache TTL" default:"0s"` + AggregateDBCachePeriod string `help:"Aggregate database cache period" default:"30s"` + + AggregateHostDBCacheTTL string `help:"AggregateHost database cache TTL" default:"0s"` + AggregateHostDBCachePeriod string `help:"AggregateHost database cache period" default:"30s"` + + NetworksDBCacheTTL string `help:"Networks database cache TTL" default:"0s"` + NetworksDBCachePeriod string `help:"Networks database cache period" default:"5m"` + + NetinterfaceDBCacheTTL string `help:"Netinterfaces database cache TTL" default:"0s"` + NetinterfaceDBCachePeriod string `help:"Netinterfaces database cache period" default:"5m"` + + WireDBCacheTTL string `help:"Wire database cache TTL" default:"0s"` + WireDBCachePeriod string `help:"Wire database cache period" default:"5m"` +} + +type Options struct { + // common options + structarg.BaseOptions + Port int `help:"The port that the scheduler's http service runs on" default:"8897" alias:"scheduler-port"` + Address string `help:"The IP address to serve on (set to 0.0.0.0 for all interfaces)" default:"0.0.0.0"` + + // mysql options + SqlConnection string `help:"SQL connection string" default:"root:root@tcp(127.0.0.1:3306)/mclouds?charset=utf8&parseTime=True"` + + // log options + LogLevel string `help:"log level" default:"info" choices:"debug|info|warn|error"` + LogVerboseLevel int `help:"log verbosity level" default:"0"` + + // gin http framework mode + GinMode string `help:"gin http framework work mode" default:"debug" choices:"debug|release"` + + // cloud auth options + Region string `help:"Region name" default:"Beijing"` + AuthURL string `help:"Keystone auth URL" default:"http://10.168.26.241:35357/v2.0" alias:"auth-uri"` + AdminUser string `help:"Admin username" default:"regionadmin"` + AdminPasswd string `help:"Admin password" default:"eBVVSNaMeyzDnD8F" alias:"admin-password"` + AdminTenant string `help:"Admin tenant" default:"system" alias:"admin-tenant-name"` + + // scheduler options + SchedulerOptions +} + +var options Options + +func GetOptions() *Options { + return &options +} + +func Parse() { + parser, e := structarg.NewArgumentParser(&options, + "scheduler", + `Yunion cloud scheduler`, + `Yunion Technology @ 2018`) + if e != nil { + log.Fatalf("Error define argument parser: %v", e) + } + + e = parser.ParseArgs(os.Args[1:], false) + if e != nil { + log.Fatalf("Parse arguments error: %v", e) + } + + if len(options.Config) > 0 { + e := parser.ParseFile(options.Config) + if e != nil { + log.Fatalf("Parse configuration file: %v", e) + } + } + + if options.Help { + fmt.Println(parser.HelpString()) + os.Exit(0) + } + + if options.Version { + fmt.Printf("Yunion cloud version:\n%s", version.GetJsonString()) + os.Exit(0) + } + + // log configuration + log.SetVerboseLevel(int32(options.LogVerboseLevel)) + e = log.SetLogLevelByString(log.Logger(), options.LogLevel) + if e != nil { + log.Fatalf("Set log level %q: %v", options.LogLevel, e) + } + + log.V(10).Debugf("Parsed options: %#v", options) + + // gin http framework mode configuration + gin.SetMode(options.GinMode) +} diff --git a/pkg/scheduler/algorithm/helper.go b/pkg/scheduler/algorithm/helper.go new file mode 100644 index 0000000000..1fa9814adb --- /dev/null +++ b/pkg/scheduler/algorithm/helper.go @@ -0,0 +1,24 @@ +package algorithm + +import ( + "fmt" + + "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +func ToHostCandidate(c core.Candidater) (*candidate.HostDesc, error) { + d, ok := c.(*candidate.HostDesc) + if !ok { + return nil, fmt.Errorf("Can't convert %#v to '*candidate.HostDesc'", c) + } + return d, nil +} + +func ToBaremetalCandidate(c core.Candidater) (*candidate.BaremetalDesc, error) { + d, ok := c.(*candidate.BaremetalDesc) + if !ok { + return nil, fmt.Errorf("Can't convert %#v to '*candidate.BaremetalDesc'", c) + } + return d, nil +} diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go new file mode 100644 index 0000000000..c3590c3b30 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -0,0 +1,218 @@ +package predicates + +import ( + "fmt" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" +) + +// NOTE: Aggregate Description +// require: Must be scheduled to the specified host +// prefer: Priority to the specified host +// avoid: Try to avoid scheduling to the specified host +// exclude: Do not allow scheduling on the specified host + +// AggregatePredicate is designed to quickly filter unavailable +// hosts and improve scheduling efficiency by tabbing whether +// the host is available. +type AggregatePredicate struct { + BasePredicate + AggregateHosts hostsAggregatesMap + RequireAggregates []api.Aggregate + ExcludeAggregates []api.Aggregate + AvoidAggregates []api.Aggregate + PreferAggregates []api.Aggregate + AggregateMap map[string]api.Aggregate +} + +type hostAggregates []*models.Aggregate + +type hostsAggregatesMap map[string]hostAggregates + +func (p *AggregatePredicate) Name() string { + return "host_aggregate" +} + +func (p *AggregatePredicate) Clone() core.FitPredicate { + return &AggregatePredicate{ + AggregateMap: make(map[string]api.Aggregate, 0), + } +} + +func hostsAggregatesInfo(cs []core.Candidater) (hostsAggregatesMap, []*models.Aggregate) { + ret := make(map[string]hostAggregates, 0) + allAggs := make([]*models.Aggregate, 0) + for _, c := range cs { + hostAggs := c.GetHostAggregates() + ret[c.IndexKey()] = hostAggs + } + if len(cs) > 0 { + allAggs = cs[0].GetAggregates() + } + return ret, allAggs +} + +func (p *AggregatePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + data := u.SchedData() + + if len(data.Candidates) > 0 { + return false, nil + } + + hsMap, allAggs := hostsAggregatesInfo(cs) + p.AggregateHosts = hsMap + appendedAggIds := make(map[string]int, len(data.Aggregates)) + + for _, aggregate := range data.Aggregates { + switch aggregate.Strategy { + case api.AggregateStrategyRequire: + p.RequireAggregates = append(p.RequireAggregates, aggregate) + case api.AggregateStrategyExclude: + p.ExcludeAggregates = append(p.ExcludeAggregates, aggregate) + + case api.AggregateStrategyPrefer: + p.PreferAggregates = append(p.PreferAggregates, aggregate) + + case api.AggregateStrategyAvoid: + p.AvoidAggregates = append(p.AvoidAggregates, aggregate) + } + + p.AggregateMap[aggregate.Idx] = aggregate + appendedAggIds[aggregate.Idx] = 1 + } + + for _, aggregate := range allAggs { + _, nameOk := appendedAggIds[aggregate.Name] + _, idOk := appendedAggIds[aggregate.ID] + if !(nameOk || idOk) { + agg := api.Aggregate{Idx: aggregate.ID, Strategy: aggregate.DefaultStrategy} + switch agg.Strategy { + case api.AggregateStrategyRequire: + p.RequireAggregates = append(p.RequireAggregates, agg) + case api.AggregateStrategyExclude: + p.ExcludeAggregates = append(p.ExcludeAggregates, agg) + + case api.AggregateStrategyPrefer: + p.PreferAggregates = append(p.PreferAggregates, agg) + + case api.AggregateStrategyAvoid: + p.AvoidAggregates = append(p.AvoidAggregates, agg) + } + } + } + + u.AppendSelectPlugin(p) + return true, nil +} + +func getHostAggregateCount(inAggs []api.Aggregate, hAggs []*models.Aggregate, strategy string) (countMap map[string]int) { + countMap = make(map[string]int) + + in := func(hAgg *models.Aggregate, inAggs []api.Aggregate) bool { + for _, agg := range inAggs { + if agg.Idx == hAgg.ID || agg.Idx == hAgg.Name { + return true + } + } + return false + } + + for _, hAgg := range hAggs { + if in(hAgg, inAggs) { + countMap[fmt.Sprintf("%s:%s:%s", hAgg.ID, hAgg.Name, strategy)]++ + } + } + return +} + +func (p *AggregatePredicate) OnSelect(u *core.Unit, c core.Candidater) bool { + hostAggs, ok := p.AggregateHosts[c.IndexKey()] + if !ok { + return true + } + + avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid) + preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer) + + setScore := func(aggCountMap map[string]int, postiveScore bool) { + stepScore := core.PriorityStep + if !postiveScore { + stepScore = -stepScore + } + for n, count := range aggCountMap { + u.IncreaseScore(c.IndexKey(), n, count*stepScore) + } + } + + setScore(avoidCountMap, false) + setScore(preferCountMap, true) + + return true +} + +func (p *AggregatePredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {} + +func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := NewPredicateHelper(p, u, c) + + if errMsg := p.exec(h); len(errMsg) > 0 { + h.Exclude(errMsg) + } + + return h.GetResult() +} + +func (p *AggregatePredicate) exec(h *PredicateHelper) string { + ahs := p.AggregateHosts + candidateID := h.Candidate.IndexKey() + + log.V(10).Debugf(">>>> ExcludeAggregates: %#v, RequireAggregates: %#v, AvoidAggregates: %#v, PreferAggregates: %#v, candidateID: %v", p.ExcludeAggregates, p.RequireAggregates, p.AvoidAggregates, p.PreferAggregates, candidateID) + + if len(p.ExcludeAggregates) > 0 { + inExclude := func(a *models.Aggregate) bool { + for _, agg := range p.ExcludeAggregates { + if agg.Idx == a.ID || agg.Idx == a.Name { + return true + } + } + + return false + } + + if ah, ok := ahs[candidateID]; ok { + for _, a := range ah { + if inExclude(a) { + return fmt.Sprintf("exclude by aggregate: '%s:%s'", a.Name, a.ID) + } + } + } + } + + if len(p.RequireAggregates) > 0 { + var as []*models.Aggregate = nil + if ah, ok := ahs[candidateID]; ok { + as = ah + } + + inRequire := func(agg api.Aggregate) bool { + for _, a := range as { + if a.ID == agg.Idx || a.Name == agg.Idx { + return true + } + } + + return false + } + + for _, agg := range p.RequireAggregates { + if !inRequire(agg) { + return fmt.Sprintf("need aggregate: '%s'", agg.Idx) + } + } + } + + return "" +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/base.go b/pkg/scheduler/algorithm/predicates/baremetal/base.go new file mode 100644 index 0000000000..11a8fbee64 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/base.go @@ -0,0 +1,18 @@ +package baremetal + +import ( + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type BasePredicate struct { + predicates.BasePredicate +} + +func (p *BasePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + if o.GetOptions().DisableBaremetalPredicates { + return false, nil + } + return true, nil +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go new file mode 100644 index 0000000000..cfa381efe1 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/cpu_predicate.go @@ -0,0 +1,39 @@ +package baremetal + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type CPUPredicate struct { + BasePredicate +} + +func (p *CPUPredicate) Name() string { + return "baremetal_cpu" +} + +func (p *CPUPredicate) Clone() core.FitPredicate { + return &CPUPredicate{} +} + +func (p *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + d := u.SchedData() + + freeCPUCount := h.GetInt64("FreeCPUCount", 0) + reqCPUCount := d.VCPUCount + if freeCPUCount < d.VCPUCount { + totalCPUCount := h.GetInt64("CPUCount", 0) + h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) + h.SetCapacity(0) + } else { + if reqCPUCount/freeCPUCount != 1 { + h.Exclude2("cpu", freeCPUCount, reqCPUCount) + } else { + h.SetCapacity(1) + } + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go new file mode 100644 index 0000000000..39868fd498 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/memory_predicate.go @@ -0,0 +1,39 @@ +package baremetal + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type MemoryPredicate struct { + BasePredicate +} + +func (p *MemoryPredicate) Name() string { + return "baremetal_memory" +} + +func (p *MemoryPredicate) Clone() core.FitPredicate { + return &MemoryPredicate{} +} + +func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + d := u.SchedData() + + freeMemSize := h.GetInt64("FreeMemSize", 0) + reqMemSize := d.VMEMSize + if freeMemSize < reqMemSize { + totalMemSize := h.GetInt64("MemSize", 0) + h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) + h.SetCapacity(0) + } else { + if reqMemSize/freeMemSize != 1 { + h.Exclude2("memory", freeMemSize, reqMemSize) + } else { + h.SetCapacity(1) + } + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go new file mode 100644 index 0000000000..43d8d935c2 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go @@ -0,0 +1,150 @@ +package baremetal + +import ( + "fmt" + "strings" + "sync" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/utils" +) + +type NetworkPredicate struct { + BasePredicate + SelectedNetworks sync.Map +} + +func (p *NetworkPredicate) Name() string { + return "baremetal_network" +} + +func (p *NetworkPredicate) Clone() core.FitPredicate { + return &NetworkPredicate{} +} + +func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + notIgnore, _ := p.BasePredicate.PreExecute(u, cs) + if !notIgnore { + return false, nil + } + + u.AppendSelectPlugin(p) + d := u.SchedData() + if len(d.HostID) > 0 && len(d.Networks) == 0 { + return false, nil + } + + return true, nil +} + +func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + schedData := u.SchedData() + + candidate, err := h.BaremetalCandidate() + if err != nil { + return false, nil, err + } + + counters := core.NewCounters() + + isMigrate := func() bool { + return len(schedData.HostID) > 0 + } + + isRandomNetworkAvailable := func(private bool, exit bool, wire string) string { + var errMsgs []string + for _, network := range candidate.Networks { + appendError := func(errMsg string) { + errMsgs = append(errMsgs, fmt.Sprintf("%s: %s", network.ID, errMsg)) + } + if !((network.Ports > 0 || isMigrate()) && network.IsExit == exit) { + appendError(predicates.ErrNoPorts) + } + if wire != "" && !utils.HasPrefix(wire, network.Wire) && !utils.HasPrefix(wire, network.WireID) { // re + appendError(predicates.ErrWireIsNotMatch) + } + if (!private && network.IsPublic) || (private && !network.IsPublic && network.TenantID == schedData.OwnerTenantID) { + // TODO: support reservedNetworks + reservedNetworks := 0 + restPort := int64(network.Ports - reservedNetworks) + if restPort == 0 { + appendError("not enough network port") + continue + } + counter := u.CounterManager.GetOrCreate("net:"+network.ID, func() core.Counter { + return core.NewNormalCounter(restPort) + }) + + u.SharedResourceManager.Add(network.ID, counter) + counters.Add(counter) + p.SelectedNetworks.Store(network.ID, counter.GetCount()) + return "" + } else { + appendError(predicates.ErrNotOwner) + } + } + + return strings.Join(errMsgs, ";") + } + + filterByRandomNetwork := func() { + if err_msg := isRandomNetworkAvailable(false, false, ""); err_msg != "" { + h.AppendPredicateFailMsg(err_msg) + } + h.SetCapacityCounter(counters) + } + + isNetworkAvaliable := func(network *api.Network) string { + if network.Idx == "" { + return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire) + } + for _, net := range candidate.Networks { + if (network.Idx == net.ID || network.Idx == net.Name) && (net.IsPublic || net.TenantID == schedData.OwnerTenantID) && (net.Ports > 0 || isMigrate()) { + h.SetCapacity(1) + return "" + } + } + + return predicates.ErrUnknown + } + + filterBySpecifiedNetworks := func() { + var errMsgs []string + + for _, network := range schedData.Networks { + if err_msg := isNetworkAvaliable(network); err_msg != "" { + errMsgs = append(errMsgs, err_msg) + } + } + + if len(errMsgs) > 0 { + h.AppendPredicateFailMsg(strings.Join(errMsgs, ", ")) + } + } + + loadUnknownNetworks := func() bool { + return true // TODO: ??? + } + + loadUnknownNetworks() + + // Randomly assign networks if no network is specified. + if len(schedData.Networks) == 0 { + filterByRandomNetwork() + } else { + filterBySpecifiedNetworks() + } + + return h.GetResult() +} + +func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { + u.SetFiltedData(c.IndexKey(), "networks", p.SelectedNetworks) + return true +} + +func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/status_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/status_predicate.go new file mode 100644 index 0000000000..a84f96d63d --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/status_predicate.go @@ -0,0 +1,51 @@ +package baremetal + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/util/sets" +) + +var ( + ExpectedStatus = sets.NewString("running", "start_convert") +) + +type StatusPredicate struct { + BasePredicate +} + +func (p *StatusPredicate) Name() string { + return "baremetal_status" +} + +func (p *StatusPredicate) Clone() core.FitPredicate { + return &StatusPredicate{} +} + +func (p *StatusPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + bm, err := h.BaremetalCandidate() + if err != nil { + return false, nil, err + } + + if !ExpectedStatus.Has(bm.Status) { + h.Exclude2("status", bm.Status, ExpectedStatus) + return h.GetResult() + } + + if !bm.Enabled { + h.Exclude2("enable_status", "disable", "enable") + return h.GetResult() + } + + if bm.ServerID == "" { + h.SetCapacity(1) + } else { + h.AppendPredicateFailMsg(predicates.ErrBaremetalHasAlreadyBeenOccupied) + h.SetCapacity(0) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go new file mode 100644 index 0000000000..d1fe380f96 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go @@ -0,0 +1,46 @@ +package baremetal + +import ( + "fmt" + + //"github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/util/baremetal" +) + +type StoragePredicate struct { + BasePredicate +} + +func (p *StoragePredicate) Name() string { + return "baremetal_storage" +} + +func (p *StoragePredicate) Clone() core.FitPredicate { + return &StoragePredicate{} +} + +func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + schedData := u.SchedData() + + candidate, err := h.BaremetalCandidate() + if err != nil { + return false, nil, err + } + + layouts, err := baremetal.CalculateLayout( + schedData.BaremetalDiskConfigs, + candidate.Storages, + ) + + if err == nil && baremetal.CheckDisksAllocable(layouts, schedData.Disks) { + h.SetCapacity(int64(1)) + } else { + h.SetCapacity(int64(0)) + h.AppendPredicateFailMsg(fmt.Sprintf("%s err: %v", predicates.ErrNoEnoughStorage, err)) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/error.go b/pkg/scheduler/algorithm/predicates/error.go new file mode 100644 index 0000000000..11b8640943 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/error.go @@ -0,0 +1,79 @@ +package predicates + +import ( + "fmt" +) + +// Here are all the errors that may appear in the preselection predicates. +const ( + ErrServerTypeIsNotMatch = `server type is not match` + ErrExitIsNotMatch = `exit is not match` + ErrWireIsNotMatch = `wire is not match` + + ErrNoPorts = `no ports` + ErrNotOwner = `not owner` + ErrNoEnoughStorage = `no enough storage` + ErrNoAvailableNetwork = `no available network on this host` + ErrNoEnoughAvailableGPUs = `no enough available GPUs` + ErrNotSupportNest = `nested function not supported` + + ErrRequireMvs = `require mvs` + ErrRequireNoMvs = `require not mvs` + ErrHostIsSpecifiedForMigration = `host_id specified for migration` + ErrMoreThanOneSizeUnspecificSplit = `more than 1 size unspecific split` + ErrNoMoreSpaceForUnspecificSplit = `no more space for an unspecific split` + ErrSubtotalOfSplitExceedsDiskSize = `subtotal of split exceeds disk size` + ErrBaremetalHasAlreadyBeenOccupied = `baremetal has already been occupied` + + ErrUnknown = `unknown error` +) + +// InsufficientResourceError is an error type that indicates what kind of resource limit is +// hit and caused the unfitting failure. +type InsufficientResourceError struct { + // resourceName is the name of the resource that is insufficient + ResourceName string + requested int64 + total int64 + free int64 +} + +func NewInsufficientResourceError(resourceName string, requested, total, free int64) *InsufficientResourceError { + return &InsufficientResourceError{ + ResourceName: resourceName, + requested: requested, + total: total, + free: free, + } +} + +func (ire *InsufficientResourceError) Error() string { + return fmt.Sprintf("no enough resource: %s, requested: %d, total: %d, free: %d", + ire.ResourceName, ire.requested, ire.total, ire.free) +} + +func (ire *InsufficientResourceError) GetReason() string { + return ire.Error() +} + +type UnexceptedResourceError struct { + message string +} + +func NewUnexceptedResourceError(message string) *UnexceptedResourceError { + return &UnexceptedResourceError{ + message: message, + } +} + +func Error(message string) *UnexceptedResourceError { + return NewUnexceptedResourceError(message) +} + +func (ure *UnexceptedResourceError) Error() string { + return ure.message +} + +func (ure *UnexceptedResourceError) GetReason() string { + return ure.Error() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go new file mode 100644 index 0000000000..613f3cf0e7 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/cpu_predicate.go @@ -0,0 +1,54 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// CPUPredicate check the current resources of the CPU is available, +// it returns the maximum available capacity. +type CPUPredicate struct { + predicates.BasePredicate +} + +func (f *CPUPredicate) Name() string { + return "host_cpu" +} + +func (f *CPUPredicate) Clone() core.FitPredicate { + return &CPUPredicate{} +} + +func (f *CPUPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + if u.IsPublicCloudProvider() { + return false, nil + } + + data := u.SchedData() + + if data.VCPUCount <= 0 { + return false, nil + } + + return true, nil +} + +func (f *CPUPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(f, u, c) + d := u.SchedData() + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + useRsvd := h.UseReserved() + freeCPUCount := hc.GetFreeCPUCount(useRsvd) + reqCPUCount := d.VCPUCount + if freeCPUCount < reqCPUCount { + totalCPUCount := hc.GetTotalCPUCount(useRsvd) + h.AppendInsufficientResourceError(reqCPUCount, totalCPUCount, freeCPUCount) + } + + h.SetCapacity(freeCPUCount / reqCPUCount) + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go new file mode 100644 index 0000000000..a07dc63f02 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go @@ -0,0 +1,103 @@ +package guest + +import ( + "fmt" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// GroupPredicate filter the packet based on the label information, +// the same group of guests should avoid schedule on same host. +type GroupPredicate struct { + predicates.BasePredicate + + ExcludeGroups []string + RequireGroups []string + AvoidGroups []string + PreferGroups []string +} + +func (p *GroupPredicate) Name() string { + return "host_group" +} + +func (p *GroupPredicate) Clone() core.FitPredicate { + return &GroupPredicate{} +} + +func (p *GroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + d := u.SchedData() + if len(d.GroupRelations) == 0 { + return false, nil + } + + for _, r := range d.GroupRelations { + if r.Strategy == "exclude" { + p.ExcludeGroups = append(p.ExcludeGroups, r.GroupID) + } else if r.Strategy == "require" { + p.RequireGroups = append(p.RequireGroups, r.GroupID) + } else if r.Strategy == "avoid" { + p.AvoidGroups = append(p.AvoidGroups, r.GroupID) + } else if r.Strategy == "prefer" { + p.PreferGroups = append(p.PreferGroups, r.GroupID) + } + } + u.AppendSelectPlugin(p) + return true, nil +} + +func (p *GroupPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { + if len(p.ExcludeGroups) > 0 { + return false + } + + if len(p.RequireGroups) > 0 { + // TODO: what? + } + + if len(p.AvoidGroups) > 0 { + u.IncreaseScore(c.IndexKey(), + p.Name()+":avoid", -core.PriorityStep*len(p.AvoidGroups), + ) + } + + if len(p.PreferGroups) > 0 { + u.IncreaseScore(c.IndexKey(), + p.Name()+":prefer", core.PriorityStep*len(p.PreferGroups), + ) + } + + return true +} + +func (p *GroupPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { +} + +func (p *GroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + + h := predicates.NewPredicateHelper(p, u, c) + + g, err := h.GetGroupCounts() + if err != nil { + return false, nil, err + } + + if len(p.ExcludeGroups) > 0 { + for _, groupId := range p.ExcludeGroups { + if g.ExistsGroup(groupId) { + h.Exclude(fmt.Sprintf("exclude by %v:exclude", groupId)) + break + } + } + } else if len(p.RequireGroups) > 0 { + for _, groupId := range p.RequireGroups { + if !g.ExistsGroup(groupId) { + h.Exclude(fmt.Sprintf("exclude by %v:require", groupId)) + break + } + } + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go b/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go new file mode 100644 index 0000000000..1b56b68fc2 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/hypervisor_predicate.go @@ -0,0 +1,52 @@ +package guest + +import ( + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +const ( + CONTAINER_ALLOWED_TAG = "container" +) + +// HypervisorPredicate is to select candidates match guest hyperviosr +// runtime +type HypervisorPredicate struct { + predicates.BasePredicate +} + +func (f *HypervisorPredicate) Name() string { + return "host_hypervisor_runtime" +} + +func (f *HypervisorPredicate) Clone() core.FitPredicate { + return &HypervisorPredicate{} +} + +func hostHasContainerTag(c core.Candidater) bool { + aggs := c.GetHostAggregates() + for _, agg := range aggs { + if agg.Name == CONTAINER_ALLOWED_TAG { + return true + } + } + return false +} + +func (f *HypervisorPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(f, u, c) + + hostType := c.Get("HostType") + guestNeedType := u.SchedData().Hypervisor + + if guestNeedType != hostType { + if guestNeedType == api.SchedTypeContainer && hostHasContainerTag(c) { + log.Debugf("Host %q has %q tag, allow it run container", c.IndexKey(), CONTAINER_ALLOWED_TAG) + return h.GetResult() + } + h.Exclude2(f.Name(), hostType, guestNeedType) + } + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go new file mode 100644 index 0000000000..ba5bcaf0cd --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go @@ -0,0 +1,112 @@ +package guest + +import ( + "fmt" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// IsolatedDevicePredicate check mode, and number of scheduled +// device configurations and current resources. +type IsolatedDevicePredicate struct { + predicates.BasePredicate +} + +func (f *IsolatedDevicePredicate) Name() string { + return "host_isolated_device" +} + +func (f *IsolatedDevicePredicate) Clone() core.FitPredicate { + return &IsolatedDevicePredicate{} +} + +func (f *IsolatedDevicePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + data := u.SchedData() + if len(data.IsolatedDevices) == 0 { + return false, nil + } + return true, nil +} + +func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(f, u, c) + reqIsoDevs := u.SchedData().IsolatedDevices + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + minCapacity := int64(0xFFFFFFFF) + + // check by specify device id + for _, dev := range reqIsoDevs { + if len(dev.ID) == 0 { + continue + } + if fDev := hc.GetIsolatedDevice(dev.ID); fDev != nil { + if len(fDev.GuestID) != 0 { + h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.ID, fDev.GuestID)) + return h.GetResult() + } + } else { + h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.ID)) + return h.GetResult() + } + minCapacity = 1 + } + + reqCount := len(reqIsoDevs) + freeCount := len(hc.UnusedIsolatedDevices()) + totalCount := len(hc.IsolatedDevices) + + // check host isolated device count + if freeCount < reqCount { + h.AppendInsufficientResourceError(int64(reqCount), int64(totalCount), int64(freeCount)) + h.Exclude(fmt.Sprintf( + "IsolatedDevice count not enough, request: %d, hostTotal: %d, hostFree: %d", + reqCount, totalCount, freeCount)) + return h.GetResult() + } + + // check host device by type + devTypeRequest := make(map[string]int, 0) + for _, dev := range reqIsoDevs { + if len(dev.Type) != 0 { + devTypeRequest[dev.Type] += 1 + } + } + for devType, reqCount := range devTypeRequest { + freeCount := len(hc.UnusedIsolatedDevicesByType(devType)) + if freeCount < reqCount { + h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount)) + return h.GetResult() + } + cap := freeCount / reqCount + if int64(cap) < minCapacity { + minCapacity = int64(cap) + } + } + + // check host device by model + devVendorModelRequest := make(map[string]int, 0) + for _, dev := range reqIsoDevs { + if len(dev.Model) != 0 { + devVendorModelRequest[fmt.Sprintf("%s:%s", dev.Vendor, dev.Model)] += 1 + } + } + for vendorModel, reqCount := range devVendorModelRequest { + freeCount := len(hc.UnusedIsolatedDevicesByVendorModel(vendorModel)) + if freeCount < reqCount { + h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: %d", vendorModel, reqCount, freeCount)) + return h.GetResult() + } + cap := freeCount / reqCount + if int64(cap) < minCapacity { + minCapacity = int64(cap) + } + } + + h.SetCapacity(minCapacity) + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go new file mode 100644 index 0000000000..8c101e090e --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/memory_predicate.go @@ -0,0 +1,55 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// MemoryPredicate filter current resources free memory capacity is meet, +// if it is satisfied to return the size of the memory that +// can carry the scheduling request. +type MemoryPredicate struct { + predicates.BasePredicate +} + +func (p *MemoryPredicate) Name() string { + return "host_memory" +} + +func (p *MemoryPredicate) Clone() core.FitPredicate { + return &MemoryPredicate{} +} + +func (p *MemoryPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + if u.IsPublicCloudProvider() { + return false, nil + } + + data := u.SchedData() + + if data.VMEMSize <= 0 { + return false, nil + } + + return true, nil +} + +func (p *MemoryPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + d := u.SchedData() + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + useRsvd := h.UseReserved() + freeMemSize := hc.GetFreeMemSize(useRsvd) + reqMemSize := d.VMEMSize + if freeMemSize < reqMemSize { + totalMemSize := hc.GetTotalMemSize(useRsvd) + h.AppendInsufficientResourceError(reqMemSize, totalMemSize, freeMemSize) + } + + h.SetCapacity(freeMemSize / reqMemSize) + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go new file mode 100644 index 0000000000..7fe3ae7360 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go @@ -0,0 +1,33 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// MigratePredicate filters whether the current candidate can be migrated. +type MigratePredicate struct { + predicates.BasePredicate +} + +func (p *MigratePredicate) Name() string { + return "host_migrate" +} + +func (p *MigratePredicate) Clone() core.FitPredicate { + return &MigratePredicate{} +} + +func (p *MigratePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + return len(u.SchedData().HostID) > 0, nil +} + +func (p *MigratePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + if u.SchedData().HostID == c.IndexKey() { + h.Exclude(predicates.ErrHostIsSpecifiedForMigration) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go new file mode 100644 index 0000000000..29d448789b --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/nest_predicate.go @@ -0,0 +1,40 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +// NestPredicate will filter whether the current host is turned on KVM, +//if the scheduling specified settings are inconsistent, then the host +// will be filtered out. +type NestPredicate struct { + predicates.BasePredicate +} + +func (p *NestPredicate) Name() string { + return "host_nest" +} + +func (p *NestPredicate) Clone() core.FitPredicate { + return &NestPredicate{} +} + +func (p *NestPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + d := u.SchedData() + + if d.Meta["kvm"] == "enabled" { + if hc.Metadata["nest"] != "enabled" { + h.Exclude(predicates.ErrNotSupportNest) + } + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go new file mode 100644 index 0000000000..6b854cc687 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -0,0 +1,207 @@ +package guest + +import ( + "fmt" + "strings" + "sync" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" + networks "github.com/yunionio/onecloud/pkg/scheduler/db/models" + "github.com/yunionio/pkg/utils" +) + +// NetworkPredicate will filter the current network information with +// the specified scheduling information to match, if not specified will +// randomly match the available network resources. +type NetworkPredicate struct { + predicates.BasePredicate + SelectedNetworks sync.Map +} + +func (p *NetworkPredicate) Name() string { + return "host_network" +} + +func (p *NetworkPredicate) Clone() core.FitPredicate { + return &NetworkPredicate{} +} + +func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + data := u.SchedData() + if len(data.HostID) > 0 && len(data.Networks) == 0 { + return false, nil + } + + return true, nil +} + +func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + d := u.SchedData() + + isMigrate := func() bool { + return len(d.HostID) > 0 + } + + // ServerType's value is 'guest' or ''(support all type) will return true. + isMatchServerType := func(network *networks.NetworkSchedResult) bool { + return network.ServerType == "guest" || network.ServerType == "" + } + + counterOfNetwork := func(u *core.Unit, n *networks.NetworkSchedResult, r int) core.Counter { + counter := u.CounterManager.GetOrCreate("net:"+n.ID, func() core.Counter { + return core.NewNormalCounter(int64(n.Ports - r)) + }) + + u.SharedResourceManager.Add(n.ID, counter) + return counter + } + + isRandomNetworkAvailable := func(private bool, exit bool, wire string, + counters core.MultiCounter) string { + + var fullErrMsgs []string + found := false + + for _, n := range hc.Networks { + errMsgs := []string{} + appendError := func(errMsg string) { + errMsgs = append(errMsgs, errMsg) + } + + if !isMatchServerType(n) { + appendError(predicates.ErrServerTypeIsNotMatch) + } + + if n.IsExit != exit { + appendError(predicates.ErrExitIsNotMatch) + } + + if !(n.Ports > 0 || isMigrate()) { + appendError(predicates.ErrNoPorts) + } + + if wire != "" && !utils.HasPrefix(wire, n.Wire) && !utils.HasPrefix(wire, n.WireID) { // re + appendError(predicates.ErrWireIsNotMatch) + } + + if !((!private && n.IsPublic) || (private && !n.IsPublic && n.TenantID == d.OwnerTenantID)) { + appendError(predicates.ErrNotOwner) + } + + if len(errMsgs) == 0 { + // add resource + reservedNetworks := 0 + counter := counterOfNetwork(u, n, reservedNetworks) + p.SelectedNetworks.Store(n.ID, counter.GetCount()) + counters.Add(counter) + found = true + + if counters.GetCount() >= d.Count { + break + } + } else { + fullErrMsgs = append(fullErrMsgs, + fmt.Sprintf("%s: %s", n.ID, strings.Join(errMsgs, ",")), + ) + } + } + + if !found { + return strings.Join(fullErrMsgs, "; ") + } + + return "" + } + + filterByRandomNetwork := func() { + counters := core.NewCounters() + if err_msg := isRandomNetworkAvailable(false, false, "", counters); err_msg != "" { + h.AppendPredicateFailMsg(err_msg) + } + h.SetCapacityCounter(counters) + } + + isNetworkAvaliable := func(n *api.Network, counters *core.MinCounters, + networks []*networks.NetworkSchedResult) string { + if n.Idx == "" { + counters0 := core.NewCounters() + ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0) + counters.Add(counters0) + return ret_msg + } + if len(hc.Networks) == 0 { + return predicates.ErrNoAvailableNetwork + } + + errMsgs := make([]string, 0) + + for _, net := range hc.Networks { + if !isMatchServerType(net) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID)) + continue + } + if !(n.Idx == net.ID || n.Idx == net.Name) { + //errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.ID)) + } else if !(net.IsPublic || net.TenantID == d.OwnerTenantID) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.ID, net.TenantID, d.OwnerTenantID)) + } else if !(net.Ports > 0 || isMigrate()) { + errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.ID)) + } else { + // add resource + reservedNetworks := 0 + counter := counterOfNetwork(u, net, reservedNetworks) + p.SelectedNetworks.Store(net.ID, counter.GetCount()) + counters.Add(counter) + return "" + } + } + + if len(errMsgs) == 0 { + return predicates.ErrUnknown + } + + return strings.Join(errMsgs, "; ") + } + + filterBySpecifiedNetworks := func() { + counters := core.NewMinCounters() + var errMsgs []string + + for _, n := range d.Networks { + if err_msg := isNetworkAvaliable(n, counters, hc.Networks); err_msg != "" { + errMsgs = append(errMsgs, err_msg) + } + } + + if len(errMsgs) > 0 { + h.AppendPredicateFailMsg(strings.Join(errMsgs, ", ")) + } else { + h.SetCapacityCounter(counters) + } + } + + if len(d.Networks) == 0 { + filterByRandomNetwork() + } else { + filterBySpecifiedNetworks() + } + + return h.GetResult() +} + +func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { + u.SetFiltedData(c.IndexKey(), "networks", p.SelectedNetworks) + return true +} + +func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { +} diff --git a/pkg/scheduler/algorithm/predicates/guest/status_predicate.go b/pkg/scheduler/algorithm/predicates/guest/status_predicate.go new file mode 100644 index 0000000000..c3f70e2e7a --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/status_predicate.go @@ -0,0 +1,48 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +const ( + ExpectedStatus = "running" + ExpectedHostStatus = "online" + ExpectedEnableStatus = "enable" +) + +// StatusPredicate is to filter the current state of host is available, +// not available host's capacity will be set to 0 and filtered out. +type StatusPredicate struct { + predicates.BasePredicate +} + +func (p *StatusPredicate) Name() string { + return "host_status" +} + +func (p *StatusPredicate) Clone() core.FitPredicate { + return &StatusPredicate{} +} + +func (p *StatusPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + curStatus := h.Get("Status").(string) + curHostStatus := h.Get("HostStatus").(string) + curEnableStatus := h.Get("EnableStatus").(string) + + if curStatus != ExpectedStatus { + h.Exclude2("status", curStatus, ExpectedStatus) + } + + if curHostStatus != ExpectedHostStatus { + h.Exclude2("host_status", curHostStatus, ExpectedHostStatus) + } + + if curEnableStatus != ExpectedEnableStatus { + h.Exclude2("enable_status", curEnableStatus, ExpectedEnableStatus) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go new file mode 100644 index 0000000000..c5f5884c89 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go @@ -0,0 +1,134 @@ +package guest + +import ( + "fmt" + "strings" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/utils" +) + +// StoragePredicate used to filter whether the storage capacity of the +// current candidate matches the type of the disk. If not matched, the +// storage capacity will be set to 0. +type StoragePredicate struct { + predicates.BasePredicate +} + +func (p *StoragePredicate) Name() string { + return "host_storage" +} + +func (p *StoragePredicate) Clone() core.FitPredicate { + return &StoragePredicate{} +} + +func (p *StoragePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + if u.IsPublicCloudProvider() { + return false, nil + } + return true, nil +} + +func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := predicates.NewPredicateHelper(p, u, c) + + hc, err := h.HostCandidate() + if err != nil { + return false, nil, err + } + + d := u.SchedData() + + isMigrate := func() bool { + return len(d.HostID) > 0 + } + + isLocalhostBackend := func(backend string) bool { + return utils.IsLocalStorage(backend) + } + + isStorageAccessible := func(storage string) bool { + for _, s := range hc.Storages { + if storage == s.ID || storage == s.Name { + return true + } + } + + return false + } + + getStorageCapacity := func(backend string, reqMaxSize int64, reqTotalSize int64, useRsvd bool) (int64, int64) { + totalFree := hc.GetFreeStorageSizeOfType(backend, useRsvd) + capacity := totalFree / utils.Max(reqTotalSize, 1) + + return capacity, totalFree + } + + getReqSizeStr := func(backend string) string { + ss := make([]string, 0, len(d.Disks)) + for _, disk := range d.Disks { + if disk.Backend == backend { + ss = append(ss, fmt.Sprintf("%v", disk.Size)) + } + } + + return strings.Join(ss, "+") + } + + getStorageFreeStr := func(backend string, useRsvd bool) string { + ss := []string{} + for _, s := range hc.Storages { + if s.StorageType == backend { + total := int64(float64(s.Capacity) * s.Cmtbound) + free := total - s.UsedCapacity - s.WasteCapacity + ss = append(ss, fmt.Sprintf("(%v-%v-%v=%v)", total, s.UsedCapacity, s.WasteCapacity, free)) + } + } + return strings.Join(ss, " + ") + } + + sizeRequest := make(map[string]map[string]int64, 0) + storeRequest := make(map[string]int64, 0) + for _, disk := range d.Disks { + if isMigrate() && !isLocalhostBackend(disk.Backend) { + storeRequest[*disk.Storage] = 1 + } else { + if _, ok := sizeRequest[disk.Backend]; !ok { + sizeRequest[disk.Backend] = map[string]int64{"max": -1, "total": 0} + } + max := sizeRequest[disk.Backend]["max"] + if max < disk.Size { + sizeRequest[disk.Backend]["max"] = disk.Size + } + sizeRequest[disk.Backend]["total"] += disk.Size + } + } + + for store := range storeRequest { + if !isStorageAccessible(store) { + h.Exclude(fmt.Sprintf("storage %v not accessible", store)) + return h.GetResult() + } + } + + useRsvd := h.UseReserved() + minCapacity := int64(0xFFFFFFFF) + for be, req := range sizeRequest { + capacity, totalFree := getStorageCapacity(be, req["max"], req["total"], useRsvd) + if capacity == 0 { + s := fmt.Sprintf("no enough %q storage, req=%v(%v), free=%v(%v)", + be, req["total"], getReqSizeStr(be), totalFree, getStorageFreeStr(be, useRsvd)) + h.AppendPredicateFailMsg(s) + } + minCapacity = utils.Min(minCapacity, capacity) + } + + h.SetCapacity(minCapacity) + if minCapacity <= 0 { + h.AppendPredicateFailMsg(predicates.ErrNoEnoughStorage) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go new file mode 100644 index 0000000000..5db5b40f85 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -0,0 +1,166 @@ +package predicates + +import ( + "fmt" + "strings" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/algorithm" + "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/data_manager" +) + +// BasePredicate is a default struct for all the predicates that will +// include it and implement it's Name() and PreExecute() methods. +type BasePredicate struct{} + +func (b *BasePredicate) Name() string { + return "base_predicate_should_not_be_called" +} + +func (b *BasePredicate) PreExecute(unit *core.Unit, candis []core.Candidater) (bool, error) { + return true, nil +} + +type PredicateHelper struct { + predicate core.FitPredicate + predicateFails []core.PredicateFailureReason + capacity int64 + Unit *core.Unit + Candidate core.Candidater +} + +func (h *PredicateHelper) getResult() (bool, []core.PredicateFailureReason, error) { + if len(h.predicateFails) > 0 { + return false, h.predicateFails, nil + } + + if h.capacity == 0 { + return false, []core.PredicateFailureReason{}, nil + } + + return true, nil, nil +} + +func (h *PredicateHelper) GetResult() (bool, []core.PredicateFailureReason, error) { + ok, reasons, err := h.getResult() + if !ok { + log.Warningf("[Filter Result] candidate: %q, filter: %q, is_ok: %v, reason: %q, error: %v\n", + h.Candidate.IndexKey(), h.predicate.Name(), ok, getReasonsString(reasons), err) + } + return ok, reasons, err +} + +func getReasonsString(reasons []core.PredicateFailureReason) string { + if len(reasons) == 0 { + return "" + } + + ss := make([]string, 0, len(reasons)) + for _, reason := range reasons { + ss = append(ss, reason.GetReason()) + } + return strings.Join(ss, ", ") +} + +func NewPredicateHelper(pre core.FitPredicate, unit *core.Unit, candi core.Candidater) *PredicateHelper { + h := &PredicateHelper{ + predicate: pre, + capacity: core.EmptyCapacity, + predicateFails: []core.PredicateFailureReason{}, + Unit: unit, + Candidate: candi, + } + return h +} + +func (h *PredicateHelper) GetFailedResult(err error) (bool, []core.PredicateFailureReason, error) { + return false, nil, err +} + +func (h *PredicateHelper) AppendPredicateFail(reason core.PredicateFailureReason) { + h.predicateFails = append(h.predicateFails, reason) +} + +func (h *PredicateHelper) AppendPredicateFailMsg(reason string) { + h.AppendPredicateFail(NewUnexceptedResourceError(reason)) +} + +func (h *PredicateHelper) AppendInsufficientResourceError(req, total, free int64) { + h.AppendPredicateFail( + NewInsufficientResourceError(h.Candidate.Get("Name").(string), req, total, free)) +} + +// SetCapacity returns the current resource capacity calculated by a filter. +// And 'capacity' default is -1. +func (h *PredicateHelper) SetCapacity(capacity int64) { + if capacity < 0 { + capacity = 0 + } + + h.SetCapacityCounter(core.NewNormalCounter(capacity)) +} + +func (h *PredicateHelper) SetCapacityCounter(counter core.Counter) { + capacity := counter.GetCount() + if capacity < core.EmptyCapacity { + capacity = core.EmptyCapacity + } + + h.capacity = capacity + h.Unit.SetCapacity(h.Candidate.IndexKey(), h.predicate.Name(), counter) +} + +func (h *PredicateHelper) Exclude(reason string) { + h.SetCapacity(0) + h.AppendPredicateFailMsg(reason) +} + +func (h *PredicateHelper) Exclude2(predicateName string, current, expected interface{}) { + h.Exclude(fmt.Sprintf("%s is '%v', expected '%v'", predicateName, current, expected)) +} + +func (h *PredicateHelper) Get(key string) interface{} { + return h.Candidate.Get(key) +} + +func (h *PredicateHelper) GetInt64(key string, def int64) int64 { + value := h.Get(key) + if value == nil { + return def + } + return value.(int64) +} + +func (h *PredicateHelper) GetGroupCounts() (*data_manager.GroupResAlgorithmResult, error) { + value := h.Get("Groups") + if value == nil { + return nil, nil + } + + if r, ok := value.(*data_manager.GroupResAlgorithmResult); ok { + return r, nil + } + + return nil, fmt.Errorf("type error: not *data_manager.GroupResAlgorithmResult (GetGroupCounts)") +} + +func (h *PredicateHelper) HostCandidate() (*candidate.HostDesc, error) { + return algorithm.ToHostCandidate(h.Candidate) +} + +func (h *PredicateHelper) BaremetalCandidate() (*candidate.BaremetalDesc, error) { + return algorithm.ToBaremetalCandidate(h.Candidate) +} + +// UseReserved check whether the unit can use guest reserved resource +func (h *PredicateHelper) UseReserved() bool { + usable := false + data := h.Unit.SchedData() + isoDevs := data.IsolatedDevices + if len(isoDevs) > 0 { + usable = true + } + return usable +} diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go new file mode 100644 index 0000000000..922b29efff --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_cluster.go @@ -0,0 +1,55 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type AvoidSameClusterPriority struct { + priorities.BasePriority + ClusterTbl map[string]int +} + +func (p *AvoidSameClusterPriority) Name() string { + return "guest_avoid_same_cluster" +} + +func (p *AvoidSameClusterPriority) Clone() core.Priority { + return &AvoidSameClusterPriority{ClusterTbl: make(map[string]int)} +} + +func (p *AvoidSameClusterPriority) PreExecute(u *core.Unit, cs []core.Candidater) (bool, []core.PredicateFailureReason, error) { + d := u.SchedData() + clusterTbl := make(map[string]int, 0) + ownerTenantID := d.OwnerTenantID + + for _, c := range cs { + hc, err := p.HostCandidate(c) + if err != nil { + return false, nil, err + } + + if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { + clusterId := hc.ClusterID + + if count0, ok := clusterTbl[clusterId]; ok { + clusterTbl[clusterId] = count0 + int(count) + } else { + clusterTbl[clusterId] = int(count) + } + } + } + + p.ClusterTbl = clusterTbl + return true, nil, nil +} + +func (p *AvoidSameClusterPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + h := priorities.NewPriorityHelper(p, u, c) + + if count, ok := p.ClusterTbl[c.Get("ClusterID").(string)]; ok { + h.SetScore(-20 * count) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go new file mode 100644 index 0000000000..22e2559e83 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go @@ -0,0 +1,34 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type AvoidSameHostPriority struct { + priorities.BasePriority +} + +func (p *AvoidSameHostPriority) Name() string { + return "guest_avoid_same_host" +} + +func (p *AvoidSameHostPriority) Clone() core.Priority { + return &AvoidSameHostPriority{} +} + +func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + h := priorities.NewPriorityHelper(p, u, c) + + hc, err := p.HostCandidate(c) + if err != nil { + return core.HostPriority{}, err + } + + ownerTenantID := u.SchedData().OwnerTenantID + if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { + h.SetScore(-50 * int(count)) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/priorities/guest/capacity.go b/pkg/scheduler/algorithm/priorities/guest/capacity.go new file mode 100644 index 0000000000..04f668e350 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/capacity.go @@ -0,0 +1,27 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type CapacityPriority struct { + priorities.BasePriority +} + +func (p *CapacityPriority) Name() string { + return "host_capacity" +} + +func (p *CapacityPriority) Clone() core.Priority { + return &CapacityPriority{} +} + +func (p *CapacityPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + h := priorities.NewPriorityHelper(p, u, c) + + capacity := u.GetCapacity(c.IndexKey()) + h.SetScore(50 * int(capacity)) + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/priorities/guest/creating.go b/pkg/scheduler/algorithm/priorities/guest/creating.go new file mode 100644 index 0000000000..f737653e84 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/creating.go @@ -0,0 +1,35 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type CreatingPriority struct { + priorities.BasePriority +} + +func (p *CreatingPriority) Name() string { + return "creating" +} + +func (p *CreatingPriority) Clone() core.Priority { + return &CreatingPriority{} +} + +func (p *CreatingPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + + h := priorities.NewPriorityHelper(p, u, c) + + hc, err := p.HostCandidate(c) + if err != nil { + return core.HostPriority{}, err + } + + if hc.CreatingGuestCount > 0 { + score := -int(hc.CreatingGuestCount) * 20 + h.SetScore(score) + } + + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/priorities/guest/lowload.go b/pkg/scheduler/algorithm/priorities/guest/lowload.go new file mode 100644 index 0000000000..a4a16a9ef6 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/lowload.go @@ -0,0 +1,35 @@ +package guest + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type LowLoadPriority struct { + priorities.BasePriority +} + +func (p *LowLoadPriority) Name() string { + return "host_lowload" +} + +func (p *LowLoadPriority) Clone() core.Priority { + return &LowLoadPriority{} +} + +func (p *LowLoadPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + h := priorities.NewPriorityHelper(p, u, c) + + hc, err := p.HostCandidate(c) + if err != nil { + return core.HostPriority{}, err + } + + cpuCommitRate := float64(hc.RunningCPUCount) / float64(hc.TotalCPUCount) + memCommitRate := float64(hc.RunningMemSize) / float64(hc.TotalMemSize) + if cpuCommitRate < 0.5 && memCommitRate < 0.5 { + score := 20 * (1 - cpuCommitRate - memCommitRate) + h.SetScore(int(score)) + } + return h.GetResult() +} diff --git a/pkg/scheduler/algorithm/priorities/priorities.go b/pkg/scheduler/algorithm/priorities/priorities.go new file mode 100644 index 0000000000..1af551b6b4 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/priorities.go @@ -0,0 +1,73 @@ +package priorities + +import ( + "math" + + "github.com/yunionio/onecloud/pkg/scheduler/algorithm" + "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +func aggPriority(x float64) float64 { + return math.Log(x + math.Sqrt(x*x+1)) +} + +// PriorityHelper is a struct that as a base interface for all priorities. +type PriorityHelper struct { + priority core.Priority + unit *core.Unit + Candidate core.Candidater + score int + err error +} + +func NewPriorityHelper(p core.Priority, u *core.Unit, c core.Candidater) *PriorityHelper { + return &PriorityHelper{ + priority: p, + unit: u, + Candidate: c, + } +} + +func (h *PriorityHelper) SetScore(score int) { + h.score = score + h.unit.SetScore(h.Candidate.IndexKey(), h.priority.Name(), score) +} + +func (h *PriorityHelper) SetError(err error) { + h.err = err +} + +func (h *PriorityHelper) GetResult() (core.HostPriority, error) { + return core.HostPriority{ + Host: h.Candidate.IndexKey(), + Score: h.score, + Candidate: h.Candidate, + }, h.err +} + +// BasePriority is a default struct for priority that all the priorities +// will contain it and implement its PreExecute(),Map(),Reduce() and +// Name() methods. +type BasePriority struct { +} + +func (b *BasePriority) PreExecute(u *core.Unit, cs []core.Candidater) (bool, []core.PredicateFailureReason, error) { + return true, nil, nil +} + +func (b *BasePriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + return core.HostPriority{}, nil +} + +func (b *BasePriority) Reduce(u *core.Unit, cs []core.Candidater, result core.HostPriorityList) error { + return nil +} + +func (b *BasePriority) Name() string { + return "base_priorites_should_not_be_called" +} + +func (b *BasePriority) HostCandidate(c core.Candidater) (*candidate.HostDesc, error) { + return algorithm.ToHostCandidate(c) +} diff --git a/pkg/scheduler/algorithmprovider/baremetal.go b/pkg/scheduler/algorithmprovider/baremetal.go new file mode 100644 index 0000000000..9256b452bd --- /dev/null +++ b/pkg/scheduler/algorithmprovider/baremetal.go @@ -0,0 +1,24 @@ +package algorithmprovider + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + predicatebm "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates/baremetal" + "github.com/yunionio/onecloud/pkg/scheduler/factory" + "github.com/yunionio/pkg/util/sets" +) + +func init() { + // Register BaremetalProvider algorithm provider + factory.RegisterAlgorithmProvider(factory.BaremetalProvider, baremetalPredicates(), nil) +} + +func baremetalPredicates() sets.String { + return sets.NewString( + factory.RegisterFitPredicate("a-BaremetalStatusFilter", &predicatebm.StatusPredicate{}), + factory.RegisterFitPredicate("b-BaremetalAggregateFilter", &predicates.AggregatePredicate{}), + factory.RegisterFitPredicate("c-BaremetalCPUFilter", &predicatebm.CPUPredicate{}), + factory.RegisterFitPredicate("d-BaremetalMemoryFilter", &predicatebm.MemoryPredicate{}), + factory.RegisterFitPredicate("e-BaremetalStorageFilter", &predicatebm.StoragePredicate{}), + factory.RegisterFitPredicate("f-BaremetalNetFilter", &predicatebm.NetworkPredicate{}), + ) +} diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go new file mode 100644 index 0000000000..254ec8b3b3 --- /dev/null +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -0,0 +1,39 @@ +package algorithmprovider + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates" + predicateguest "github.com/yunionio/onecloud/pkg/scheduler/algorithm/predicates/guest" + priorityguest "github.com/yunionio/onecloud/pkg/scheduler/algorithm/priorities/guest" + "github.com/yunionio/onecloud/pkg/scheduler/factory" + "github.com/yunionio/pkg/util/sets" +) + +func init() { + factory.RegisterAlgorithmProvider(factory.DefaultProvider, defaultPredicates(), defaultPriorities()) +} + +func defaultPredicates() sets.String { + return sets.NewString( + factory.RegisterFitPredicate("a-GuestHostStatusFilter", &predicateguest.StatusPredicate{}), + factory.RegisterFitPredicate("b-GuestHypervisorFilter", &predicateguest.HypervisorPredicate{}), + factory.RegisterFitPredicate("c-GuestAggregateFilter", &predicates.AggregatePredicate{}), + factory.RegisterFitPredicate("d-GuestMigrateFilter", &predicateguest.MigratePredicate{}), + factory.RegisterFitPredicate("e-GuestNestFilter", &predicateguest.NestPredicate{}), + factory.RegisterFitPredicate("f-GuestGroupFilter", &predicateguest.GroupPredicate{}), + factory.RegisterFitPredicate("g-GuestCPUFilter", &predicateguest.CPUPredicate{}), + factory.RegisterFitPredicate("h-GuestMemoryFilter", &predicateguest.MemoryPredicate{}), + factory.RegisterFitPredicate("i-GuestStorageFilter", &predicateguest.StoragePredicate{}), + factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicateguest.NetworkPredicate{}), + factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicateguest.IsolatedDevicePredicate{}), + ) +} + +func defaultPriorities() sets.String { + return sets.NewString( + factory.RegisterPriority("guest-avoid-same-cluster", &priorityguest.AvoidSameClusterPriority{}, 1), + factory.RegisterPriority("guest-avoid-same-host", &priorityguest.AvoidSameHostPriority{}, 1), + factory.RegisterPriority("guest-lowload", &priorityguest.LowLoadPriority{}, 1), + factory.RegisterPriority("guest-creating", &priorityguest.CreatingPriority{}, 1), + factory.RegisterPriority("guest-capacity", &priorityguest.CapacityPriority{}, 1), + ) +} diff --git a/pkg/scheduler/api/candidate.go b/pkg/scheduler/api/candidate.go new file mode 100644 index 0000000000..8027ee51fe --- /dev/null +++ b/pkg/scheduler/api/candidate.go @@ -0,0 +1,142 @@ +package api + +import ( + "strconv" + + "github.com/bitly/go-simplejson" + + "github.com/yunionio/log" +) + +// CandidateListArgs is a struct just for parsing candidate +// resource list parameters. +type CandidateListArgs struct { + Type string + Zone string + Pool string + Limit int64 + Offset int64 + Avaliable bool +} + +type ResultResource struct { + Free float64 `json:"free"` + Reserved float64 `json:"reserverd"` + Total float64 `json:"total"` +} + +func NewResultResourceString(free, reserverd, total string) (*ResultResource, error) { + f, err := strconv.ParseFloat(free, 64) + r, err := strconv.ParseFloat(reserverd, 64) + t, err := strconv.ParseFloat(total, 64) + if err != nil { + return nil, err + } + return NewResultResource(f, r, t), nil +} + +func NewResultResource(f, r, t float64) *ResultResource { + return &ResultResource{ + Free: f, + Reserved: r, + Total: t, + } +} + +func NewResultResourceInt64(f, r, t int64) *ResultResource { + free := float64(f) + reserverd := float64(r) + total := float64(t) + return NewResultResource(free, reserverd, total) +} + +type CandidateListResultItem struct { + ID string `json:"id"` + Name string `json:"name"` + Cpu ResultResource `json:"cpu"` + Mem ResultResource `json:"mem"` + Storage ResultResource `json:"storage"` + Status string `json:"status"` + HostStatus string `json:"host_status"` + EnableStatus string `json:"enable_status"` + HostType string `json:"host_type"` +} + +type CandidateListResult struct { + Data []CandidateListResultItem `json:"data"` + Total int64 `json:"total"` + Limit int64 `json:"limit"` + Offset int64 `json:"offset"` +} + +const ( + DefaultCandidateListArgsLimit = 20 +) + +// NewCandidateListArgs provides a function that +// will parse candidate's list args from a json data. +func NewCandidateListArgs(sjson *simplejson.Json) (*CandidateListArgs, error) { + args := &CandidateListArgs{ + Limit: DefaultCandidateListArgsLimit, + } + if argsType, ok := sjson.CheckGet("type"); ok { + args.Type = argsType.MustString() + } else { + args.Type = "all" + } + + if zone, ok := sjson.CheckGet("zone"); ok { + args.Zone = zone.MustString() + } + + if pool, ok := sjson.CheckGet("pool"); ok { + args.Pool = pool.MustString() + } + + if limit, ok := sjson.CheckGet("limit"); ok { + limitv, err := limit.Int64() + if err != nil { + limitv, err = strconv.ParseInt(limit.MustString(), 10, 64) + if err != nil { + log.Errorln(err) + } + } + args.Limit = limitv + } + + if offset, ok := sjson.CheckGet("offset"); ok { + args.Offset = offset.MustInt64() + } + + if avaliable, ok := sjson.CheckGet("avaliable"); ok { + args.Avaliable = avaliable.MustBool() + } + + return args, nil +} + +// CandidateDetailArgs is a struct just for parsing candidate +// resource parameters. +type CandidateDetailArgs struct { + ID string + Type string +} + +type CandidateDetailResult struct { + Candidate interface{} `json:"candidate"` +} + +// NewCandidateDetailArgs provides a function that +// will parse candidate's args from a json data. +func NewCandidateDetailArgs(sjson *simplejson.Json, id string) (*CandidateDetailArgs, error) { + args := new(CandidateDetailArgs) + args.ID = id + + if argsType, ok := sjson.CheckGet("type"); ok { + args.Type = argsType.MustString() + } else { + args.Type = HostTypeHost + } + + return args, nil +} diff --git a/pkg/scheduler/api/cleanup.go b/pkg/scheduler/api/cleanup.go new file mode 100644 index 0000000000..70ef8cf435 --- /dev/null +++ b/pkg/scheduler/api/cleanup.go @@ -0,0 +1,21 @@ +package api + +import ( + "github.com/bitly/go-simplejson" +) + +type CleanupArgs struct { + ResType string +} + +type CleanupResult struct { +} + +func NewCleanupArgs(sjson *simplejson.Json) (*CleanupArgs, error) { + args := new(CleanupArgs) + if resType, ok := sjson.CheckGet("res_type"); ok { + args.ResType = resType.MustString() + } + + return args, nil +} diff --git a/pkg/scheduler/api/completed.go b/pkg/scheduler/api/completed.go new file mode 100644 index 0000000000..af0b6cb4e5 --- /dev/null +++ b/pkg/scheduler/api/completed.go @@ -0,0 +1,19 @@ +package api + +import ( + "github.com/bitly/go-simplejson" +) + +type CompletedNotifyArgs struct { + SessionID string +} + +type CompletedNotifyResult struct { +} + +func NewCompletedNotifyArgs(sjson *simplejson.Json, sessionId string) (*CompletedNotifyArgs, error) { + args := new(CompletedNotifyArgs) + args.SessionID = sessionId + + return args, nil +} diff --git a/pkg/scheduler/api/expire.go b/pkg/scheduler/api/expire.go new file mode 100644 index 0000000000..304990e557 --- /dev/null +++ b/pkg/scheduler/api/expire.go @@ -0,0 +1,9 @@ +package api + +type ExpireArgs struct { + DirtyHosts []string + DirtyBaremetals []string +} + +type ExpireResult struct { +} diff --git a/pkg/scheduler/api/history.go b/pkg/scheduler/api/history.go new file mode 100644 index 0000000000..f9646d50eb --- /dev/null +++ b/pkg/scheduler/api/history.go @@ -0,0 +1,106 @@ +package api + +import ( + "strconv" + + "github.com/bitly/go-simplejson" +) + +type HistoryArgs struct { + Offset int64 + Limit int64 + All bool +} + +type HistoryItem struct { + Time string `json:"time"` + Consuming string `json:"consuming"` + SessionID string `json:"session_id"` + Count string `json:"count"` + Tenants []string `json:"tenants"` + Status string `json:"status"` + Guests []string `json:"guests"` + IsSuggestion bool `json:"is_suggestion"` +} + +type HistoryResult struct { + Items []*HistoryItem `json:"data"` + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` +} + +func toInt64(j *simplejson.Json) int64 { + if s, err := j.String(); err == nil { + if r, err0 := strconv.Atoi(s); err0 == nil { + return int64(r) + } + } + + return j.MustInt64() +} + +func NewHistoryArgs(sjson *simplejson.Json) (*HistoryArgs, error) { + args := new(HistoryArgs) + + if offset, ok := sjson.CheckGet("offset"); ok { + args.Offset = toInt64(offset) + } + + if limit, ok := sjson.CheckGet("limit"); ok { + args.Limit = toInt64(limit) + } + + if all, ok := sjson.CheckGet("all"); ok { + args.All = all.MustBool() + } + + return args, nil +} + +type HistoryDetailArgs struct { + ID string + Raw bool + Log bool +} + +type HistoryTask struct { + Type string `json:"type"` + Status string `json:"status"` + Data *SchedData `json:"data"` + Time string `json:"time"` + Consuming string `json:"consuming"` + //Result []SchedResultItem `json:"result"` + Result interface{} `json:"result"` + Error string `json:"error"` + Logs []string `json:"logs"` +} + +type HistoryDetail struct { + Time string `json:"time"` + Consuming string `json:"consuming"` + SessionID string `json:"session_id"` + Tasks []HistoryTask `json:"tasks"` + Input string `json:"input"` + Output string `json:"output"` + Error string `json:"error"` +} + +type HistoryDetailResult struct { + Detail *HistoryDetail `json:"history"` +} + +func NewHistoryDetailArgs(sjson *simplejson.Json, id string) (*HistoryDetailArgs, error) { + args := new(HistoryDetailArgs) + args.ID = id + + if raw, ok := sjson.CheckGet("raw"); ok { + args.Raw = raw.MustBool() + } + + if log, ok := sjson.CheckGet("log"); ok { + args.Log = log.MustBool() + } + + return args, nil +} diff --git a/pkg/scheduler/api/reserve_resource.go b/pkg/scheduler/api/reserve_resource.go new file mode 100644 index 0000000000..1d22832f80 --- /dev/null +++ b/pkg/scheduler/api/reserve_resource.go @@ -0,0 +1,10 @@ +package api + +type ReservedResourcesArgs struct { + Name string + Remove string +} + +type ReservedResourcesResult struct { + Resources interface{} `json:"resources"` +} diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go new file mode 100644 index 0000000000..ab71f318dc --- /dev/null +++ b/pkg/scheduler/api/sched.go @@ -0,0 +1,824 @@ +package api + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/bitly/go-simplejson" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/pkg/utils" +) + +type Meta map[string]string + +type Disk struct { + Backend string `json:"backend"` + ImageID string `json:"image_id"` + Fs *string `json:"fs"` + Os string `json:"os"` + OSDistribution string `json:"os_distribution"` + OsVersion string `json:"os_version"` + Format string `json:"format"` + MountPoint *string `json:"mountpoint"` + Driver *string `json:"driver"` + Cache *string `json:"cache"` + ImageDiskFormat string `json:"image_disk_format"` + Size int64 `json:"size"` + Storage *string `json:"storage"` +} + +type Network struct { + Idx string `json:"idx"` + TenantId string `json:"tenant_id"` + Private bool `json:"private"` + Ports int64 `json:"ports"` + Exit bool `json:"exit"` + Wire string `json:"wire"` + Mac string `json:"mac"` + Address string `json:"address"` + Address6 string `json:"address6"` + Driver string `json:"driver"` + BwLimit int64 `json:"bw_limit"` + Vip bool `json:"vip"` + Reserved bool `json:"reserved"` +} + +type IsolatedDevice struct { + ID string `json:"id"` + Type string `json:"dev_type"` + Model string `json:"model"` + Vendor string `json:"vendor"` +} + +type ForGuest struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Aggregate struct { + Idx string `json:"idx"` + Strategy string `json:"strategy"` +} + +type GroupRelation struct { + GroupID string `json:"group_id"` + Strategy string `json:"strategy"` + Scope string `json:"scope"` +} + +type SchedData struct { + Tag string `json:"tag"` + Type string `json:"type"` + IsContainer bool `json:"is_container"` + Count int64 `json:"count"` + ZoneID string `json:"zone_id"` + PoolID string `json:"pool_id"` + HostID string `json:"host_id"` + Candidates []string `json:"candidates"` + OwnerTenantID string `json:"owner_tenant_id"` + OwnerUserID string `json:"owner_user_id"` + VMEMSize int64 `json:"vmem_size"` + VCPUCount int64 `json:"vcpu_count"` + Disks []*Disk `json:"disks"` + Name string `json:"name"` + Networks []*Network `json:"networks"` + IsolatedDevices []*IsolatedDevice `json:"isolated_devices"` + Aggregates []Aggregate `json:"aggregate_stategy"` + Meta Meta `json:"__meta__"` + ForGuests []*ForGuest `json:"for_guests"` + GuestStatus string `json:"guest_status"` + Hypervisor string `json:"hypervisor"` + + // VM + Groups []string `json:"group"` + GroupRelations []GroupRelation `json:"group_relations"` + + // baremental + BaremetalDiskConfigs []*BaremetalDiskConfig `json:"baremetal_disk_config"` +} + +func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, error) { + data := new(SchedData) + + if tag, ok := sjson.CheckGet("tag"); ok { + if str, err := tag.String(); err == nil { + data.Tag = str + } + } + + if zoneID, ok := sjson.CheckGet("prefer_zone_id"); ok { + if str, err := zoneID.String(); err == nil { + data.ZoneID = str + } + } else if zoneID, ok := sjson.CheckGet("prefer_zone"); ok { + if str, err := zoneID.String(); err == nil { + data.ZoneID = str + } + } + + if poolID, ok := sjson.CheckGet("prefer_pool_id"); ok { + if str, err := poolID.String(); err == nil { + data.PoolID = str + } + } + + candidates := make([]string, 0) + + if hostID, ok := sjson.CheckGet("prefer_host_id"); ok { + if str, err := hostID.String(); err == nil { + candidates = append(candidates, str) + } + } else if hostID, ok := sjson.CheckGet("prefer_host"); ok { + if str, err := hostID.String(); err == nil { + candidates = append(candidates, str) + } + } + + if baremetalID, ok := sjson.CheckGet("prefer_baremetal_id"); ok { + if str, err := baremetalID.String(); err == nil { + candidates = append(candidates, str) + } + } else if baremetalID, ok := sjson.CheckGet("prefer_baremetal_id"); ok { + if str, err := baremetalID.String(); err == nil { + candidates = append(candidates, str) + } + } + + data.Candidates = candidates + + err := data.reviseSchedType(sjson) + if err != nil { + return nil, err + } + + data.reviseSchedHypervisor(sjson) + + data.VMEMSize = sjson.Get("vmem_size").MustInt64() + data.OwnerTenantID = sjson.Get("owner_tenant_id").MustString() + data.OwnerUserID = sjson.Get("owner_user_id").MustString() + data.VCPUCount = sjson.Get("vcpu_count").MustInt64() + data.Name = sjson.Get("name").MustString() + data.Count = count + data.GuestStatus = sjson.Get("guest_status").MustString() + data.Meta = map[string]string{} + data.HostID = sjson.Get("host_id").MustString() + + for key, value := range sjson.Get("__meta__").MustMap() { + data.Meta[key] = fmt.Sprintf("%v", value) + } + + if err := data.fillNetworksInfo(sjson); err != nil { + return nil, err + } + + if err := data.fillIsolatedDeviceInfo(sjson); err != nil { + return nil, err + } + + if err := data.fillDisksInfo(sjson, byTest); err != nil { + return nil, err + } + + data.fillForGuests(sjson) + + if err := data.fillAggregates(sjson, byTest); err != nil { + return nil, err + } + + if err := data.fillGroupRelations(sjson); err != nil { + return nil, err + } + + if err := data.fillBaremetalDiskConfig(sjson); err != nil { + return nil, err + } + + if err := data.fillGroups(sjson); err != nil { + return nil, err + } + + data.reviseSchedData() + + return data, nil +} + +func (s *SchedData) reviseSchedHypervisor(sjson *simplejson.Json) { + var finalHypervisor string + switch hypervisor := sjson.Get("hypervisor").MustString(); hypervisor { + case "", SchedTypeKvm: + finalHypervisor = HostHypervisorForKvm + default: + finalHypervisor = hypervisor + } + s.Hypervisor = finalHypervisor +} + +func (s *SchedData) reviseSchedType(sjson *simplejson.Json) error { + var bodyType string + if reqType, ok := sjson.CheckGet("type"); ok { + bodyType = reqType.MustString() + } else { + bodyType = HostTypeHost + } + + switch bodyType { + case "", HostTypeHost, SchedTypeGuest, SchedTypeEsxi, SchedTypeKvm, SchedTypeHyperV: + s.Type = SchedTypeGuest + + case SchedTypeContainer: + s.Type = SchedTypeGuest + s.IsContainer = true + + case HostTypeBaremetal: + s.Type = SchedTypeBaremetal + + default: + return fmt.Errorf("Sched current type=%s not support", bodyType) + } + return nil +} + +func (d *SchedData) SkipDirtyMarkHost() bool { + return d.IsPublicCloudProvider() || d.IsContainer || d.Hypervisor == SchedTypeContainer +} + +func (d *SchedData) IsPublicCloudProvider() bool { + return PublicCloudProviders.Has(d.Hypervisor) +} + +func (d *SchedData) getDiskSize(backend string) int64 { + total := int64(0) + for _, disk := range d.Disks { + if disk.Backend == backend { + total += disk.Size + } + } + + return total +} + +func (d *SchedData) AllDiskBackendSize() map[string]int64 { + backendSizeMap := make(map[string]int64, len(d.Disks)) + for _, disk := range d.Disks { + newSize := disk.Size + if size, ok := backendSizeMap[disk.Backend]; ok { + newSize += size + } + + backendSizeMap[disk.Backend] = newSize + } + + return backendSizeMap +} + +func parseSize(s string, defaultUnit string, base int) (int64, error) { + if utils.IsMatch("^\\d+$", s) { + s = s + defaultUnit + } + unit := s[len(s)-1:] + size, err := strconv.ParseFloat(s[0:len(s)-1], 64) + if err != nil { + return 0, err + } + + if unit == "g" || unit == "G" { + size *= float64(base * base * base) + } else if unit == "m" || unit == "M" { + size *= float64(base * base) + } else if unit == "k" || unit == "K" { + size *= float64(base) + } + + return int64(size), nil +} + +func macUnpackHex(mac string) (string, error) { + d := strings.Split(mac, ":") + if len(d) < 6 { + d = strings.Split(mac, "-") + } + if len(d) == 6 { + return strings.Join(d, ""), nil + } + + return "", fmt.Errorf("Mac unpack hex: format error: %s", mac) +} + +func newNetworkFromDesc(desc string) (*Network, error) { + network := new(Network) + network.Driver = "virtio" + network.Idx = "" + network.Wire = "" + network.Exit = false + network.Private = false + network.Mac = "" + network.Address = "" + network.Address6 = "" + + for _, it := range strings.Split(desc, ":") { + if utils.IsMatchIP4(it) { + network.Address = it + } else if utils.IsMatchIP6(it) { + network.Address6 = it + } else if utils.IsMatchCompactMacAddr(it) { + mac, err := macUnpackHex(it) + if err != nil { + return nil, err + } + network.Mac = mac + } else if strings.HasPrefix(it, "wire=") { + network.Wire = it[len("wire="):] + } else if it == "[random_exit]" { + network.Exit = true + } else if it == "[random]" { + network.Exit = false + } else if it == "[private]" { + network.Private = true + } else if it == "[reserved]" { + network.Reserved = true + } else if it == "virtio" || it == "e1000" || it == "sriov" { + network.Driver = it + } else if utils.IsMatchSize(it) { + limit, err := parseSize(it, "M", 1000) + if err != nil { + return nil, err + } + network.BwLimit = limit / 1000 / 1000 + } else if it == "[vip]" { + network.Vip = true + } else { + network.Idx = it + } + } + + return network, nil +} + +func (d *SchedData) fillNetworksInfo(sjson *simplejson.Json) error { + networks := []*Network{} + for index := 0; ; index++ { + netIndex := fmt.Sprintf("net.%d", index) + s, ok := sjson.CheckGet(netIndex) + if !ok { + break + } + net, err := newNetworkFromDesc(s.MustString()) + if err != nil { + return err + } + networks = append(networks, net) + } + d.Networks = networks + return nil +} + +func newIsolatedDeviceFromDesc(desc string) (dev *IsolatedDevice, err error) { + dev = new(IsolatedDevice) + for _, it := range strings.Split(desc, ":") { + if utils.IsMatchUUID(it) { + dev.ID = it + } else if ValidPassthroughTypes.Has(it) { + dev.Type = it + } else if strings.HasPrefix(it, "vendor=") { + vendor := it[len("vendor="):] + vid, ok := IsolatedVendorIDMap[strings.ToUpper(vendor)] + if !ok { + vid = vendor + } + dev.Vendor = vid + } else { + dev.Model = it + } + } + if len(dev.ID) == 0 && len(dev.Model) == 0 { + dev = nil + err = fmt.Errorf("Invalid isolated device description: %s", desc) + return + } + return +} + +func (d *SchedData) fillIsolatedDeviceInfo(sjson *simplejson.Json) error { + devs := []*IsolatedDevice{} + for idx := 0; ; idx++ { + devIdx := fmt.Sprintf("isolated_device.%d", idx) + s, ok := sjson.CheckGet(devIdx) + if !ok { + break + } + dev, err := newIsolatedDeviceFromDesc(s.MustString()) + if err != nil { + return err + } + devs = append(devs, dev) + } + d.IsolatedDevices = devs + return nil +} + +func newDiskFromCmdline(dstr string) (*Disk, error) { + matchFs := func(fs string) bool { + switch fs { + case "swap", "ext2", "ext3", "ext4", "xfs", "ntfs", "fat", "hfsplus": + return true + default: + return false + } + } + + matchFormat := func(f string) bool { + switch f { + case "qcow2", "raw", "docker", "iso", "vmdk": + return true + default: + return false + } + } + + matchDriver := func(d string) bool { + switch d { + case "virtio", "ide", "scsi", "sata", "pvscsi": + return true + default: + return false + } + } + + matchCache := func(c string) bool { + switch c { + case "writeback", "none", "writethrough": + return true + default: + return false + } + } + + matchBackend := func(b string) bool { + switch b { + case "local", "baremetal", "docker": + return true + default: + return false + } + } + disk := &Disk{Backend: "local"} + for _, nd := range strings.Split(dstr, ":") { + d := nd + if utils.IsMatchSize(d) { + size, err := parseSize(d, "G", 1024) + if err != nil { + return nil, err + } + disk.Size = size / 1024 / 1024 + } + if matchFs(d) { + disk.Fs = &d + } + if matchFormat(d) { + disk.Format = d + } + if matchDriver(d) { + disk.Driver = &d + } + if matchCache(d) { + disk.Cache = &d + } + if d == "autoextend" { + disk.Size = -1 + } + if matchBackend(d) { + disk.Backend = d + } + if strings.HasPrefix(d, "/") { + disk.MountPoint = &d + } + } + return disk, nil +} + +func newDiskFromSimpleJson(sjson *simplejson.Json, byTest bool) (*Disk, error) { + disk := new(Disk) + if byTest { + return newDiskFromCmdline(sjson.MustString()) + } + disk.Backend = sjson.Get("backend").MustString() + disk.ImageID = sjson.Get("image_id").MustString() + if fs, ok := sjson.CheckGet("fs"); ok { + if str, err := fs.String(); err == nil { + disk.Fs = &str + } + } + disk.Os = sjson.Get("os").MustString() + disk.OSDistribution = sjson.Get("os_distribution").MustString() + disk.OsVersion = sjson.Get("os_version").MustString() + + disk.Format = sjson.Get("format").MustString() + if mountPoint, ok := sjson.CheckGet("mountpoint"); ok { + if str, err := mountPoint.String(); err == nil { + disk.MountPoint = &str + } + } + if driver, ok := sjson.CheckGet("driver"); ok { + if str, err := driver.String(); err == nil { + disk.Driver = &str + } + } + if cache, ok := sjson.CheckGet("cache"); ok { + if str, err := cache.String(); err == nil { + disk.Cache = &str + } + } + disk.ImageDiskFormat = sjson.Get("image_disk_format").MustString() + disk.Size = sjson.Get("size").MustInt64() + + if storage, ok := sjson.CheckGet("storage"); ok { + if str, err := storage.String(); err == nil { + disk.Storage = &str + } + } + return disk, nil +} + +func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*BaremetalDiskConfig, error) { + baremetalDiskConfig := new(BaremetalDiskConfig) + baremetalDiskConfig.Count = sjson.Get("count").MustInt64() + baremetalDiskConfig.Conf = sjson.Get("conf").MustString() + + rangeArray := make([]int64, 0) + ranges := sjson.Get("range").MustArray() + for _, size := range ranges { + rangeArray = append(rangeArray, size.(int64)) + } + baremetalDiskConfig.Range = rangeArray + + baremetalDiskConfig.Splits = sjson.Get("splits").MustString() + baremetalDiskConfig.Strip = sjson.Get("strip").MustInt64() + baremetalDiskConfig.Type = sjson.Get("type").MustString() + ada := sjson.Get("adapter").MustInt() + baremetalDiskConfig.Adapter = &ada + baremetalDiskConfig.Cachedbadbbu = sjson.Get("cachedbadbbu").MustBool() + + return baremetalDiskConfig, nil +} + +func (d *SchedData) fillDisksInfo(sjson *simplejson.Json, byTest bool) error { + disks := make([]*Disk, 0) + index := 0 + for { + diskIndex := fmt.Sprintf("disk.%d", index) + d, ok := sjson.CheckGet(diskIndex) + if !ok { + break + } + index++ + disk, err := newDiskFromSimpleJson(d, byTest) + if err != nil { + return err + } + disks = append(disks, disk) + } + if index == 0 { + return fmt.Errorf("No disk info found in json") + } + d.Disks = disks + return nil +} + +func (d *SchedData) fillBaremetalDiskConfig(sjson *simplejson.Json) error { + baremetalDiskConfigs := []*BaremetalDiskConfig{} + if d.Hypervisor != HostTypeBaremetal { + return nil + } + config, ok := sjson.CheckGet("baremetal_disk_config") + if !ok { + defaultConfs := []*BaremetalDiskConfig{&BaremetalDefaultDiskConfig} + d.BaremetalDiskConfigs = defaultConfs + log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) + return nil + } + for index := range config.MustArray() { + baremetalDiskConfig, err := newBaremetalDiskConfigFromSimpleJson(config.GetIndex(index)) + if err != nil { + return err + } + baremetalDiskConfigs = append(baremetalDiskConfigs, baremetalDiskConfig) + } + + d.BaremetalDiskConfigs = baremetalDiskConfigs + return nil +} + +func (d *SchedData) fillForGuests(sjson *simplejson.Json) { + newGuest := func(sjson *simplejson.Json) *ForGuest { + gst := new(ForGuest) + gst.ID = sjson.Get("id").MustString() + gst.Name = sjson.Get("name").MustString() + return gst + } + for i := range sjson.Get("for_guests").MustArray() { + gst := sjson.Get("for_guests").GetIndex(i) + d.ForGuests = append(d.ForGuests, newGuest(gst)) + } +} + +func NewSchedTagFromCmdline(str string) (agg Aggregate, err error) { + rs := strings.Split(str, ":") + if len(rs) == 1 || rs[1] == "" { + err = fmt.Errorf("SchedTag %q no strategy.", str) + return + } + name, strategy := rs[0], rs[1] + + err = AggregateStrategyCheck(strategy) + if err != nil { + return + } + + agg = Aggregate{name, strategy} + return +} + +func (d *SchedData) fillAggregates(sjson *simplejson.Json, byTest bool) error { + d.Aggregates = []Aggregate{} + + if !byTest { + if aggNode, ok := sjson.CheckGet("aggregate_strategy"); ok { + for name, strategy := range aggNode.MustMap() { + d.Aggregates = append(d.Aggregates, Aggregate{ + Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), + }) + } + } + } else { + index := 0 + for { + aggIndex := fmt.Sprintf("aggregate.%d", index) + a, ok := sjson.CheckGet(aggIndex) + if !ok { + break + } + agg, err := NewSchedTagFromCmdline(a.MustString()) + if err != nil { + return err + } + d.Aggregates = append(d.Aggregates, agg) + index++ + } + } + + return nil +} + +func (d *SchedData) fillGroupRelations(sjson *simplejson.Json) error { + d.GroupRelations = []GroupRelation{} + + if aggNode, ok := sjson.CheckGet("group_relations"); ok { + for index := range aggNode.MustArray() { + it := aggNode.GetIndex(index) + + r := GroupRelation{} + + if v, ok := it.CheckGet("id"); ok { + r.GroupID = v.MustString() + } else { + return fmt.Errorf("id missing in group relation") + } + + if v, ok := it.CheckGet("scope"); ok { + r.Scope = v.MustString() + } else { + r.Scope = "host" + } + + if v, ok := it.CheckGet("strategy"); ok { + r.Strategy = v.MustString() + } else { + r.Strategy = "" + } + + d.GroupRelations = append(d.GroupRelations, r) + } + } + + return nil +} + +func (d *SchedData) fillGroups(sjson *simplejson.Json) error { + d.Groups = []string{} + if groupsNode, ok := sjson.CheckGet("groups"); ok { + for index := range groupsNode.MustArray() { + it := groupsNode.GetIndex(index) + d.Groups = append(d.Groups, it.MustString()) + } + } + + return nil +} + +// TODO +func (d *SchedData) reviseSchedData() { + /*if strategy, ok := data.Meta["aggregate_strategy"]; ok { + d.Aggregates = append(d.Aggregates, Aggregate{ + Idx: fmt.Sprintf("%v", name), Strategy: fmt.Sprintf("%v", strategy), + }) + }*/ +} + +type SchedInfo struct { + Data *SchedData `json:"scheduler"` + IgnoreFilters map[string]bool `json:"ignore_filters"` + SessionID string `json:"session_id"` + IsSuggestion bool `json:"suggestion"` + ShowSuggestionDetails bool `json:"suggestion_details"` + SuggestionLimit int64 `json:"suggestion_limit"` + SuggestionAll bool `json:"suggestion_all"` + Raw string `json:"raw"` + BestEffort bool `json:"best_effort"` +} + +func NewSchedInfo(sjson *simplejson.Json, byTest bool) (*SchedInfo, error) { + info := new(SchedInfo) + + schedData, ok := sjson.CheckGet("scheduler") + if !ok { + return nil, fmt.Errorf("Not found 'scheduler' from request body") + + } + + count := sjson.Get("count").MustInt64(1) + + if data, err := NewSchedData(schedData, count, byTest); err == nil { + info.Data = data + } else { + return nil, err + } + + info.SessionID = sjson.Get("session_id").MustString() + + ignoreFiltersString := sjson.Get("ignore_filters").MustString() + filters := strings.Split(ignoreFiltersString, ",") + ignoreFilters := make(map[string]bool, len(filters)) + for _, filter := range filters { + ignoreFilters[filter] = true + + } + info.IgnoreFilters = ignoreFilters + + info.SuggestionLimit = sjson.Get("suggestion_limit").MustInt64() + if info.SuggestionLimit == 0 { + info.SuggestionLimit = int64(o.GetOptions().SchedulerTestLimit) + } + info.SuggestionAll = sjson.Get("suggestion_all").MustBool() + info.ShowSuggestionDetails = sjson.Get("suggestion_details").MustBool() + info.BestEffort = sjson.Get("best_effort").MustBool() + + if bytes, err := sjson.EncodePretty(); err == nil { + info.Raw = string(bytes) + + } + return info, nil +} + +func (i *SchedInfo) String() (string, error) { + bytes, err := json.Marshal(i) + if err != nil { + return "", err + } + return string(bytes), nil +} + +type SchedResultItem interface{} + +type SchedResult struct { + Items []SchedResultItem `json:"scheduler"` +} + +type candidateResult struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type SchedSuccItem struct { + Candidate SchedNormalResultItem `json:"candidate"` +} + +type SchedErrItem struct { + Error string `json:"error"` +} + +type SchedNormalResultItem struct { + ID string `json:"id"` + Name string `json:"name"` + Data map[string]interface{} `json:"data"` +} + +type SchedTestResult struct { + Data interface{} `json:"data"` + Total int64 `json:"total"` + Limit int64 `json:"limit"` + Offset int64 `json:"offset"` +} diff --git a/pkg/scheduler/api/sched_test.go b/pkg/scheduler/api/sched_test.go new file mode 100644 index 0000000000..61146d3016 --- /dev/null +++ b/pkg/scheduler/api/sched_test.go @@ -0,0 +1,114 @@ +package api + +import ( + "reflect" + "testing" +) + +func TestNewSchedTagFromCmdline(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + wantAgg Aggregate + wantErr bool + }{ + { + name: "test:avoid", + args: args{"test:avoid"}, + wantAgg: Aggregate{Idx: "test", Strategy: "avoid"}, + wantErr: false, + }, + { + name: "test empty string", + args: args{""}, + wantAgg: Aggregate{}, + wantErr: true, + }, + { + name: "test no Strategy string", + args: args{"no_strategy:"}, + wantAgg: Aggregate{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotAgg, err := NewSchedTagFromCmdline(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("NewSchedTagFromCmdline() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotAgg, tt.wantAgg) { + t.Errorf("NewSchedTagFromCmdline() = %v, want %v", gotAgg, tt.wantAgg) + } + }) + } +} + +func Test_newIsolatedDeviceFromDesc(t *testing.T) { + type args struct { + desc string + } + tests := []struct { + name string + args args + wantDev *IsolatedDevice + wantErr bool + }{ + { + name: "empty string should Invalid", + args: args{""}, + wantDev: nil, + wantErr: true, + }, + { + name: "parse only model", + args: args{"1050 Ti"}, + wantDev: &IsolatedDevice{Model: "1050 Ti"}, + wantErr: false, + }, + { + name: "parse uuid with model", + args: args{"1050 Ti:f5d8c180-5a76-49a5-a296-cea73c3fe5ed"}, + wantDev: &IsolatedDevice{ + ID: "f5d8c180-5a76-49a5-a296-cea73c3fe5ed", + Model: "1050 Ti", + }, + wantErr: false, + }, + { + name: "all info", + args: args{"1050 Ti:f5d8c180-5a76-49a5-a296-cea73c3fe5ed:GPU-HPC"}, + wantDev: &IsolatedDevice{ + ID: "f5d8c180-5a76-49a5-a296-cea73c3fe5ed", + Model: "1050 Ti", + Type: GPU_HPC_TYPE, + }, + wantErr: false, + }, + { + name: "wrong type", + args: args{"1050 Ti:f5d8c180-5a76-49a5-a296-cea73c3fe5ed:GPU-HPC-Wrong"}, + wantDev: &IsolatedDevice{ + ID: "f5d8c180-5a76-49a5-a296-cea73c3fe5ed", + Model: "GPU-HPC-Wrong", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDev, err := newIsolatedDeviceFromDesc(tt.args.desc) + if (err != nil) != tt.wantErr { + t.Errorf("newIsolatedDeviceFromDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotDev, tt.wantDev) { + t.Errorf("newIsolatedDeviceFromDesc() = %v, want %v", gotDev, tt.wantDev) + } + }) + } +} diff --git a/pkg/scheduler/api/types.go b/pkg/scheduler/api/types.go new file mode 100644 index 0000000000..cc7780981d --- /dev/null +++ b/pkg/scheduler/api/types.go @@ -0,0 +1,144 @@ +package api + +import ( + "fmt" + + "github.com/yunionio/pkg/util/sets" +) + +const ( + HostTypeHost = "host" + HostTypeBaremetal = "baremetal" + SchedTypeGuest = "guest" + SchedTypeBaremetal = "baremetal" + SchedTypeContainer = "container" + SchedTypeEsxi = "esxi" + SchedTypeHyperV = "hyperv" + SchedTypeKvm = "kvm" + HostHypervisorForKvm = "hypervisor" + HostTypeAliyun = "aliyun" + + AggregateStrategyRequire = "require" + AggregateStrategyExclude = "exclude" + AggregateStrategyPrefer = "prefer" + AggregateStrategyAvoid = "avoid" + + // Baremetal related const + DISK_CONF_RAID0 = "raid0" + DISK_CONF_RAID1 = "raid1" + DISK_CONF_RAID5 = "raid5" + DISK_CONF_RAID10 = "raid10" + DISK_CONF_NONE = "none" + + DEFAULT_DISK_CONF = DISK_CONF_NONE + + DISK_TYPE_ROTATE = "rotate" + DISK_TYPE_SSD = "ssd" + DISK_TYPE_HYBRID = "hybrid" + + DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + + DISK_DRIVER_MEGARAID = "MegaRaid" + DISK_DRIVER_LINUX = "Linux" + DISK_DRIVER_HPSARAID = "HPSARaid" + DISK_DRIVER_MPT2SAS = "Mpt2SAS" + DISK_DRIVER_MARVELRAID = "MarvelRaid" + DISK_DRIVER_PCIE = "PCIE" + + HDD_DISK_SPEC_TYPE = "HDD" + SSD_DISK_SPEC_TYPE = "SSD" + + // passthrough device type + DIRECT_PCI_TYPE = "PCI" + GPU_HPC_TYPE = "GPU-HPC" + GPU_VGA_TYPE = "GPU-VGA" + USB_TYPE = "USB" + NIC_TYPE = "NIC" + + // Hard code vendor const + NVIDIA = "NVIDIA" + AMD = "AMD" + NVIDIA_VENDOR_ID = "10de" + AMD_VENDOR_ID = "1002" +) + +var ( + AggregateStrategySets = sets.NewString( + AggregateStrategyRequire, + AggregateStrategyExclude, + AggregateStrategyPrefer, + AggregateStrategyAvoid, + ) + + PublicCloudProviders = sets.NewString( + HostTypeAliyun, + ) + + BaremetalDefaultDiskConfig = BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_NONE, + Count: 0, + } + + ValidGpuTypes = sets.NewString( + GPU_HPC_TYPE, + GPU_VGA_TYPE, + ) + + ValidPassthroughTypes = sets.NewString( + DIRECT_PCI_TYPE, + USB_TYPE, + NIC_TYPE, + ).Union(ValidGpuTypes) + + IsolatedVendorIDMap = map[string]string{ + NVIDIA: NVIDIA_VENDOR_ID, + AMD: AMD_VENDOR_ID, + } + + IsolatedIDVendorMap = map[string]string{} +) + +func init() { + for k, v := range IsolatedVendorIDMap { + IsolatedIDVendorMap[v] = k + } +} + +func AggregateStrategyCheck(strategy string) (err error) { + if !AggregateStrategySets.Has(strategy) { + err = fmt.Errorf("Strategy %q must in set %v", strategy, AggregateStrategySets.List()) + } + return +} + +type BaremetalStorage struct { + Slot int `json:"slot"` + Status string `json:"status"` + Rotate bool `json:"rotate"` + Adapter int `json:"adapter"` + Driver string `json:"driver"` + Model string `json:"model"` + Enclosure int `json:"enclousure"` + Size int64 `json:"size"` + MinStripSize int64 `json:"min_strip_size,omitempty"` + MaxStripSize int64 `json:"max_strip_size,omitempty"` + Index int64 `json:"index"` +} + +type BaremetalDiskConfig struct { + // disk type + Type string `json:"type"` + // raid config + Conf string `json:"conf"` + Count int64 `json:"count"` + Range []int64 `json:"range"` + Splits string `json:"splits"` + Adapter *int `json:"adapter"` + Driver string `json:"driver"` + Cachedbadbbu bool `json:"cachedbadbbu"` + Strip int64 `json:"strip"` + RA bool `json:"ra"` + WT bool `json:"wt"` + Direct bool `json:"direct"` +} diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go new file mode 100644 index 0000000000..d5a9356b41 --- /dev/null +++ b/pkg/scheduler/cache/cache.go @@ -0,0 +1,209 @@ +package cache + +import ( + "fmt" + "reflect" + "sync" + + "github.com/yunionio/log" + expirationcache "github.com/yunionio/pkg/util/cache" + "github.com/yunionio/pkg/util/wait" +) + +var ( + normalError = fmt.Errorf("%s", "no need update all") +) + +var ( + // Full update every 10 minutes(30s * 20), but The first implementation subtracts initialization + fullUpdateHostsCounter = 0 + fullUpdateBaremetalsCounter = 0 +) + +func NewCache(kind string, item CachedItem) Cache { + cache := newSchedulerCache(kind, item) + return cache +} + +type schedulerCache struct { + kind string + item CachedItem + cache expirationcache.Store + readyCh chan struct{} + cacheCandidate sync.Map +} + +func newSchedulerCache( + kind string, + item CachedItem, +) *schedulerCache { + return &schedulerCache{ + kind: kind, + item: item, + cache: expirationcache.NewTTLStore(item.Key, item.TTL()), + readyCh: make(chan struct{}), + } +} + +func (c *schedulerCache) Name() string { + return fmt.Sprintf("%s - %s", c.kind, c.item.Name()) +} + +func (c *schedulerCache) Get(key string) (interface{}, error) { + value, ok, err := c.cache.GetByKey(key) + if err != nil { + return nil, err + } + if !ok { + log.Infof("Update %s, id: %s", c.Name(), key) + objs, err := c.item.Update([]string{key}) + if err != nil { + return nil, err + } + + if len(objs) < 1 { + return nil, fmt.Errorf("object %v not found", key) + } + + obj := objs[0] + err = c.cache.Add(obj) + if err != nil { + return nil, err + } + + return obj, nil + } + + return value, nil +} + +func (c *schedulerCache) Add(obj interface{}) error { + return c.cache.Add(obj) +} + +func (c *schedulerCache) Update(obj interface{}) error { + return c.Add(obj) +} + +func (c *schedulerCache) Delete(obj interface{}) error { + return c.cache.Delete(obj) +} + +func (c *schedulerCache) List() []interface{} { + return c.cache.List() +} + +func (c *schedulerCache) Start(stop <-chan struct{}) { + f := c.updateAllObjects + p := c.item.Period() + + go wait.Until(f, p, stop) +} + +func (c *schedulerCache) Reload(keys []string) ([]interface{}, error) { + return c.loadObjects(keys) +} + +func (c *schedulerCache) ReloadAll() ([]interface{}, error) { + return c.loadObjects(nil) +} + +func (c *schedulerCache) WaitForReady() { + readyCh := c.readyCh + if readyCh != nil { + <-c.readyCh + } +} + +func (c *schedulerCache) updateAllObjects() { + defer func() { + if c.readyCh != nil { + close(c.readyCh) + c.readyCh = nil + } + }() + // Get the data you need to update. + ids, err := c.item.GetUpdate(c.List()) + // if ids is nil and err is nil,than update all. + if len(ids) == 0 && err == nil { + c.loadObjects(nil) + } else if len(ids) == 0 && reflect.DeepEqual(err, normalError) { + // if ids is nil and err is normalError then return. + return + } else if len(ids) > 0 { + log.V(10).Debugf("Update host/baremetal status list: %v", ids) + c.loadObjects(ids) + } +} + +func (c *schedulerCache) loadObjects(ids []string) ([]interface{}, error) { + log.Infof("Start load %s, period: %v, ttl: %v", c.Name(), c.item.Period(), c.item.TTL()) + + defer func() { + log.Infof("End load %s", c.Name()) + }() + + var ( + objects []interface{} + needUpdate map[string]bool + err error + ) + + if ids == nil { + needUpdate = make(map[string]bool, 0) + c.cacheCandidate.Range(func(key, _ interface{}) bool { + if key != nil && key.(string) != "" { + needUpdate[key.(string)] = true + } + + return true + }) + objects, err = c.item.Load() + } else { + needUpdate = make(map[string]bool, len(ids)) + for _, id := range ids { + if id != "" { + needUpdate[id] = true + } + } + objects, err = c.item.Update(ids) + } + if err != nil { + log.Errorf("Load %s: %v", c.Name(), err) + return nil, err + } + + log.V(4).Infof("%v objects loaded", len(objects)) + + for _, obj := range objects { + // Add the load new data into cache. + err := c.Add(obj) + if err != nil { + log.Errorf("Add %v object to %s cache: %v", obj, c.Name(), err) + continue + } + + if id, err := c.item.Key(obj); err == nil { + // If exist the id then the id is valid and we set it to false. + if _, ok := needUpdate[id]; ok { + needUpdate[id] = false + } + // Add or update new data into global cache. + c.cacheCandidate.Store(id, obj) + } + } + + // If status is true,then the host must have been deleted. + for id, status := range needUpdate { + if status { + // Load the need delete object and will delete it from chache and scheduler'cache. + object, ok := c.cacheCandidate.Load(id) + if ok { + c.cacheCandidate.Delete(id) + c.Delete(object) + } + } + } + + return objects, err +} diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go new file mode 100644 index 0000000000..c8dcd78d6b --- /dev/null +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -0,0 +1,309 @@ +package candidate + +import ( + "fmt" + "strings" + "time" + + fjson "github.com/json-iterator/go" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + "github.com/yunionio/onecloud/pkg/scheduler/cache/db" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" +) + +type BaremetalDesc struct { + baseHostDesc + + Storages []*api.BaremetalStorage `json:"storages"` + StorageType string `json:"storage_type"` + StorageSize int64 `json:"storage_size"` + StorageInfo string `json:"storage_info"` + StorageDriver string `json:"storage_driver"` + ServerID string `json:"server_id"` +} + +type BaremetalBuilder struct { + baremetalAgents cache.Cache + baremetals []interface{} + + residentTenantDict map[string]map[string]interface{} +} + +func (bd *BaremetalDesc) String() string { + s, _ := fjson.Marshal(bd) + return string(s) +} + +func (bd *BaremetalDesc) Type() int { + // Baremetal type + return 1 +} + +func (bd *BaremetalDesc) Get(key string) interface{} { + switch key { + case "ID": + return bd.ID + + case "Name": + return bd.Name + + case "Status": + return bd.Status + + case "PoolID": + return bd.PoolID + + case "ZoneID": + return bd.ZoneID + + case "ServerID": + return bd.ServerID + + case "CPUCount": + return int64(bd.CPUCount) + + case "FreeCPUCount": + return bd.FreeCPUCount() + + case "NodeCount": + return int64(bd.NodeCount) + + case "MemSize": + return bd.MemSize + + case "FreeMemSize": + return bd.FreeMemSize() + + case "Storages": + return bd.StorageType + + case "StorageSize": + return bd.StorageSize + + case "StorageType": + return bd.StorageType + + case "StorageInfo": + return bd.StorageInfo + + case "StorageDriver": + return bd.StorageDriver + + case "FreeStorageSize": + return bd.FreeStorageSize() + + case "HostStatus": + return bd.HostStatus + + default: + return nil + } +} + +func (bd *BaremetalDesc) XGet(key string, kind core.Kind) interface{} { + return core.XGetCalculator(bd, key, kind) +} + +func (bd *BaremetalDesc) IndexKey() string { + return bd.ID +} + +func (bd *BaremetalDesc) FreeCPUCount() int64 { + if bd.ServerID == "" { + return bd.CPUCount + } + return 0 +} + +func (bd *BaremetalDesc) FreeMemSize() int64 { + if bd.ServerID == "" { + return bd.MemSize + } + return 0 +} + +func (bd *BaremetalDesc) FreeStorageSize() int64 { + if bd.ServerID == "" { + return bd.StorageSize + } + return 0 +} + +func (bb *BaremetalBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGroupCacher) error { + agents, err := dbCache.Get(db.BaremetalAgentDBCache) + if err != nil { + return err + } + + bms, err := models.FetchBaremetalHostByIDs(ids) + if err != nil { + return err + } + + bb.baremetalAgents = agents + bb.baremetals = bms + + wg := &WaitGroupWrapper{} + errMessageChannel := make(chan error, 2) + defer close(errMessageChannel) + + setFuncs := []func(){} + + for _, f := range setFuncs { + wg.Wrap(f) + } + + if ok := waitTimeOut(wg, time.Duration(20*time.Second)); !ok { + log.Errorln("BaremetalBuilder waitgroup timeout.") + } + + if len(errMessageChannel) != 0 { + errMessages := make([]string, 0) + lengthChan := len(errMessageChannel) + for ; lengthChan >= 0; lengthChan-- { + errMessages = append(errMessages, fmt.Sprintf("%s", <-errMessageChannel)) + } + return fmt.Errorf("%s\n", strings.Join(errMessages, ";")) + } + + return nil +} + +func (bb *BaremetalBuilder) Clone() BuildActor { + return &BaremetalBuilder{} +} + +func (bb *BaremetalBuilder) Type() string { + return BaremetalDescBuilder +} + +func (bb *BaremetalBuilder) AllIDs() ([]string, error) { + return models.AllBaremetalIDs() +} + +func (bb *BaremetalBuilder) Do(ids []string, dbCache DBGroupCacher, syncCache SyncGroupCacher) ([]interface{}, error) { + err := bb.init(ids, dbCache, syncCache) + if err != nil { + return nil, err + } + + descs, err := bb.build() + if err != nil { + return nil, err + } + return descs, nil +} + +func (bb *BaremetalBuilder) build() ([]interface{}, error) { + schedDescs := []interface{}{} + for _, bm := range bb.baremetals { + desc, err := bb.buildOne(bm.(*models.Host)) + if err != nil { + log.Errorf("BaremetalBuilder error: %v", err) + continue + } + schedDescs = append(schedDescs, desc) + } + return schedDescs, nil +} + +func (bb *BaremetalBuilder) buildOne(bm *models.Host) (interface{}, error) { + desc := new(BaremetalDesc) + desc.ID = bm.ID + desc.Name = bm.Name + desc.UpdatedAt = bm.UpdatedAt + desc.Status = bm.Status + desc.CPUCount = bm.CPUCount + desc.NodeCount = bm.NodeCount + desc.MemSize = int64(bm.MemSize) + desc.StorageDriver = bm.StorageDriver + desc.StorageType = bm.StorageType + desc.StorageSize = int64(bm.StorageSize) + desc.StorageInfo = bm.StorageInfo + desc.PoolID = bm.PoolID + desc.ZoneID = bb.getZoneID(bm) + desc.Enabled = bm.Enabled + desc.ClusterID = bm.ClusterID + + desc.HostStatus = bm.HostStatus + desc.Enabled = bm.Enabled + desc.HostType = bm.HostType + desc.IsBaremetal = bm.IsBaremetal + + var baremetalStorages []*api.BaremetalStorage + err := fjson.Unmarshal([]byte(bm.StorageInfo), &baremetalStorages) + if err != nil { + // StorageInfo maybe is NULL + if bm.StorageInfo != "" { + log.Errorln(err) + } + } + desc.Storages = baremetalStorages + desc.Tenants = make(map[string]int64, 0) + + err = bb.fillServerID(desc, bm) + if err != nil { + return nil, err + } + + err = bb.fillResidentTenants(desc, bm) + if err != nil { + return nil, err + } + + // data from db + err = bb.fillNetworks(desc, bm) + if err != nil { + return nil, err + } + + err = desc.fillAggregates() + if err != nil { + return nil, err + } + + return desc, nil +} + +func (bb *BaremetalBuilder) fillServerID(desc *BaremetalDesc, b *models.Host) error { + guests, err := models.FetchGuestByHostIDsWithCond([]string{b.ID}, + map[string]interface{}{ + "hypervisor": "baremetal", + }) + if err != nil { + return err + } + + if len(guests) == 0 { + desc.ServerID = "" + } else if len(guests) == 1 { + desc.ServerID = guests[0].(*models.Guest).ID + } else { + return fmt.Errorf("One baremetal %q contains %d guests, %v", b.Name, len(guests), guests) + } + + return nil +} + +func (bb *BaremetalBuilder) fillNetworks(desc *BaremetalDesc, b *models.Host) error { + return desc.fillNetworks(b.ID) +} + +func (b *BaremetalBuilder) fillResidentTenants(desc *BaremetalDesc, host *models.Host) error { + rets, err := HostResidentTenantCount(host.ID) + if err != nil { + return err + } + + desc.Tenants = rets + + return nil +} + +func (b *BaremetalBuilder) getZoneID(bm *models.Host) string { + return bm.ZoneID +} diff --git a/pkg/scheduler/cache/candidate/candidate.go b/pkg/scheduler/cache/candidate/candidate.go new file mode 100644 index 0000000000..b3b43cd241 --- /dev/null +++ b/pkg/scheduler/cache/candidate/candidate.go @@ -0,0 +1,14 @@ +package candidate + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/cache" +) + +type candidateItem struct { + cache.CachedItem +} + +func NewCandidateManager(db DBGroupCacher, sync SyncGroupCacher, stopCh <-chan struct{}) *cache.GroupManager { + items := defaultCadidateItems(db, sync) + return cache.NewGroupManager(CacheKind, items, stopCh) +} diff --git a/pkg/scheduler/cache/candidate/candidate_test.go b/pkg/scheduler/cache/candidate/candidate_test.go new file mode 100644 index 0000000000..620936a486 --- /dev/null +++ b/pkg/scheduler/cache/candidate/candidate_test.go @@ -0,0 +1,72 @@ +package candidate + +import ( + "flag" + "testing" + "time" + + "github.com/yunionio/onecloud/pkg/scheduler/cache" + "github.com/yunionio/onecloud/pkg/scheduler/cache/db" + "github.com/yunionio/onecloud/pkg/scheduler/cache/sync" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" +) + +var ( + // flag to connect database + dialect = flag.String("db-dialect", "mysql", "db dialect") + dbURL = flag.String("db-url", "root:root@tcp(127.0.0.1:3306)/mclouds?charset=utf8&parseTime=True", "db url") + + // Kinds of cache manager + testDBMan *cache.GroupManager + testSyncMan *cache.GroupManager + //testCandiMan *cache.GroupManager +) + +func init() { + if err := models.Init(*dialect, *dbURL); err != nil { + panic(err) + } + + stopCh := make(chan struct{}) + testDBMan = db.NewCacheManager(stopCh) + testDBMan.Run() + testSyncMan = sync.NewSyncManager(stopCh) + testSyncMan.Run() + //testCandiMan = NewCandidateManager(testCacheMan, testSyncMan, stopCh) + //testCandiMan.Run() +} + +func TestHostBuildOne(t *testing.T) { + time.Sleep(3 * time.Second) + builder := &HostBuilder{} + err := builder.init([]string{"01ee5aca-3d63-404b-957c-fb9ea2306770"}, testDBMan, testSyncMan) + if err != nil { + t.Fatal(err) + } + descs, err := builder.buildOne(builder.hosts[0].(*models.Host)) + if err != nil { + t.Fatal(err) + } + t.Log(descs) +} + +func BenchmarkParallelizeBuild(b *testing.B) { + time.Sleep(2 * time.Second) + builder := &HostBuilder{} + ids, err := models.AllIDs(models.Hosts) + if err != nil { + b.Fatal(err) + } + err = builder.init(ids, testDBMan, testSyncMan) + if err != nil { + b.Fatal(err) + } + for n := 0; n < b.N; n++ { + for _, h := range builder.hosts { + _, err = builder.buildOne(h.(*models.Host)) + if err != nil { + b.Fatal(err) + } + } + } +} diff --git a/pkg/scheduler/cache/candidate/default.go b/pkg/scheduler/cache/candidate/default.go new file mode 100644 index 0000000000..d6a23106c6 --- /dev/null +++ b/pkg/scheduler/cache/candidate/default.go @@ -0,0 +1,160 @@ +package candidate + +import ( + "fmt" + "reflect" + gosync "sync" + "time" + + "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + u "github.com/yunionio/pkg/utils" +) + +const ( + CacheKind = "CandidateCache" + + HostCandidateCache = "Hosts" + BaremetalCandidateCache = "Baremetals" + + HostDescBuilder = HostCandidateCache + BaremetalDescBuilder = BaremetalCandidateCache +) + +func defaultCadidateItems(db DBGroupCacher, sync SyncGroupCacher) []cache.CachedItem { + return []cache.CachedItem{ + newHostCache(db, sync), + newBaremetalCache(db, sync), + } +} + +func uuidKey(obj interface{}) (string, error) { + return obj.(descer).UUID(), nil +} + +func generalUpdateFunc(db DBGroupCacher, sync SyncGroupCacher, act BuildActor, mutex *gosync.Mutex) cache.UpdateFunc { + return func(ids []string) ([]interface{}, error) { + mutex.Lock() + defer mutex.Unlock() + newAct := act.Clone() + builder := NewDescBuilder(db, sync, newAct) + descs, err := builder.Build(ids) + if err != nil { + return nil, err + } + + return descs, nil + } +} + +func generalLoadFunc(db DBGroupCacher, sync SyncGroupCacher, act BuildActor, mutex *gosync.Mutex) cache.LoadFunc { + return func() ([]interface{}, error) { + mutex.Lock() + defer mutex.Unlock() + newAct := act.Clone() + builder := NewDescBuilder(db, sync, newAct) + + ids, err := act.AllIDs() + if err != nil { + return nil, err + } + descs, err := builder.Build(ids) + if err != nil { + return nil, err + } + return descs, nil + } +} + +// generalGetUpdateFunc provides the ability to generate regularly updated data. +func generalGetUpdateFunc(isBaremetal bool) cache.GetUpdateFunc { + // The purpose of the counter is to update the data in full at regular intervals. + fullUpdateCounter := 0 + return func(d []interface{}) ([]string, error) { + // Full update every 10 minutes(30s * 20) + if isBaremetal && fullUpdateCounter >= options.GetOptions().BaremetalCandidateCacheReloadCount { + fullUpdateCounter = 1 + log.Infof("FullUpdateCounter: %d, update all baremetals.", fullUpdateCounter) + return nil, nil + } + + if !isBaremetal && fullUpdateCounter >= options.GetOptions().HostCandidateCacheReloadCount { + fullUpdateCounter = 1 + log.Infof("FullUpdateCounter: %d, update all hosts.", fullUpdateCounter) + return nil, nil + } + + allStatus := make(map[string]time.Time, len(d)) + // This will reflect the key `ID` and `UpdatedAt`,maybe one day can optimize this part. + for _, item := range d { + r := reflect.ValueOf(item) + f := reflect.Indirect(r) + key := f.FieldByName("ID") + value := f.FieldByName("UpdatedAt") + if key.IsValid() && value.IsValid() { + allStatus[key.String()] = value.Interface().(time.Time) + } else { + log.Errorf("get `ID` and `UpdatedAt` errror in host:%v\n", item) + } + } + + fullUpdateCounter++ + modified, err := models.AllHostStatus(isBaremetal) + if err != nil { + return nil, err + } + modifiedIds := make([]string, 0, len(modified)) + // Aggregate the updated hosts + for _, status := range modified { + // If host does not exist[ok=false] or has updated will be in update list. + if t, ok := allStatus[status.ID]; !ok || !t.Equal(status.UpdatedAt) { + modifiedIds = append(modifiedIds, status.ID) + } + } + if len(modifiedIds) == 0 { + return nil, fmt.Errorf("%s", "no need update all") + } + return modifiedIds, nil + } +} + +func newHostCache(db DBGroupCacher, sync SyncGroupCacher) cache.CachedItem { + mutex := new(gosync.Mutex) + update := generalUpdateFunc(db, sync, &HostBuilder{}, mutex) + load := generalLoadFunc(db, sync, &HostBuilder{}, mutex) + getUpdate := generalGetUpdateFunc(false) + item := new(candidateItem) + + item.CachedItem = cache.NewCacheItem( + HostCandidateCache, + u.ToDuration(options.GetOptions().HostCandidateCacheTTL), + u.ToDuration(options.GetOptions().HostCandidateCachePeriod), + uuidKey, + update, + load, + getUpdate, + ) + return item +} + +func newBaremetalCache(db DBGroupCacher, sync SyncGroupCacher) cache.CachedItem { + // The mutex solves the possible dirty data asked lead to over-commit. + mutex := new(gosync.Mutex) + update := generalUpdateFunc(db, sync, &BaremetalBuilder{}, mutex) + load := generalLoadFunc(db, sync, &BaremetalBuilder{}, mutex) + getUpdate := generalGetUpdateFunc(true) + item := new(candidateItem) + + item.CachedItem = cache.NewCacheItem( + BaremetalCandidateCache, + u.ToDuration(options.GetOptions().BaremetalCandidateCacheTTL), + u.ToDuration(options.GetOptions().BaremetalCandidateCachePeriod), + uuidKey, + update, + load, + getUpdate, + ) + return item +} diff --git a/pkg/scheduler/cache/candidate/desc.go b/pkg/scheduler/cache/candidate/desc.go new file mode 100644 index 0000000000..a4c28054a3 --- /dev/null +++ b/pkg/scheduler/cache/candidate/desc.go @@ -0,0 +1,137 @@ +package candidate + +import ( + "time" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + "github.com/yunionio/pkg/utils" +) + +type baseDesc struct { + ID string `json:"id"` + Name string `json:"name"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (b *baseDesc) UUID() string { + return b.ID +} + +type baseHostDesc struct { + baseDesc + + Status string `json:"status"` + CPUCount int64 `json:"cpu_count"` + MemSize int64 `json:"mem_size"` + Networks []*models.NetworkSchedResult `json:"networks"` + HostStatus string `json:"host_status"` + Enabled bool `json:"enabled"` + HostType string `json:"host_type"` + IsBaremetal bool `json:"is_baremetal"` + NodeCount int64 `json:"node_count"` + Tenants map[string]int64 `json:"tenants"` + ZoneID string `json:"zone_id"` + PoolID string `json:"pool_id"` + ClusterID string `json:"cluster_id"` + Aggregates []*models.Aggregate `json:"aggregates"` + HostAggregates []*models.Aggregate `json:"host_aggregates"` +} + +func (b *baseHostDesc) fillAggregates() error { + b.Aggregates = make([]*models.Aggregate, 0) + objs, err := models.All(models.Aggregates) + if err != nil { + return err + } + for _, obj := range objs { + agg := obj.(*models.Aggregate) + b.Aggregates = append(b.Aggregates, agg) + } + + aggs, err := models.HostAggregates(b.ID) + if err != nil { + return err + } + b.HostAggregates = aggs + return nil +} + +func (b *baseHostDesc) GetAggregates() []*models.Aggregate { + return b.Aggregates +} + +func (b *baseHostDesc) GetHostAggregates() []*models.Aggregate { + return b.HostAggregates +} + +func (b *baseHostDesc) fillNetworks(hostID string) error { + net, err := models.HostNetworkSchedResults(hostID) + if err != nil { + return err + } + b.Networks = net + return nil +} + +func (h *baseHostDesc) GetEnableStatus() string { + if h.Enabled { + return "enable" + } + return "disable" +} + +func (h *baseHostDesc) GetHostType() string { + if h.HostType == api.HostTypeBaremetal && h.IsBaremetal { + return api.HostTypeBaremetal + } + return h.HostType +} + +func HostsResidentTenantStats(hostIDs []string) (map[string]map[string]interface{}, error) { + residentTenantStats, err := models.ResidentTenantsInHosts(hostIDs) + if err != nil { + return nil, err + } + stat3 := make([]utils.StatItem3, len(residentTenantStats)) + for i, item := range residentTenantStats { + stat3[i] = item + } + return utils.ToStatDict3(stat3) +} + +func HostResidentTenantCount(id string) (map[string]int64, error) { + residentTenantDict, err := HostsResidentTenantStats([]string{id}) + if err != nil { + return nil, err + } + tenantMap, ok := residentTenantDict[id] + if !ok { + log.V(10).Infof("Not found host ID: %s when fill resident tenants, may be no guests on it.", id) + return nil, nil + } + rets := make(map[string]int64, len(tenantMap)) + for tenantID, countObj := range tenantMap { + rets[tenantID] = countObj.(int64) + } + return rets, nil +} + +type DescBuilder struct { + dbGroupCache DBGroupCacher + syncGroupCache SyncGroupCacher + actor BuildActor +} + +func NewDescBuilder(db DBGroupCacher, sync SyncGroupCacher, act BuildActor) *DescBuilder { + return &DescBuilder{ + dbGroupCache: db, + syncGroupCache: sync, + actor: act, + } +} + +func (d *DescBuilder) Build(ids []string) ([]interface{}, error) { + return d.actor.Do(ids, d.dbGroupCache, d.syncGroupCache) +} diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go new file mode 100644 index 0000000000..9551fd499f --- /dev/null +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -0,0 +1,1410 @@ +package candidate + +import ( + "encoding/json" + "fmt" + "strings" + gosync "sync" + "sync/atomic" + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + "github.com/yunionio/onecloud/pkg/scheduler/cache/db" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + "github.com/yunionio/pkg/util/errors" + "github.com/yunionio/pkg/util/sets" + "github.com/yunionio/pkg/utils" + "github.com/yunionio/pkg/util/workqueue" + + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +type HostDesc struct { + baseHostDesc + + // cpu + CPUMHZ int64 `json:"cpu_mhz"` + CPUCmtbound float64 `json:"cpu_cmtbound"` + CPUDesc string `json:"cpu_desc"` + CPUCache int64 `json:"cpu_cache"` + CPUReserved int64 `json:"cpu_reserved"` + CPUBoundCount int64 `json:"cpu_bound_count"` + CPULoad *float64 `json:"cpu_load"` + TotalCPUCount int64 `json:"total_cpu_count"` + RunningCPUCount int64 `json:"running_cpu_count"` + CreatingCPUCount int64 `json:"creating_cpu_count"` + RequiredCPUCount int64 `json:"required_cpu_count"` + FakeDeletedCPUCount int64 `json:"fake_deleted_cpu_count"` + FreeCPUCount int64 `json:"free_cpu_count"` + + // memory + MemCmtbound float64 `json:"mem_cmtbound"` + MemReserved int64 `json:"mem_reserved"` + TotalMemSize int64 `json:"total_mem_size"` + FreeMemSize int64 `json:"free_mem_size"` + RunningMemSize int64 `json:"running_mem_size"` + CreatingMemSize int64 `json:"creating_mem_size"` + RequiredMemSize int64 `json:"required_mem_size"` + FakeDeletedMemSize int64 `json:"fake_deleted_mem_size"` + + // storage + Storages []*Storage `json:"storages"` + storageTypes []string `json:"storage_types"` + + // IO + IOBoundCount int64 `json:"io_bound_count"` + IOLoad *float64 `json:"io_load"` + + // server + GuestCount int64 `json:"guest_count"` + CreatingGuestCount int64 `json:"creating_guest_count"` + RunningGuestCount int64 `json:"running_guest_count"` + + Groups *GroupCounts `json:"groups"` + Metadata map[string]string `json:"metadata"` + IsolatedDevices []*IsolatedDeviceDesc `json:"isolated_devices"` + ManagerID *string `json:"manager_id"` + IsMaintenance bool `json:"is_maintenance"` + GuestReservedResource *ReservedResource `json:"guest_reserved_resource"` + GuestReservedResourceUsed *ReservedResource `json:"guest_reserved_used"` +} + +type ReservedResource struct { + CPUCount int64 `json:"cpu_count"` + MemorySize int64 `json:"memory_size"` + StorageSize int64 `json:"storage_size"` +} + +func NewReservedResource(cpu, mem, storage int64) *ReservedResource { + return &ReservedResource{ + CPUCount: cpu, + MemorySize: mem, + StorageSize: storage, + } +} + +func CpuIsolatedDevReservedCount() int64 { + return o.GetOptions().CpuReservedPerIsolatedDevice +} + +func MemIsolatedDevReservedSize() int64 { + return o.GetOptions().MemoryReservedPerIsolatedDevice +} + +func StorageIsolatedDevReservedSize() int64 { + return o.GetOptions().StorageReservedPerIsolatedDevice +} + +func NewGuestReservedResourceByBuilder(b *HostBuilder, host *models.Host) (ret *ReservedResource) { + ret = NewReservedResource(0, 0, 0) + //isoDevs := b.getUnusedIsolatedDevices(host.ID) + isoDevs := b.getIsolatedDevices(host.ID) + hostDevsCount := int64(len(isoDevs)) + if hostDevsCount == 0 { + return + } + + cpuPerDevRsvd := CpuIsolatedDevReservedCount() + memPerDevRsvd := MemIsolatedDevReservedSize() + storagePerDevRsvd := StorageIsolatedDevReservedSize() + ret.CPUCount = hostDevsCount * cpuPerDevRsvd + ret.MemorySize = hostDevsCount * memPerDevRsvd + ret.StorageSize = hostDevsCount * storagePerDevRsvd + + return +} + +func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *models.Host) (ret *ReservedResource, err error) { + ret = NewReservedResource(0, 0, 0) + gst := b.getIsolatedDeviceGuests(host.ID) + if len(gst) == 0 { + return + } + var ( + cpu int64 = 0 + mem int64 = 0 + disk int64 = 0 + ) + for _, g := range gst { + dSize, err2 := g.DiskSize(true) + if err2 != nil { + err = err2 + return + } + disk += dSize + if o.GetOptions().IgnoreNonRunningGuests && !g.IsRunning() { + continue + } + cpu += g.VCPUCount + mem += g.VMemSize + } + ret.CPUCount = cpu + ret.MemorySize = mem + ret.StorageSize = disk + return +} + +type Storage struct { + ID string `json:"id"` + Name string `json:"name"` + Capacity int64 `json:"capacity"` + StorageType string `json:"type"` + UsedCapacity int64 `json:"used"` + WasteCapacity int64 `json:"waste"` + FreeCapacity int64 `json:"free"` + VCapacity int64 `json:"vcapacity"` + Cmtbound float64 `json:"cmtbound"` + StorageDriver string `json:"driver"` + Adapter string `json:"adapter"` + Splits []string `json:"splits"` + Range string `json:"range"` + Conf string `json:"conf"` + MinStripSize int `json:"min_strip_size"` + MaxStripSize int `json:"max_strip_size"` + Size int `json:"size"` +} + +func (storage *Storage) GetFreeSize() int64 { + return storage.GetTotalSize() - storage.UsedCapacity - storage.WasteCapacity +} + +func (storage *Storage) GetTotalSize() int64 { + return int64(float64(storage.Capacity) * storage.Cmtbound) +} + +func (storage *Storage) IsLocal() bool { + return utils.IsLocalStorage(storage.StorageType) +} + +type HostBuilder struct { + clusters cache.Cache + + residentTenantDict map[string]map[string]interface{} + + hosts []interface{} + hostDict map[string]interface{} + + guests []interface{} + guestDict map[string]interface{} + guestIDs []string + + hostStorages []interface{} + hostStoragesDict map[string][]interface{} + storages []interface{} + storageDict map[string]interface{} + storageStatesSizeDict map[string]map[string]interface{} + + hostGuests map[string][]interface{} + + groupGuests []interface{} + groups []interface{} + groupDict map[string]interface{} + hostGroupCountDict HostGroupCountDict + + hostMetadatas []interface{} + hostMetadatasDict map[string][]interface{} + guestMetadatas []interface{} + guestMetadatasDict map[string][]interface{} + + diskStats []models.StorageCapacity + isolatedDevicesDict map[string][]interface{} + + cpuIOLoads map[string]map[string]float64 +} + +func (h *HostDesc) String() string { + s, _ := json.Marshal(h) + return string(s) +} + +func (h *HostDesc) Type() int { + // Guest type + return 0 +} + +func (h *HostDesc) Get(key string) interface{} { + switch key { + case "ID": + return h.ID + + case "Name": + return h.Name + + case "CPUCount": + return h.CPUCount + + case "MemSize": + return h.MemSize + + case "PoolID": + return h.PoolID + + case "ZoneID": + return h.ZoneID + + case "ClusterID": + return h.ClusterID + + case "TotalCPUCount": + return h.GetTotalCPUCount(true) + + case "FreeCPUCount": + return h.GetFreeCPUCount(false) + + case "TotalMemSize": + return h.GetTotalMemSize(true) + + case "FreeMemSize": + return h.GetFreeMemSize(false) + + case "Groups": + return h.Groups + + case "IsolatedDevices": + return h.IsolatedDevices + + case "Status": + return h.Status + + case "TotalStorageSize": + return h.totalStorageSize(false, true) + + case "TotalLocalStorageSize": + return h.totalStorageSize(true, true) + + case "FreeStorageSize": + return h.freeStorageSize(false, false) + + case "FreeLocalStorageSize": + return h.freeStorageSize(true, false) + + case "StorageTypes": + return h.storageTypes + + case "HostStatus": + return h.HostStatus + + case "EnableStatus": + return h.GetEnableStatus() + + case "HostType": + return h.HostType + + case "IsBaremetal": + return h.IsBaremetal + + default: + index := strings.Index(key, ":") + if index >= 0 { + masterKey := key[0:index] + slaveKey := key[index+1:] + + switch masterKey { + case "FreeStorageSize": + storageType := slaveKey + return h.freeStorageSizeOfType(storageType, false) + } + } + return nil + } +} + +func (h *HostDesc) XGet(key string, kind core.Kind) interface{} { + return core.XGetCalculator(h, key, kind) +} + +func (h *HostDesc) GetTotalLocalStorageSize(useRsvd bool) int64 { + return h.totalStorageSize(true, useRsvd) +} + +func (h *HostDesc) GetFreeLocalStorageSize(useRsvd bool) int64 { + return h.freeStorageSize(true, useRsvd) +} + +func (h *HostDesc) totalStorageSize(onlyLocal, useRsvd bool) int64 { + + total := int64(0) + for _, storage := range h.Storages { + if !onlyLocal || storage.IsLocal() { + total += storage.GetTotalSize() + } + } + + if onlyLocal { + return reservedResourceMinusCal(total, h.GuestReservedResource.StorageSize, useRsvd) + } + return total +} + +func (h *HostDesc) freeStorageSize(onlyLocal, useRsvd bool) int64 { + total := int64(0) + for _, storage := range h.Storages { + if !onlyLocal || storage.IsLocal() { + total += storage.GetFreeSize() + } + } + + total = total + h.GuestReservedResourceUsed.StorageSize - h.GetReservedStorageSize() + sizeSub := h.GuestReservedResource.StorageSize - h.GuestReservedResourceUsed.StorageSize + if sizeSub < 0 { + total += sizeSub + } + if useRsvd { + return reservedResourceAddCal(total, h.GuestReservedStorageSizeFree(), useRsvd) + } + + return total +} + +func (h *HostDesc) GetFreeStorageSizeOfType(sType string, useRsvd bool) int64 { + return h.freeStorageSizeOfType(sType, useRsvd) +} + +func (h *HostDesc) freeStorageSizeOfType(storageType string, useRsvd bool) int64 { + total := int64(0) + for _, storage := range h.Storages { + if storage.StorageType == storageType { + total += storage.GetFreeSize() + } + } + if utils.IsLocalStorage(storageType) { + total = total + h.GuestReservedResourceUsed.StorageSize - h.GetReservedStorageSize() + sizeSub := h.GuestReservedResource.StorageSize - h.GuestReservedResourceUsed.StorageSize + if sizeSub < 0 { + total += sizeSub + } + } + if useRsvd { + return reservedResourceAddCal(total, h.GuestReservedStorageSizeFree(), useRsvd) + } + + return total +} + +func reservedResourceCal( + curRes, rsvdRes int64, + useRsvd, minusRsvd bool, +) int64 { + actRes := curRes + if useRsvd { + if minusRsvd { + actRes -= rsvdRes + } else { + actRes += rsvdRes + } + } + return actRes +} + +func reservedResourceAddCal(curRes, rsvdRes int64, useRsvd bool) int64 { + return reservedResourceCal(curRes, rsvdRes, useRsvd, false) +} + +func reservedResourceMinusCal(curRes, rsvdRes int64, useRsvd bool) int64 { + return reservedResourceCal(curRes, rsvdRes, useRsvd, true) +} + +func (h *HostDesc) GetTotalMemSize(useRsvd bool) int64 { + return reservedResourceMinusCal(h.TotalMemSize, h.GuestReservedResource.MemorySize, useRsvd) +} + +func (h *HostDesc) GetFreeMemSize(useRsvd bool) int64 { + return reservedResourceAddCal(h.FreeMemSize, h.GuestReservedMemSizeFree(), useRsvd) +} + +func (h *HostDesc) GuestReservedMemSizeFree() int64 { + return h.GuestReservedResource.MemorySize - h.GuestReservedResourceUsed.MemorySize +} + +func (h *HostDesc) GuestReservedCPUCountFree() int64 { + return h.GuestReservedResource.CPUCount - h.GuestReservedResourceUsed.CPUCount +} + +func (h *HostDesc) GuestReservedStorageSizeFree() int64 { + return h.GuestReservedResource.StorageSize - h.GuestReservedResourceUsed.StorageSize +} + +func (h *HostDesc) GetReservedMemSize() int64 { + return h.GuestReservedResource.MemorySize + h.MemReserved +} + +func (h *HostDesc) GetReservedCPUCount() int64 { + return h.GuestReservedResource.CPUCount + h.CPUReserved +} + +func (h *HostDesc) GetReservedStorageSize() int64 { + return h.GuestReservedResource.StorageSize +} + +func (h *HostDesc) GetTotalCPUCount(useRsvd bool) int64 { + return reservedResourceMinusCal(h.TotalCPUCount, h.GuestReservedResource.CPUCount, useRsvd) +} + +func (h *HostDesc) GetFreeCPUCount(useRsvd bool) int64 { + return reservedResourceAddCal(h.FreeCPUCount, h.GuestReservedCPUCountFree(), useRsvd) +} + +func (h *HostDesc) IndexKey() string { + return h.ID +} + +func (h *HostDesc) UnusedIsolatedDevices() []*IsolatedDeviceDesc { + ret := make([]*IsolatedDeviceDesc, 0) + for _, dev := range h.IsolatedDevices { + if len(dev.GuestID) == 0 { + ret = append(ret, dev) + } + } + return ret +} + +func (h *HostDesc) UnusedIsolatedDevicesByType(devType string) []*IsolatedDeviceDesc { + ret := make([]*IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if dev.DevType == devType { + ret = append(ret, dev) + } + } + return ret +} + +func (h *HostDesc) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc { + ret := make([]*IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if strings.Contains(dev.VendorModel(), vendorModel) { + ret = append(ret, dev) + } + } + return ret +} + +func (h *HostDesc) UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc { + ret := make([]*IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if strings.Contains(dev.Model, model) { + ret = append(ret, dev) + } + } + return ret +} + +func (h *HostDesc) GetIsolatedDevice(devID string) *IsolatedDeviceDesc { + for _, dev := range h.IsolatedDevices { + if dev.ID == devID { + return dev + } + } + return nil +} + +func (h *HostDesc) UnusedGpuDevices() []*IsolatedDeviceDesc { + ret := make([]*IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if strings.HasPrefix(dev.DevType, "GPU") { + ret = append(ret, dev) + } + } + return ret +} + +type WaitGroupWrapper struct { + gosync.WaitGroup +} + +func (w *WaitGroupWrapper) Wrap(cb func()) { + w.Add(1) + go func() { + cb() + w.Done() + }() +} + +func waitTimeOut(wg *WaitGroupWrapper, timeout time.Duration) bool { + ch := make(chan struct{}) + go func() { + wg.Wait() + close(ch) + }() + select { + case <-ch: + return true + case <-time.After(timeout): + return false + } +} + +func (b *HostBuilder) init(ids []string, dbCache DBGroupCacher, syncCache SyncGroupCacher) error { + wg := &WaitGroupWrapper{} + errMessageChannel := make(chan error, 12) + defer close(errMessageChannel) + setFuncs := []func(){ + func() { b.setClusters(dbCache, errMessageChannel) }, + func() { b.setHosts(ids, errMessageChannel) }, + func() { + b.setGuests(ids, errMessageChannel) + b.setGroupInfo(errMessageChannel) + b.setMetadataInfo(ids, errMessageChannel) + }, + func() { + b.setStorages(ids, errMessageChannel) + b.setDiskStats(errMessageChannel) + }, + func() { b.setResidentTenantStats(ids, errMessageChannel) }, + func() { b.setMetadataInfo(ids, errMessageChannel) }, + func() { b.setIsolatedDevs(ids, errMessageChannel) }, + func() { b.setCPUIOLoadInfo(errMessageChannel) }, + } + + for _, f := range setFuncs { + wg.Wrap(f) + } + + if ok := waitTimeOut(wg, time.Duration(20*time.Second)); !ok { + log.Errorln("HostBuilder waitgroup timeout.") + } + + if len(errMessageChannel) != 0 { + errMessages := make([]string, 0) + lengthChan := len(errMessageChannel) + for ; lengthChan > 0; lengthChan-- { + msg := fmt.Sprintf("%s", <-errMessageChannel) + log.Errorf("Get error from chan: %s", msg) + errMessages = append(errMessages, msg) + } + return fmt.Errorf("%s\n", strings.Join(errMessages, ";")) + } + + return nil +} + +func (b *HostBuilder) setClusters(dbCache DBGroupCacher, errMessageChannel chan error) { + clusters, err := dbCache.Get(db.ClusterDBCache) + if err != nil { + errMessageChannel <- err + return + } + b.clusters = clusters + return +} + +func (b *HostBuilder) setHosts(ids []string, errMessageChannel chan error) { + hosts, err := models.FetchHypervisorHostByIDs(ids) + if err != nil { + errMessageChannel <- err + return + } + hostDict, err := utils.ToDict(hosts, func(obj interface{}) (string, error) { + host, ok := obj.(*models.Host) + if !ok { + return "", utils.ConvertError(obj, "*models.Host") + } + return host.ID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + b.hosts = hosts + b.hostDict = hostDict + return +} + +func (b *HostBuilder) setStorages(ids []string, errMessageChannel chan error) { + hostStorages, err := models.FetchByHostIDs(models.HostStorages, ids) + if err != nil { + errMessageChannel <- err + return + } + storageIDs := make([]string, len(hostStorages)) + func() { + for i, s := range hostStorages { + storageIDs[i] = s.(*models.HostStorage).StorageID + } + }() + storages, err := models.FetchByIDs(models.Storages, storageIDs) + if err != nil { + errMessageChannel <- err + return + } + + hostStoragesDict, err := utils.GroupBy(hostStorages, func(obj interface{}) (string, error) { + storage, ok := obj.(*models.HostStorage) + if !ok { + return "", utils.ConvertError(obj, "*models.HostStorage") + } + return storage.HostID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + storageDict, err := utils.ToDict(storages, func(obj interface{}) (string, error) { + storage, ok := obj.(*models.Storage) + if !ok { + return "", utils.ConvertError(obj, "*models.Storage") + } + return storage.ID, nil + }) + + if err != nil { + errMessageChannel <- err + return + } + b.hostStorages = hostStorages + b.hostStoragesDict = hostStoragesDict + b.storages = storages + b.storageDict = storageDict + return +} + +func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { + guests, err := models.FetchGuestByHostIDs(ids) + if err != nil { + errMessageChannel <- err + return + } + guestIDs := make([]string, len(guests)) + func() { + for i, gst := range guests { + guestIDs[i] = gst.(*models.Guest).ID + } + }() + + hostGuests, err := utils.GroupBy(guests, func(obj interface{}) (string, error) { + gst, ok := obj.(*models.Guest) + if !ok { + return "", utils.ConvertError(obj, "*models.Guest") + } + return gst.HostID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + guestDict, err := utils.ToDict(guests, func(obj interface{}) (string, error) { + gst, ok := obj.(*models.Guest) + if !ok { + return "", utils.ConvertError(obj, "*models.Guest") + } + return gst.ID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + b.guestIDs = guestIDs + b.guests = guests + b.hostGuests = hostGuests + b.guestDict = guestDict + return +} + +func (b *HostBuilder) setResidentTenantStats(hostIDs []string, errMessageChannel chan error) { + residentTenantDict, err := HostsResidentTenantStats(hostIDs) + if err != nil { + errMessageChannel <- err + return + } + b.residentTenantDict = residentTenantDict + return +} + +func (b *HostBuilder) setGroupInfo(errMessageChannel chan error) { + groupGuests, err := models.FetchByGuestIDs(models.GroupGuests, b.guestIDs) + if err != nil { + errMessageChannel <- err + return + } + + groupIds, err := utils.SelectDistinct(groupGuests, func(obj interface{}) (string, error) { + g, ok := obj.(*models.GroupGuest) + if !ok { + return "", utils.ConvertError(obj, "*models.GroupGuest") + } + return g.GroupID, nil + }) + + if err != nil { + errMessageChannel <- err + return + } + + groups, err := models.FetchGroupByIDs(groupIds) + if err != nil { + errMessageChannel <- err + return + } + + groupDict, err := utils.ToDict(groups, func(obj interface{}) (string, error) { + grp, ok := obj.(*models.Group) + if !ok { + return "", utils.ConvertError(obj, "*models.Group") + } + return grp.ID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + b.groups = groups + b.groupDict = groupDict + b.groupGuests = groupGuests + hostGroupCountDict, err := b.toHostGroupCountDict(groupGuests) + if err != nil { + errMessageChannel <- err + return + } + b.hostGroupCountDict = hostGroupCountDict + return +} + +type GroupCount struct { + ID string `json:"id"` // group id + Name string `json:"name"` // group name + Count int64 `json:"count"` // guest count +} + +type GroupCounts struct { + Data map[string]*GroupCount `json:"data"` // group_id: group_count +} + +func NewGroupCounts() *GroupCounts { + return &GroupCounts{ + Data: make(map[string]*GroupCount), + } +} + +type HostGroupCountDict map[string]*GroupCounts + +func (b *HostBuilder) toHostGroupCountDict(groupGuests []interface{}) (HostGroupCountDict, error) { + d := make(map[string]*GroupCounts) + for _, groupGuestObj := range groupGuests { + groupGuest := groupGuestObj.(*models.GroupGuest) + groupObj, grpOK := b.groupDict[groupGuest.GroupID] + guestObj, gstOK := b.guestDict[*groupGuest.GuestID] + if !grpOK || !gstOK { + continue + } + hostObj, ok := b.hostDict[guestObj.(*models.Guest).HostID] + if !ok { + continue + } + host := hostObj.(*models.Host) + group := groupObj.(*models.Group) + + counts, ok := d[host.ID] + if !ok { + counts = NewGroupCounts() + d[host.ID] = counts + } + count, ok := counts.Data[group.ID] + if !ok { + count = &GroupCount{ID: group.ID, Name: group.Name, Count: 1} + counts.Data[group.ID] = count + } else { + count.Count++ + } + counts.Data[host.ID] = count + } + return d, nil +} + +func (b *HostBuilder) setMetadataInfo(hostIDs []string, errMessageChannel chan error) { + hostMetadataNames := []string{"dynamic_load_cpu_percent", "dynamic_load_io_util", + "enable_sriov", "bridge_driver"} + hostMetadataNames = append(hostMetadataNames, models.HostExtraFeature...) + hostMetadatas, err := models.FetchMetadatas(models.HostResourceName, hostIDs, hostMetadataNames) + if err != nil { + errMessageChannel <- err + return + } + guestMetadataNames := []string{"app_tags"} + guestMetadatas, err := models.FetchMetadatas(models.GuestResourceName, b.guestIDs, guestMetadataNames) + if err != nil { + errMessageChannel <- err + return + } + idFunc := func(obj interface{}) (string, error) { + metadata, ok := obj.(*models.Metadata) + if !ok { + return "", utils.ConvertError(obj, "*models.Metadata") + } + id := strings.Split(metadata.ID, "::")[1] + return id, nil + } + hostMetadatasDict, err := utils.GroupBy(hostMetadatas, idFunc) + if err != nil { + errMessageChannel <- err + return + } + guestMetadatasDict, err := utils.GroupBy(guestMetadatas, idFunc) + if err != nil { + errMessageChannel <- err + return + } + b.hostMetadatas = hostMetadatas + b.hostMetadatasDict = hostMetadatasDict + b.guestMetadatas = guestMetadatas + b.guestMetadatasDict = guestMetadatasDict + return +} + +func (b *HostBuilder) setIsolatedDevs(ids []string, errMessageChannel chan error) { + devs, err := models.FetchByHostIDs(models.IsolatedDevices, ids) + if err != nil { + errMessageChannel <- err + return + } + dict, err := utils.GroupBy(devs, func(obj interface{}) (string, error) { + dev, ok := obj.(*models.IsolatedDevice) + if !ok { + return "", utils.ConvertError(obj, "*models.IsolatedDevice") + } + return dev.HostID, nil + }) + if err != nil { + errMessageChannel <- err + return + } + b.isolatedDevicesDict = dict +} + +func (b *HostBuilder) setDiskStats(errMessageChannel chan error) { + storageIDs := make([]string, len(b.storages)) + func() { + for i, s := range b.storages { + storageIDs[i] = s.(*models.Storage).ID + } + }() + capacities, err := models.GetStorageCapacities(storageIDs) + stat3 := make([]utils.StatItem3, len(capacities)) + for i, item := range capacities { + stat3[i] = item + } + if err != nil { + errMessageChannel <- err + return + } + storageStatesSizeDict, _ := utils.ToStatDict3(stat3) + b.storageStatesSizeDict = storageStatesSizeDict + b.diskStats = capacities + return +} + +func (b *HostBuilder) setCPUIOLoadInfo(errMessageChannel chan error) { + return +} + +func (b *HostBuilder) Clone() BuildActor { + return &HostBuilder{} +} + +func (b *HostBuilder) Type() string { + return HostDescBuilder +} + +func (b *HostBuilder) AllIDs() ([]string, error) { + return models.AllHostIDs() +} + +func (b *HostBuilder) Do(ids []string, dbCache DBGroupCacher, syncCache SyncGroupCacher) ([]interface{}, error) { + err := b.init(ids, dbCache, syncCache) + if err != nil { + return nil, err + } + descs, err := b.build() + if err != nil { + return nil, err + } + return descs, nil +} + +func (b *HostBuilder) build() ([]interface{}, error) { + schedDescs := make([]interface{}, len(b.hosts)) + errs := []error{} + var descResultLock gosync.Mutex + var descedLen int32 + + buildOne := func(i int) { + if i >= len(b.hosts) { + log.Errorf("invalid host index[%d] in b.hosts:%v\n", i, b.hosts) + return + } + host := b.hosts[i] + desc, err := b.buildOne(host.(*models.Host)) + if err != nil { + descResultLock.Lock() + errs = append(errs, err) + descResultLock.Unlock() + return + } + descResultLock.Lock() + schedDescs[atomic.AddInt32(&descedLen, 1)-1] = desc + descResultLock.Unlock() + } + + workqueue.Parallelize(o.GetOptions().HostBuildParallelizeSize, len(b.hosts), buildOne) + schedDescs = schedDescs[:descedLen] + if len(errs) > 0 { + //return nil, errors.NewAggregate(errs) + err := errors.NewAggregate(errs) + log.Warningf("Build schedule descs error: %s", err) + } + + return schedDescs, nil +} + +func (b *HostBuilder) buildOne(host *models.Host) (interface{}, error) { + desc := new(HostDesc) + desc.ID = host.ID + desc.UpdatedAt = host.UpdatedAt + desc.Name = host.Name + desc.Status = host.Status + desc.Metadata = make(map[string]string) + desc.HostStatus = host.HostStatus + desc.Enabled = host.Enabled + desc.HostType = host.HostType + desc.IsBaremetal = host.IsBaremetal + desc.ManagerID = host.ManagerID + desc.IsMaintenance = host.IsMaintenance + + // TODO: purge PoolID & ClusterID not used + desc.PoolID = host.PoolID + desc.ClusterID = host.ClusterID + + desc.CPUCount = host.CPUCount + desc.CPUCmtbound = host.CPUOverCommitBound() + desc.CPUDesc = host.CPUDesc + desc.CPUCache = host.CPUCache + desc.CPUReserved = host.CPUReserved + desc.NodeCount = host.NodeCount + desc.CPUMHZ = host.CPUMHZ + + desc.MemSize = host.MemSize + desc.MemCmtbound = host.MemOverCommitBound() + desc.MemReserved = host.MemReserved + + desc.GuestReservedResource = NewGuestReservedResourceByBuilder(b, host) + guestRsvdUsed, err := NewGuestReservedResourceUsedByBuilder(b, host) + if err != nil { + return nil, err + } + desc.GuestReservedResourceUsed = guestRsvdUsed + + fillFuncs := []func(*HostDesc, *models.Host) error{ + b.fillGuestsResourceInfo, + b.fillZoneID, + b.fillStorages, + b.fillResidentTenants, + b.fillResidentGroups, + b.fillMetadata, + b.fillIsolatedDevices, + b.fillCPUIOLoads, + b.fillNetworks, + } + + for _, f := range fillFuncs { + err := f(desc, host) + if err != nil { + return nil, err + } + } + + err = desc.fillAggregates() + if err != nil { + return nil, err + } + + return desc, nil +} + +func (b *HostBuilder) fillNetworks(desc *HostDesc, host *models.Host) error { + return desc.fillNetworks(host.ID) +} + +func _in(s string, ss []string) bool { + for _, str := range ss { + if s == str { + return true + } + } + return false +} + +func (b *HostBuilder) fillGuestsResourceInfo(desc *HostDesc, host *models.Host) error { + var ( + guestCount int64 + runningCount int64 + memSize int64 + memReqSize int64 + memFakeDeletedSize int64 + cpuCount int64 + cpuReqCount int64 + cpuBoundCount int64 + cpuFakeDeletedCount int64 + ioBoundCount int64 + creatingMemSize int64 + creatingCPUCount int64 + creatingGuestCount int64 + ) + guestsOnHost, ok := b.hostGuests[host.ID] + if !ok { + guestsOnHost = []interface{}{} + } + + for _, gst := range guestsOnHost { + guest := gst.(*models.Guest) + if guest.IsRunning() { + runningCount++ + memSize += guest.VMemSize + cpuCount += guest.VCPUCount + } else if guest.IsCreating() { + creatingGuestCount++ + creatingMemSize += guest.VMemSize + creatingCPUCount += guest.VCPUCount + } else if guest.IsGuestFakeDeleted() && _in(guest.Status, []string{models.VmReady}) { + memFakeDeletedSize += guest.VMemSize + cpuFakeDeletedCount += guest.VCPUCount + } + guestCount++ + cpuReqCount += guest.VCPUCount + memReqSize += guest.VMemSize + + appTags := b.guestAppTags(guest) + for _, tag := range appTags { + if tag == "cpu_bound" { + cpuBoundCount += guest.VCPUCount + } else if tag == "io_bound" { + ioBoundCount++ + } + } + } + desc.GuestCount = guestCount + desc.CreatingGuestCount = creatingGuestCount + desc.RunningGuestCount = runningCount + desc.RunningMemSize = memSize + desc.RequiredMemSize = memReqSize + desc.CreatingMemSize = creatingMemSize + desc.FakeDeletedMemSize = memFakeDeletedSize + desc.RunningCPUCount = cpuCount + desc.RequiredCPUCount = cpuReqCount + desc.CreatingCPUCount = creatingCPUCount + desc.FakeDeletedCPUCount = cpuFakeDeletedCount + + desc.TotalMemSize = int64(float64(desc.MemSize) * desc.MemCmtbound) + desc.TotalCPUCount = int64(float64(desc.CPUCount) * desc.CPUCmtbound) + + var memFreeSize int64 + var cpuFreeCount int64 + if o.GetOptions().IgnoreNonRunningGuests { + memFreeSize = desc.TotalMemSize - desc.RunningMemSize - desc.CreatingMemSize + cpuFreeCount = desc.TotalCPUCount - desc.RunningCPUCount - desc.CreatingCPUCount + } else { + memFreeSize = desc.TotalMemSize - desc.RequiredMemSize + cpuFreeCount = desc.TotalCPUCount - desc.RequiredCPUCount + if o.GetOptions().IgnoreFakeDeletedGuests { + memFreeSize += memFakeDeletedSize + cpuFreeCount += cpuFakeDeletedCount + } + } + + // free memory size calculate + rsvdUseMem := desc.GuestReservedResourceUsed.MemorySize + memFreeSize = memFreeSize + rsvdUseMem - desc.GetReservedMemSize() + memSub := desc.GuestReservedResource.MemorySize - desc.GuestReservedResourceUsed.MemorySize + if memSub < 0 { + memFreeSize += memSub + } + desc.FreeMemSize = memFreeSize + + // free cpu count calculate + rsvdUseCPU := desc.GuestReservedResourceUsed.CPUCount + cpuFreeCount = cpuFreeCount + rsvdUseCPU - desc.GetReservedCPUCount() + cpuSub := desc.GuestReservedResource.CPUCount - desc.GuestReservedResourceUsed.CPUCount + if cpuSub < 0 { + cpuFreeCount += cpuSub + } + desc.FreeCPUCount = cpuFreeCount + + desc.CPUBoundCount = cpuBoundCount + desc.IOBoundCount = ioBoundCount + + return nil +} + +func (b *HostBuilder) guestAppTags(guest *models.Guest) []string { + metadatas, ok := b.guestMetadatasDict[guest.ID] + if !ok { + return []string{} + } + for _, obj := range metadatas { + metadata, ok := obj.(*models.Metadata) + if !ok { + log.Errorf("%v", utils.ConvertError(obj, "*models.Metadata")) + return []string{} + } + if metadata.Key == "app_tags" { + tagsStr := metadata.Value + if len(tagsStr) > 0 { + return strings.Split(tagsStr, ",") + } + } + } + return []string{} +} + +func (b *HostBuilder) fillZoneID(desc *HostDesc, host *models.Host) error { + if host.ZoneID != "" { + desc.ZoneID = host.ZoneID + return nil + } + + zoneID, err := b.zoneID(host.ClusterID) + if err != nil { + panic(fmt.Errorf("host: %v, fillZoneID error: %v", host.ID, err)) + return err + } + desc.ZoneID = zoneID + return nil +} + +func (b *HostBuilder) zoneID(clusterID string) (string, error) { + obj, err := b.clusters.Get(clusterID) + if err != nil { + return "", err + } + cluster, ok := obj.(*models.Cluster) + if !ok { + return "", utils.ConvertError(obj, "*models.Cluster") + } + return cluster.ZoneID, nil +} + +func (b *HostBuilder) fillStorages(desc *HostDesc, host *models.Host) error { + objs, ok := b.hostStoragesDict[host.ID] + if !ok { + return nil + } + + var ( + rets = make([]*Storage, 0) + storageTypeMap = make(map[string]int, 0) + ) + for _, obj := range objs { + hostStorage, ok := obj.(*models.HostStorage) + if !ok { + return utils.ConvertError(obj, "*models.HostStorage") + } + storageID := hostStorage.StorageID + storageObj, ok := b.storageDict[storageID] + if !ok { + log.Warningf("Storage ID: %q not found when fill it", storageID) + return nil + } + storageModel := storageObj.(*models.Storage) + storage := new(Storage) + storage.ID = storageModel.ID + storage.Name = storageModel.Name + storage.Capacity = storageModel.Capacity + storage.StorageType = storageModel.StorageType + storage.UsedCapacity = b.storageUsedCapacity(storageModel, true) + storage.WasteCapacity = b.storageUsedCapacity(storageModel, false) + storage.Cmtbound = storageModel.OverCommitBound() + storage.VCapacity = storage.GetTotalSize() + storage.FreeCapacity = storage.GetFreeSize() + rets = append(rets, storage) + + storageTypeMap[storage.StorageType] = 0 + } + + desc.Storages = rets + + for storageType := range storageTypeMap { + desc.storageTypes = append(desc.storageTypes, storageType) + } + + return nil +} + +func (b *HostBuilder) storageUsedCapacity(storage *models.Storage, ready bool) int64 { + d, ok := b.storageStatesSizeDict[storage.ID] + if !ok { + return 0 + } + if ready { + obj, ok := d[models.DiskReady] + if !ok { + return 0 + } + return obj.(int64) + } + var total int64 + for status, sizeObj := range d { + if (status == models.DiskReady && ready) || (status != models.DiskReady && !ready) { + total += sizeObj.(int64) + } + } + return total +} + +func (b *HostBuilder) fillResidentTenants(desc *HostDesc, host *models.Host) error { + + tenantMap, ok := b.residentTenantDict[host.ID] + if !ok { + log.V(10).Infof("Not found host ID: %s when fill resident tenants, may be no guests on it.", host.ID) + return nil + } + + rets := make(map[string]int64, len(tenantMap)) + for tenantID, countObj := range tenantMap { + rets[tenantID] = countObj.(int64) + } + desc.Tenants = rets + + return nil +} + +func (b *HostBuilder) fillResidentGroups(desc *HostDesc, host *models.Host) error { + groups, ok := b.hostGroupCountDict[host.ID] + if !ok { + desc.Groups = nil + return nil + } + desc.Groups = groups + return nil +} + +func (b *HostBuilder) fillMetadata(desc *HostDesc, host *models.Host) error { + metadataObjs, ok := b.hostMetadatasDict[host.ID] + if !ok { + return nil + } + for _, obj := range metadataObjs { + metadata, ok := obj.(*models.Metadata) + if !ok { + return utils.ConvertError(obj, "*models.Metadata") + } + desc.Metadata[metadata.Key] = metadata.Value + } + return nil +} + +type IsolatedDeviceDesc struct { + ID string + GuestID string + HostID string + DevType string + Model string + Addr string + VendorDeviceID string +} + +func (i *IsolatedDeviceDesc) VendorID() string { + return strings.Split(i.VendorDeviceID, ":")[0] +} + +func (i *IsolatedDeviceDesc) VendorModel() string { + return fmt.Sprintf("%s:%s", i.VendorID(), i.Model) +} + +func (b *HostBuilder) getIsolatedDevices(hostID string) (devs []*models.IsolatedDevice) { + devObjs, ok := b.isolatedDevicesDict[hostID] + devs = make([]*models.IsolatedDevice, 0) + if !ok { + return + } + for _, obj := range devObjs { + dev := obj.(*models.IsolatedDevice) + devs = append(devs, dev) + } + return +} + +func (b *HostBuilder) getUsedIsolatedDevices(hostID string) (devs []*models.IsolatedDevice) { + devs = make([]*models.IsolatedDevice, 0) + for _, dev := range b.getIsolatedDevices(hostID) { + if len(dev.GuestID) != 0 { + devs = append(devs, dev) + } + } + return +} + +func (b *HostBuilder) getIsolatedDeviceGuests(hostID string) (guests []*models.Guest) { + guests = make([]*models.Guest, 0) + usedDevs := b.getUsedIsolatedDevices(hostID) + if len(usedDevs) == 0 { + return + } + ids := sets.NewString() + for _, dev := range usedDevs { + g, ok := b.guestDict[dev.GuestID] + if !ok { + continue + } + guest := g.(*models.Guest) + if !ids.Has(guest.ID) { + ids.Insert(guest.ID) + guests = append(guests, guest) + } + } + return +} + +func (b *HostBuilder) getUnusedIsolatedDevices(hostID string) (devs []*models.IsolatedDevice) { + devs = make([]*models.IsolatedDevice, 0) + for _, dev := range b.getIsolatedDevices(hostID) { + if len(dev.GuestID) == 0 { + devs = append(devs, dev) + } + } + return +} + +func (b *HostBuilder) fillIsolatedDevices(desc *HostDesc, host *models.Host) error { + + allDevs := b.getIsolatedDevices(host.ID) + if len(allDevs) == 0 { + return nil + } + + devs := make([]*IsolatedDeviceDesc, len(allDevs)) + for index, devModel := range allDevs { + dev := &IsolatedDeviceDesc{ + ID: devModel.ID, + GuestID: devModel.GuestID, + HostID: devModel.HostID, + DevType: devModel.DevType, + Model: devModel.Model, + Addr: devModel.Addr, + VendorDeviceID: devModel.VendorDeviceID, + } + devs[index] = dev + } + desc.IsolatedDevices = devs + + return nil +} + +func (b *HostBuilder) fillCPUIOLoads(desc *HostDesc, host *models.Host) error { + desc.CPULoad = b.loadByName(host.ID, "cpu_load") + desc.IOLoad = b.loadByName(host.ID, "io_load") + return nil +} + +func (b *HostBuilder) loadByName(hostID, name string) *float64 { + if b.cpuIOLoads == nil { + return nil + } + loads, ok := b.cpuIOLoads[hostID] + if !ok { + return nil + } + value := loads[name] + if value >= 0.0 && value <= 1.0 { + return &value + } + return nil +} diff --git a/pkg/scheduler/cache/candidate/interface.go b/pkg/scheduler/cache/candidate/interface.go new file mode 100644 index 0000000000..b725620f10 --- /dev/null +++ b/pkg/scheduler/cache/candidate/interface.go @@ -0,0 +1,28 @@ +package candidate + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/cache" +) + +type groupCacher interface { + Get(string) (cache.Cache, error) +} + +type DBGroupCacher interface { + groupCacher +} + +type SyncGroupCacher interface { + groupCacher +} + +type descer interface { + UUID() string +} + +type BuildActor interface { + Clone() BuildActor + Type() string + AllIDs() ([]string, error) + Do(ids []string, db DBGroupCacher, sync SyncGroupCacher) ([]interface{}, error) +} diff --git a/pkg/scheduler/cache/db/db.go b/pkg/scheduler/cache/db/db.go new file mode 100644 index 0000000000..88b5e90db9 --- /dev/null +++ b/pkg/scheduler/cache/db/db.go @@ -0,0 +1,14 @@ +package db + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/cache" +) + +type dbItem struct { + cache.CachedItem + fields []string +} + +func NewCacheManager(stopCh <-chan struct{}) *cache.GroupManager { + return cache.NewGroupManager(CacheKind, DefaultCachedItems(), stopCh) +} diff --git a/pkg/scheduler/cache/db/default.go b/pkg/scheduler/cache/db/default.go new file mode 100644 index 0000000000..ecbce0e495 --- /dev/null +++ b/pkg/scheduler/cache/db/default.go @@ -0,0 +1,133 @@ +package db + +import ( + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + u "github.com/yunionio/pkg/utils" +) + +const ( + CacheKind = "DBCache" + + StorageDBCache = "Storages" + WireDBCache = "Wires" + + GroupDBCache = "Groups" + GroupGuestDBCache = "Groupguests" + GroupHostDBCache = "Grouphosts" + HostDBCache = "Hosts" + + ClusterDBCache = "Clusters" + ClusterHostDBCache = "Clusterhosts" + HostWireDBCache = "Hostwires" + + AggregateDBCache = "Aggregates" + AggregateHostDBCache = "AggregateHosts" + AggregateBaremetalDBCache = "AggregateBaremetals" + + BaremetalAgentDBCache = "BaremetalAgents" + + NetworksDBCache = "Networks" + NetInterfacesDBCache = "NetInterfaces" + WiresDBCache = "Wires" +) + +func getUpdate(d []interface{}) ([]string, error) { + return nil, nil +} + +func DefaultCachedItems() []cache.CachedItem { + if !models.DBValid() { + panic("DB not init before cache items") + } + return []cache.CachedItem{ + newClusterDBCache(), + newBaremetalAgentDBCache(), + newAggregateDBCache(), + newAggregateHostDBCache(), + } +} + +func NewCachedItems(items []string) (cachedItems []cache.CachedItem) { + if !models.DBValid() { + panic("DB not init before cache items") + } + for _, item := range items { + switch item { + case NetworksDBCache: + cachedItems = append(cachedItems, newNetworksDBCache()) + case NetInterfacesDBCache: + cachedItems = append(cachedItems, newNetInterfacesDBCache()) + case WiresDBCache: + cachedItems = append(cachedItems, newWiresDBCache()) + default: + return nil + } + } + return +} + +func uuidKey(obj interface{}) (string, error) { + return obj.(models.Modeler).UUID(), nil +} + +func newDBCache(name string, r models.Resourcer, ttl, period time.Duration) cache.CachedItem { + update := func(ids []string) ([]interface{}, error) { + return models.FetchByIDs(r, ids) + } + + load := func() ([]interface{}, error) { + return models.All(r) + } + + item := new(dbItem) + item.CachedItem = cache.NewCacheItem( + name, ttl, period, uuidKey, update, load, getUpdate, + ) + return item +} + +func newClusterDBCache() cache.CachedItem { + return newDBCache(ClusterDBCache, models.Clusters, + u.ToDuration(o.GetOptions().ClusterDBCacheTTL), + u.ToDuration(o.GetOptions().ClusterDBCachePeriod)) +} + +func newBaremetalAgentDBCache() cache.CachedItem { + return newDBCache(BaremetalAgentDBCache, models.BaremetalAgents, + u.ToDuration(o.GetOptions().BaremetalAgentDBCacheTTL), + u.ToDuration(o.GetOptions().BaremetalAgentDBCachePeriod)) +} + +func newAggregateDBCache() cache.CachedItem { + return newDBCache(AggregateDBCache, models.Aggregates, + u.ToDuration(o.GetOptions().AggregateDBCacheTTL), + u.ToDuration(o.GetOptions().AggregateDBCachePeriod)) +} + +func newAggregateHostDBCache() cache.CachedItem { + return newDBCache(AggregateHostDBCache, models.AggregateHosts, + u.ToDuration(o.GetOptions().AggregateHostDBCacheTTL), + u.ToDuration(o.GetOptions().AggregateHostDBCachePeriod)) +} + +func newNetworksDBCache() cache.CachedItem { + return newDBCache(NetworksDBCache, models.Networks, + u.ToDuration(o.GetOptions().NetworksDBCacheTTL), + u.ToDuration(o.GetOptions().NetworksDBCachePeriod)) +} + +func newNetInterfacesDBCache() cache.CachedItem { + return newDBCache(AggregateHostDBCache, models.NetInterfaces, + u.ToDuration(o.GetOptions().NetinterfaceDBCacheTTL), + u.ToDuration(o.GetOptions().NetinterfaceDBCachePeriod)) +} + +func newWiresDBCache() cache.CachedItem { + return newDBCache(AggregateHostDBCache, models.Wires, + u.ToDuration(o.GetOptions().WireDBCacheTTL), + u.ToDuration(o.GetOptions().WireDBCachePeriod)) +} diff --git a/pkg/scheduler/cache/group_manager.go b/pkg/scheduler/cache/group_manager.go new file mode 100644 index 0000000000..30eb1d102c --- /dev/null +++ b/pkg/scheduler/cache/group_manager.go @@ -0,0 +1,44 @@ +package cache + +import ( + "fmt" + + "github.com/yunionio/log" +) + +type GroupManager struct { + Group map[string]Cache + stopCh <-chan struct{} +} + +func NewGroupManager(kind string, items []CachedItem, ch <-chan struct{}) *GroupManager { + man := new(GroupManager) + man.Group = make(map[string]Cache) + man.stopCh = ch + for _, item := range items { + man.Group[item.Name()] = NewCache(kind, item) + } + return man +} + +func (m *GroupManager) Run() { + go func() { + m.run() + select {} + }() +} + +func (m *GroupManager) run() { + for name, c := range m.Group { + log.V(3).Infof("Start cache: %v", name) + c.Start(m.stopCh) + } +} + +func (m *GroupManager) Get(name string) (Cache, error) { + entity, ok := m.Group[name] + if !ok { + return nil, fmt.Errorf("DB cache item: %s not found", name) + } + return entity, nil +} diff --git a/pkg/scheduler/cache/interface.go b/pkg/scheduler/cache/interface.go new file mode 100644 index 0000000000..d557a9370f --- /dev/null +++ b/pkg/scheduler/cache/interface.go @@ -0,0 +1,33 @@ +package cache + +import ( + "time" +) + +type CacheGroup interface { + Run() + Get(string) (Cache, error) +} + +type Cache interface { + Add(obj interface{}) error + Update(obj interface{}) error + Delete(obj interface{}) error + List() []interface{} + Get(string) (item interface{}, err error) + Start(<-chan struct{}) + + Reload(keys []string) (items []interface{}, err error) + ReloadAll() (items []interface{}, err error) + WaitForReady() +} + +type CachedItem interface { + TTL() time.Duration + Name() string + Period() time.Duration + Update(keys []string) ([]interface{}, error) + Load() ([]interface{}, error) + Key(obj interface{}) (string, error) + GetUpdate(d []interface{}) ([]string, error) +} diff --git a/pkg/scheduler/cache/item.go b/pkg/scheduler/cache/item.go new file mode 100644 index 0000000000..9213ff6fb7 --- /dev/null +++ b/pkg/scheduler/cache/item.go @@ -0,0 +1,74 @@ +package cache + +import ( + "time" + + expirationcache "github.com/yunionio/pkg/util/cache" +) + +type UpdateFunc func([]string) ([]interface{}, error) +type LoadFunc func() ([]interface{}, error) +type GetUpdateFunc func(d []interface{}) ([]string, error) + +// Item implement CachedItem interface +type Item struct { + // Name of this cache item + name string + // Time to live duration + ttl time.Duration + // Load all objects period + period time.Duration + // Function to get index like id or name of this cache item + keyFunc expirationcache.KeyFunc + // Function to update range of cache item by their key + update UpdateFunc + // Function to load all cache items + load LoadFunc + // Function to get item must be updated + getUpdate GetUpdateFunc +} + +// NewCacheItem new a Item implement CachedItem interface +func NewCacheItem(name string, ttl, period time.Duration, + keyf expirationcache.KeyFunc, + update UpdateFunc, load LoadFunc, + getUpdate GetUpdateFunc, +) CachedItem { + return &Item{ + name: name, + ttl: ttl, + period: period, + keyFunc: keyf, + update: update, + load: load, + getUpdate: getUpdate, + } +} + +func (h *Item) TTL() time.Duration { + return h.ttl +} + +func (h *Item) Key(obj interface{}) (string, error) { + return h.keyFunc(obj) +} + +func (h *Item) Name() string { + return h.name +} + +func (h *Item) Period() time.Duration { + return h.period +} + +func (h *Item) Update(ids []string) ([]interface{}, error) { + return h.update(ids) +} + +func (h *Item) Load() ([]interface{}, error) { + return h.load() +} + +func (h *Item) GetUpdate(d []interface{}) ([]string, error) { + return h.getUpdate(d) +} diff --git a/pkg/scheduler/cache/sync/default.go b/pkg/scheduler/cache/sync/default.go new file mode 100644 index 0000000000..a71a5423ba --- /dev/null +++ b/pkg/scheduler/cache/sync/default.go @@ -0,0 +1,80 @@ +package sync + +import ( + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + networks_db "github.com/yunionio/onecloud/pkg/scheduler/cache/sync/networks/db" + u "github.com/yunionio/pkg/utils" +) + +const ( + CacheKind = "SyncCache" + + //GlanceSyncCache = "Glance" + NetworkSyncCache = "Network" + NetworksDataSyncCache = "NetworkData" +) + +func getUpdate(d []interface{}) ([]string, error) { + return nil, nil +} + +func defaultSyncItems() []cache.CachedItem { + return []cache.CachedItem{ + //newGlanceCache(), + //newNetworkCache(), + //newNetworksDataCache(), + } +} + +func noneUpdate(id []string) ([]interface{}, error) { + return []interface{}{}, nil +} + +/* +func newGlanceCache() cache.CachedItem { + item := new(syncItem) + + item.CachedItem = cache.NewCacheItem( + GlanceSyncCache, + viper.GetDuration("cache.glance_cache.ttl"), + viper.GetDuration("cache.glance_cache.period"), + imageUUIDKey, + noneUpdate, + loadImages, + getUpdate, + ) + return item +}*/ + +func newNetworkCache() cache.CachedItem { + item := new(syncItem) + + // data from db + item.CachedItem = cache.NewCacheItem( + NetworkSyncCache, + u.ToDuration(o.GetOptions().NetworkCacheTTL), + u.ToDuration(o.GetOptions().NetworkCachePeriod), + networks_db.BuilderCacheKey, + networks_db.UpdateNetworkDescBuilder, + networks_db.LoadNetworkDescBuilder, + getUpdate, + ) + + return item +} + +func newNetworksDataCache() cache.CachedItem { + item := new(syncItem) + + item.CachedItem = cache.NewCacheItem( + NetworksDataSyncCache, + u.ToDuration(o.GetOptions().NetworkCacheTTL), + u.ToDuration(o.GetOptions().NetworkCachePeriod), + BuilderNetworkCacheKey, + updateNetworksBuilder, + loadNetworksBuilder, + getUpdate, + ) + return item +} diff --git a/pkg/scheduler/cache/sync/network.go b/pkg/scheduler/cache/sync/network.go new file mode 100644 index 0000000000..4d532837c5 --- /dev/null +++ b/pkg/scheduler/cache/sync/network.go @@ -0,0 +1,33 @@ +package sync + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + "github.com/yunionio/pkg/utils" +) + +const ( + HostNetworkDescBuilderCache = "HostNetworkDescBuilderCache" + BaremetalNetworkDescBuilderCache = "BaremetalNetworkDescBuilderCache" +) + +func avaliableAddress(network *models.WireNetwork) (int, error) { + totalAddress := utils.IpRangeCount(network.GuestIpStart, network.GuestIpEnd) + guestNicCount, err := models.GuestNicCountsWithNetworkID(network.ID) + if err != nil { + return 0, err + } + groupNicCount, err := models.GroupNicCountsWithNetworkID(network.ID) + if err != nil { + return 0, err + } + baremetalNicCount, err := models.BaremetalNicCountsWithNetworkID(network.ID) + if err != nil { + return 0, err + } + reserveNicCount, err := models.ReserveNicCountsWithNetworkID(network.ID) + if err != nil { + return 0, err + } + + return totalAddress - guestNicCount.Count - groupNicCount.Count - baremetalNicCount.Count - reserveNicCount.Count, nil +} diff --git a/pkg/scheduler/cache/sync/network_data.go b/pkg/scheduler/cache/sync/network_data.go new file mode 100644 index 0000000000..ded69a8ed2 --- /dev/null +++ b/pkg/scheduler/cache/sync/network_data.go @@ -0,0 +1,245 @@ +package sync + +import ( + "fmt" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + "github.com/yunionio/pkg/utils" +) + +const ( + NetworksBuilderCache = "NetworksBuilderCache" + + GuestNiCount = "GuestNiCount" + GroupNicCount = "GroupNicCount" + BaremetalNicCount = "BaremetalNicCount" + ReserveDipNicCount = "ReserveDipNicCount" +) + +type SchedNetworkBuildResult struct { + ID string `json:"id"` + Name string `json:"name"` + TenantID string `json:"tenant_id"` + IsPublic bool `json:"is_public"` + ServerType string `json:"server_type"` + Ports int `json:"ports"` + IsExit bool `json:"is_exit"` + Wire string `json:"wire_name"` + WireID string `json:"wire_id"` +} + +func BuilderNetworkCacheKey(obj interface{}) (string, error) { + builder, ok := obj.(NetworksBuilder) + if !ok { + return "", fmt.Errorf("Not a NetworksDataSyncCache: %v", obj) + } + + return builder.GetKey(), nil +} + +func updateNetworksBuilder(keys []string) ([]interface{}, error) { + builders := make([]NetworksBuilder, 0) + + for _, key := range keys { + switch key { + case NetworksBuilderCache: + builders = append(builders, NewNetworksBuilder()) + default: + return nil, fmt.Errorf("Not support update, key: %v", key) + } + } + + ret := []interface{}{} + for _, builder := range builders { + _, err := builder.LoadAll() + if err != nil { + log.Errorf("Network load error: %v", err) + } + ret = append(ret, builder) + } + + return ret, nil +} + +type NetworksBuilder interface { + LoadAll() (map[string]*SchedNetworkBuildResult, error) + Load(ids []string) (map[string]*SchedNetworkBuildResult, error) + GetKey() string + GetNetworksData(ids []string) []*SchedNetworkBuildResult +} + +func loadNetworksBuilder() ([]interface{}, error) { + builders := []NetworksBuilder{ + NewNetworksBuilder(), + } + + rets := []interface{}{} + + for _, builder := range builders { + _, err := builder.LoadAll() + if err != nil { + log.Errorf("Network load error: %v", err) + } else { + rets = append(rets, builder) + } + } + + return rets, nil +} + +type NetworksDataBuilder struct { + GuestNicCount map[string]int + GroupNicCount map[string]int + BaremetalNicCount map[string]int + ReserveDipNicCount map[string]int + Wires map[string]string + data map[string]*SchedNetworkBuildResult +} + +func NewNetworksBuilder() *NetworksDataBuilder { + return &NetworksDataBuilder{ + GuestNicCount: make(map[string]int), + GroupNicCount: make(map[string]int), + BaremetalNicCount: make(map[string]int), + ReserveDipNicCount: make(map[string]int), + data: make(map[string]*SchedNetworkBuildResult), + } +} + +func (b *NetworksDataBuilder) GetKey() string { + return NetworksBuilderCache +} + +func (b *NetworksDataBuilder) LoadAll() (map[string]*SchedNetworkBuildResult, error) { + return b.Load(nil) +} + +func (b *NetworksDataBuilder) toSchedNetworkBuildResult(network *models.Network) (*SchedNetworkBuildResult, error) { + if network == nil { + return nil, fmt.Errorf("empty network resource.") + } + + res := new(SchedNetworkBuildResult) + res.WireID = network.WireID + res.Wire = b.Wires[network.WireID] + res.ID = network.ID + ports, err := b.avaliableAddress(network) + if err != nil { + return nil, err + } else { + res.Ports = ports + } + res.Name = network.Name + res.TenantID = network.TenantID + res.IsPublic = network.IsPublic == 1 + res.ServerType = network.ServerType + res.IsExit = utils.IsExitAddress(network.GuestIpStart) + + return res, nil +} + +func (b *NetworksDataBuilder) avaliableAddress(network *models.Network) (int, error) { + totalAddress := utils.IpRangeCount(network.GuestIpStart, network.GuestIpEnd) + + return totalAddress - b.GuestNicCount[network.ID] - b.GroupNicCount[network.ID] - b.BaremetalNicCount[network.ID] - b.ReserveDipNicCount[network.ID], nil +} + +func getNiCount(nicName string) (map[string]int, error) { + countsMap := make(map[string]int) + switch nicName { + case GuestNiCount: + counts, err := models.GuestNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + case GroupNicCount: + counts, err := models.GroupNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + + case BaremetalNicCount: + counts, err := models.BaremetalNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + + case ReserveDipNicCount: + counts, err := models.ReserveNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + } + + return countsMap, nil +} + +func (b *NetworksDataBuilder) Load(ids []string) (map[string]*SchedNetworkBuildResult, error) { + wireInfos, err := models.LoadAllWires() + if err != nil { + return nil, err + } + wiresMap := make(map[string]string, len(wireInfos)) + for _, wire := range wireInfos { + wiresMap[wire.ID] = wire.Name + } + b.Wires = wiresMap + + b.GuestNicCount, err = getNiCount(GuestNiCount) + if err != nil { + log.Errorln(err) + } + b.GroupNicCount, err = getNiCount(GroupNicCount) + if err != nil { + log.Errorln(err) + } + b.BaremetalNicCount, err = getNiCount(BaremetalNicCount) + if err != nil { + log.Errorln(err) + } + b.ReserveDipNicCount, err = getNiCount(ReserveDipNicCount) + if err != nil { + log.Errorln(err) + } + + networks, err := models.All(models.Networks) + if err != nil { + return nil, err + } + + for _, network := range networks { + n := network.(*models.Network) + b.data[n.ID], err = b.toSchedNetworkBuildResult(n) + if err != nil { + log.Errorln(err) + } + } + + return b.data, nil +} + +func (b *NetworksDataBuilder) GetNetworksData(ids []string) (networks []*SchedNetworkBuildResult) { + for _, networkID := range ids { + if network, ok := b.data[networkID]; ok { + networks = append(networks, network) + } + } + return +} diff --git a/pkg/scheduler/cache/sync/networks/db/db.go b/pkg/scheduler/cache/sync/networks/db/db.go new file mode 100644 index 0000000000..b9ea7b2e69 --- /dev/null +++ b/pkg/scheduler/cache/sync/networks/db/db.go @@ -0,0 +1,293 @@ +package db + +import ( + "fmt" + "strings" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" +) + +const ( + HostNetworkDescBuilderCache = "HostNetworkDescBuilderCache" + BaremetalNetworkDescBuilderCache = "BaremetalNetworkDescBuilderCache" +) + +type NetworkDescBuilder interface { + LoadAll() (map[string][]string, error) + Load(ids []string) (map[string][]string, error) + GetKey() string + GetNetworkDesc(id string) ([]string, error) +} + +func BuilderCacheKey(obj interface{}) (string, error) { + builder, ok := obj.(NetworkDescBuilder) + if !ok { + return "", fmt.Errorf("Not a NetworkDescBuilder: %v", obj) + } + + return builder.GetKey(), nil +} + +func LoadNetworkDescBuilder() ([]interface{}, error) { + builders := []NetworkDescBuilder{ + NewHostNetworkDescBuilder(), + //NewBaremetalNetworkDescBuilder(), + } + + rets := []interface{}{} + + for _, builder := range builders { + _, err := builder.LoadAll() + if err != nil { + log.Errorf("Network load error: %v", err) + } else { + rets = append(rets, builder) + } + } + + return rets, nil +} + +func UpdateNetworkDescBuilder(keys []string) ([]interface{}, error) { + builders := make([]NetworkDescBuilder, 0) + + for _, key := range keys { + switch key { + case HostNetworkDescBuilderCache: + builders = append(builders, NewHostNetworkDescBuilder()) + case BaremetalNetworkDescBuilderCache: + builders = append(builders, NewBaremetalNetworkDescBuilder()) + default: + return nil, fmt.Errorf("Not support update, key: %v", key) + } + } + + ret := []interface{}{} + for _, builder := range builders { + _, err := builder.LoadAll() + if err != nil { + log.Errorf("Network load error: %v", err) + } + ret = append(ret, builder) + } + + return ret, nil +} + +type HostNetworkDescBuilder struct { + data map[string][]string + host2Wires map[string]string + wire2Networks map[string]string +} + +func NewHostNetworkDescBuilder() *HostNetworkDescBuilder { + return &HostNetworkDescBuilder{ + data: make(map[string][]string), + host2Wires: make(map[string]string), + wire2Networks: make(map[string]string), + } +} + +func (b *HostNetworkDescBuilder) GetKey() string { + return HostNetworkDescBuilderCache +} + +func (b *HostNetworkDescBuilder) LoadAll() (map[string][]string, error) { + return b.Load(nil) +} + +func (b *HostNetworkDescBuilder) Load(ids []string) (map[string][]string, error) { + // wireInfos, err := models.LoadAllWires() + // if err != nil { + // return nil, err + // } + // wiresMap := make(map[string]string, len(wireInfos)) + // for _, wire := range wireInfos { + // wiresMap[wire.ID] = wire.Name + // } + + hostAndWires, err := models.SelectHostHasWires() + if err != nil { + return nil, err + } + hostHasWires := make(map[string]string) + for _, hostAndWire := range hostAndWires { + if _, ok := hostHasWires[hostAndWire.HostID]; ok { + if !strings.Contains(hostHasWires[hostAndWire.HostID], hostAndWire.WireID) { + hostHasWires[hostAndWire.HostID] = fmt.Sprintf("%s;%s", hostHasWires[hostAndWire.HostID], hostAndWire.WireID) + } + } else { + hostHasWires[hostAndWire.HostID] = hostAndWire.WireID + } + } + b.host2Wires = hostHasWires + + wiresAndNetworks, err := models.SelectWireIDsHasNetworks() + if err != nil { + return nil, err + } + wireHasNetworks := make(map[string]string) + for _, wireAndNetwork := range wiresAndNetworks { + if _, ok := wireHasNetworks[wireAndNetwork.WireID]; ok { + if !strings.Contains(wireHasNetworks[wireAndNetwork.WireID], wireAndNetwork.ID) { + wireHasNetworks[wireAndNetwork.WireID] = fmt.Sprintf("%s;%s", wireHasNetworks[wireAndNetwork.WireID], wireAndNetwork.ID) + } + } else { + wireHasNetworks[wireAndNetwork.WireID] = wireAndNetwork.ID + } + } + b.wire2Networks = wireHasNetworks + + if len(ids) == 0 { + hostIDs, err := models.AllHostIDs() + if err != nil { + log.Errorln(err) + } + for _, hostID := range hostIDs { + networkResults, err := b.loadNetworks(hostID) + if err != nil { + log.Errorln(err) + } else { + b.data[hostID] = networkResults + } + } + } else { + for _, hostID := range ids { + networkResults, err := b.loadNetworks(hostID) + if err != nil { + log.Errorln(err) + } else { + b.data[hostID] = networkResults + } + } + } + + return b.data, nil +} + +func (b *HostNetworkDescBuilder) loadNetworks(hostID string) (networkResults []string, err error) { + wires := strings.Split(b.host2Wires[hostID], ";") + for _, wire := range wires { + networkResults = append(networkResults, strings.Split(b.wire2Networks[wire], ";")...) + } + return networkResults, nil +} + +func (b *HostNetworkDescBuilder) GetNetworkDesc(id string) ([]string, error) { + if r, ok := b.data[id]; ok { + return r, nil + } + + return nil, fmt.Errorf("can not find networks") +} + +type BaremetalNetworkDescBuilder struct { + data map[string][]string + baremetal2Wires map[string]string + wire2Networks map[string]string +} + +// TODO:we should not new a object every time, the map memory leak, if want map +// was GCed, you must set b.data = nil and so on. +func NewBaremetalNetworkDescBuilder() *BaremetalNetworkDescBuilder { + return &BaremetalNetworkDescBuilder{ + data: make(map[string][]string, 30000), // the max number of baremetal if about 27000. + baremetal2Wires: make(map[string]string, 30000), + wire2Networks: make(map[string]string, 0), + } +} + +func (b *BaremetalNetworkDescBuilder) GetKey() string { + return BaremetalNetworkDescBuilderCache +} + +func (b *BaremetalNetworkDescBuilder) LoadAll() (map[string][]string, error) { + return b.Load(nil) +} + +func (b *BaremetalNetworkDescBuilder) Load(ids []string) (map[string][]string, error) { + // wireInfos, err := models.LoadAllWires() + // if err != nil { + // return nil, err + // } + // wiresMap := make(map[string]string, len(wireInfos)) + // for _, wire := range wireInfos { + // wiresMap[wire.ID] = wire.Name + // } + + baremetalsAndWires, err := models.SelectWiresAndBaremetals() + if err != nil { + return nil, err + } + baremetalHasWires := make(map[string]string) + for _, baremetalsAndWire := range baremetalsAndWires { + if _, ok := baremetalHasWires[baremetalsAndWire.BaremetalID]; ok { + if !strings.Contains(baremetalHasWires[baremetalsAndWire.BaremetalID], baremetalsAndWire.WireID) { + baremetalHasWires[baremetalsAndWire.BaremetalID] = fmt.Sprintf("%s;%s", baremetalHasWires[baremetalsAndWire.BaremetalID], baremetalsAndWire.WireID) + } + } else { + baremetalHasWires[baremetalsAndWire.BaremetalID] = baremetalsAndWire.WireID + } + } + b.baremetal2Wires = baremetalHasWires + + wiresAndNetworks, err := models.SelectWireIDsHasNetworks() + if err != nil { + return nil, err + } + wireHasNetworks := make(map[string]string) + for _, wireAndNetwork := range wiresAndNetworks { + if _, ok := wireHasNetworks[wireAndNetwork.WireID]; ok { + if !strings.Contains(wireHasNetworks[wireAndNetwork.WireID], wireAndNetwork.ID) { + wireHasNetworks[wireAndNetwork.WireID] = fmt.Sprintf("%s;%s", wireHasNetworks[wireAndNetwork.WireID], wireAndNetwork.ID) + } + } else { + wireHasNetworks[wireAndNetwork.WireID] = wireAndNetwork.ID + } + } + b.wire2Networks = wireHasNetworks + + if len(ids) == 0 { + baremetalIDs, err := models.AllBaremetalIDs() + if err != nil { + log.Errorln(err) + } + + for _, baremetalID := range baremetalIDs { + networkResults, err := b.loadNetworks(baremetalID) + if err != nil { + log.Errorln(err) + } else { + b.data[baremetalID] = networkResults + } + } + } else { + for _, baremetalID := range ids { + networkResults, err := b.loadNetworks(baremetalID) + if err != nil { + log.Errorln(err) + } else { + b.data[baremetalID] = networkResults + } + } + } + + return b.data, nil +} + +func (b *BaremetalNetworkDescBuilder) loadNetworks(baremetalID string) (networkResults []string, err error) { + wires := strings.Split(b.baremetal2Wires[baremetalID], ";") + for _, wire := range wires { + networkResults = append(networkResults, strings.Split(b.wire2Networks[wire], ";")...) + } + return networkResults, nil +} + +func (b *BaremetalNetworkDescBuilder) GetNetworkDesc(id string) ([]string, error) { + if result, ok := b.data[id]; ok { + return result, nil + } + + return nil, fmt.Errorf("can not find networks") +} diff --git a/pkg/scheduler/cache/sync/sync.go b/pkg/scheduler/cache/sync/sync.go new file mode 100644 index 0000000000..254bf1248a --- /dev/null +++ b/pkg/scheduler/cache/sync/sync.go @@ -0,0 +1,14 @@ +package sync + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/cache" +) + +func NewSyncManager(stopCh <-chan struct{}) *cache.GroupManager { + items := defaultSyncItems() + return cache.NewGroupManager(CacheKind, items, stopCh) +} + +type syncItem struct { + cache.CachedItem +} diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go new file mode 100644 index 0000000000..5943bef592 --- /dev/null +++ b/pkg/scheduler/core/context.go @@ -0,0 +1,615 @@ +package core + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" +) + +const ( + EmptyScore int = 0x7FFFFFFFFFFFFFFF + BaseScore int = 10000 + EmptyCapacity int64 = -1 + MaxCapacity int64 = 0x7FFFFFFFFFFFFFFF +) + +var ( + EmptyScores = make(map[string]int) + EmptyCapacities = make(map[string]Counter) +) + +type SharedResourceManager struct { + resourceMap map[string]Counter + lock sync.Mutex +} + +func NewSharedResourceManager() *SharedResourceManager { + return &SharedResourceManager{ + lock: sync.Mutex{}, + resourceMap: make(map[string]Counter), + } +} + +func (m *SharedResourceManager) Add(resourceKey string, capacity Counter) { + m.lock.Lock() + defer m.lock.Unlock() + + m.resourceMap[resourceKey] = capacity +} + +type CounterManager struct { + Counters map[string]Counter + lock sync.Mutex +} + +func NewCounterManager() *CounterManager { + return &CounterManager{ + Counters: make(map[string]Counter), + lock: sync.Mutex{}, + } +} + +func (m *CounterManager) Get(key string) Counter { + m.lock.Lock() + defer m.lock.Unlock() + + if counter, ok := m.Counters[key]; ok { + return counter + } + + return nil +} + +func (m *CounterManager) GetOrCreate(key string, creator func() Counter) Counter { + m.lock.Lock() + defer m.lock.Unlock() + + if counter, ok := m.Counters[key]; ok { + return counter + } + + counter := creator() + if counter == nil { + return nil + } + + m.Counters[key] = counter + return counter +} + +type Counter interface { + GetCount() int64 +} + +type MultiCounter interface { + Counter + Add(counter Counter) +} + +type NormalCounter struct { + Value int64 +} + +func NewNormalCounter(value int64) *NormalCounter { + return &NormalCounter{ + Value: value, + } +} + +func (c *NormalCounter) GetCount() int64 { + return c.Value +} + +type Counters struct { + counters []Counter + lock sync.Mutex + sum int64 +} + +func NewCounters() *Counters { + return &Counters{ + sum: EmptyCapacity, + lock: sync.Mutex{}, + } +} + +func (c *Counters) Add(cnt Counter) { + c.lock.Lock() + defer c.lock.Unlock() + + c.counters = append(c.counters, cnt) + c.sum = EmptyCapacity +} + +func (c *Counters) GetCount() int64 { + if c.sum == EmptyCapacity { + c.sum = c.calculateCount() + } + return c.sum +} + +func (c *Counters) calculateCount() int64 { + if len(c.counters) == 0 { + return 0 + } + + c.lock.Lock() + defer c.lock.Unlock() + + value := int64(0) + for _, c := range c.counters { + count := c.GetCount() + if count != EmptyCapacity { + value += c.GetCount() + } + } + return value +} + +type MinCounters struct { + counters []Counter +} + +func NewMinCounters() *MinCounters { + return &MinCounters{} +} + +func (c *MinCounters) Add(counter Counter) { + c.counters = append(c.counters, counter) +} + +func (c *MinCounters) GetCount() int64 { + n := EmptyCapacity + for _, c0 := range c.counters { + count := c0.GetCount() + if count != EmptyCapacity && count < n { + n = count + } + } + + return n +} + +type Capacity struct { + Values map[string]Counter + MinValue int64 +} + +type Score struct { + Values map[string]int + Sum int +} + +type SchedContextDataItem struct { + Networks sync.Map + Data map[string]interface{} +} + +type SchedLog struct { + Candidate string + Action string + Message string +} + +func NewSchedLog(candidate, action, message string) SchedLog { + return SchedLog{candidate, action, message} +} + +func (log *SchedLog) String() string { + return fmt.Sprintf("%v [%v] %v", log.Candidate, log.Action, log.Message) +} + +type SchedLogList []SchedLog + +func (logList SchedLogList) Len() int { + return len(logList) +} + +func (logList SchedLogList) Less(i, j int) bool { + r := strings.Compare(logList[i].Candidate, logList[j].Candidate) + if r != 0 { + return r < 0 + } + + r = strings.Compare(logList[i].Message, logList[j].Message) + if r != 0 { + return r < 0 + } + + return strings.Compare(logList[i].Action, logList[j].Action) < 0 +} + +func (logList SchedLogList) Swap(i, j int) { + logList[i], logList[j] = logList[j], logList[i] +} + +type SchedLogManager struct { + Logs SchedLogList + lock sync.Mutex + sorted bool +} + +func NewSchedLogManager() *SchedLogManager { + return &SchedLogManager{ + lock: sync.Mutex{}, + Logs: SchedLogList{}, + } +} + +func (m *SchedLogManager) Append(candidate, action, message string) { + m.lock.Lock() + defer m.lock.Unlock() + + m.Logs = append(m.Logs, NewSchedLog(candidate, action, message)) +} + +func (m *SchedLogManager) Appends(logs []SchedLog) { + m.lock.Lock() + defer m.lock.Unlock() + + m.Logs = append(m.Logs, logs...) +} + +func (m *SchedLogManager) Read() []string { + rets := []string{} + + m.lock.Lock() + defer m.lock.Unlock() + + if len(m.Logs) == 0 { + return rets + } + + if !m.sorted { + sort.Sort(m.Logs) + m.sorted = true + } + + joinLogs := func(startIndex, endIndex int) string { + if endIndex == startIndex+1 { + return m.Logs[startIndex].String() + } + + log := m.Logs[startIndex] + actions := []string{} + for ; startIndex < endIndex; startIndex++ { + actions = append(actions, m.Logs[startIndex].Action) + } + + newLog := SchedLog{log.Candidate, strings.Join(actions, ","), log.Message} + return newLog.String() + } + + startIndex := -1 + for index, len := 0, len(m.Logs); index < len; index++ { + if startIndex < 0 { + startIndex = index + } else { + log0, log := m.Logs[startIndex], m.Logs[index] + if log0.Candidate != log.Candidate || log0.Message != log.Message { + rets = append(rets, joinLogs(startIndex, index)) + startIndex = index + } + } + } + + rets = append(rets, joinLogs(startIndex, len(m.Logs))) + return rets +} + +// Unit wraps sched input info and other log and record manager +type Unit struct { + SchedInfo *api.SchedInfo + CapacityMap map[string]*Capacity + ScoreMap map[string]Score + DataMap map[string]*SchedContextDataItem + SharedResourceManager *SharedResourceManager + CounterManager *CounterManager + + capacityLock sync.Mutex + scoreLock sync.Mutex + + FailedCandidateMap map[string]*FailedCandidates + failedCandidateMapLock sync.Mutex + + //ScoreMap map[string]Score + //LogManager *LogManager + //ReservedPool *data_manager.ReservedPool + + SchedulerManager interface{} + + selectPlugins []SelectPlugin + + LogManager *SchedLogManager +} + +func NewScheduleUnit(info *api.SchedInfo, schedManager interface{}) *Unit { + cmap := make(map[string]*Capacity) // candidate_id, Capacity + smap := make(map[string]Score) // candidate_id, Score + unit := &Unit{ + SchedInfo: info, + FailedCandidateMap: make(map[string]*FailedCandidates), + failedCandidateMapLock: sync.Mutex{}, + CapacityMap: cmap, + ScoreMap: smap, + capacityLock: sync.Mutex{}, + scoreLock: sync.Mutex{}, + DataMap: make(map[string]*SchedContextDataItem), + SharedResourceManager: NewSharedResourceManager(), + CounterManager: NewCounterManager(), + LogManager: NewSchedLogManager(), + SchedulerManager: schedManager, + } + return unit +} + +func (u *Unit) Info() string { + return fmt.Sprintf("%#v", u.SchedInfo) +} + +func (u *Unit) SessionID() string { + return u.SchedInfo.SessionID +} + +func (u *Unit) SchedData() *api.SchedData { + return u.SchedInfo.Data +} + +func (u *Unit) IsPublicCloudProvider() bool { + return u.SchedData().IsPublicCloudProvider() +} + +func (u *Unit) SkipDirtyMarkHost() bool { + return u.SchedData().SkipDirtyMarkHost() +} + +func (u *Unit) AppendFailedCandidates(fcs []FailedCandidate) { + if len(fcs) == 0 { + return + } + + u.failedCandidateMapLock.Lock() + defer u.failedCandidateMapLock.Unlock() + + for _, fc := range fcs { + fcs, ok := u.FailedCandidateMap[fc.Stage] + if !ok { + fcs = &FailedCandidates{} + u.FailedCandidateMap[fc.Stage] = fcs + } + fcs.Candidates = append(fcs.Candidates, fc) + } +} + +func (u *Unit) AppendSelectPlugin(p SelectPlugin) { + u.selectPlugins = append(u.selectPlugins, p) +} + +func (u *Unit) AllSelectPlugins() []SelectPlugin { + return u.selectPlugins +} + +func (u *Unit) GetCapacity(id string) int64 { + var ( + capacityObj *Capacity + ok bool + ) + + u.capacityLock.Lock() + defer u.capacityLock.Unlock() + + if capacityObj, ok = u.CapacityMap[id]; !ok { + return 0 + } + + if capacityObj.MinValue == EmptyCapacity { + capacity := MaxCapacity + for _, counter := range capacityObj.Values { + count := counter.GetCount() + if capacity > count { + capacity = count + } + } + + capacityObj.MinValue = capacity + } + + return capacityObj.MinValue +} + +func (u *Unit) GetCapacityOfName(id string, name string) int64 { + u.capacityLock.Lock() + defer u.capacityLock.Unlock() + + if capacityObj, ok := u.CapacityMap[id]; ok { + if counter, ok0 := capacityObj.Values[name]; ok0 { + return counter.GetCount() + } + } + + return EmptyCapacity +} + +func (u *Unit) GetCapacities(id string) map[string]Counter { + if capacityObj, ok := u.CapacityMap[id]; ok { + return capacityObj.Values + } + + return EmptyCapacities +} + +func (u *Unit) SetCapacity(id string, name string, capacity Counter) error { + u.capacityLock.Lock() + defer u.capacityLock.Unlock() + + // Capacity must >= 0 + if !validateCapacityInput(capacity) { + return fmt.Errorf("Capacity invalid: %d", capacity) + } + + log.V(10).Debugf("%q setCapacity id: %s, capacity: %d", name, id, capacity.GetCount()) + + var ( + capacityObj *Capacity + ok bool + ) + + if capacityObj, ok = u.CapacityMap[id]; !ok { + capacityObj = &Capacity{Values: make(map[string]Counter), MinValue: EmptyCapacity} + u.CapacityMap[id] = capacityObj + } + + capacityObj.Values[name] = capacity + capacityObj.MinValue = EmptyCapacity + + return nil +} + +func validateCapacityInput(c Counter) bool { + if c != nil && c.GetCount() >= 0 { + return true + } + return false +} + +func (u *Unit) SetScore(id, name string, score int) error { + u.scoreLock.Lock() + defer u.scoreLock.Unlock() + + var ( + scoreObj Score + ok bool + ) + + if scoreObj, ok = u.ScoreMap[id]; !ok { + scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore} + u.ScoreMap[id] = scoreObj + } + + scoreObj.Values[name] = score + scoreObj.Sum = EmptyScore + + log.V(10).Infof("%q SetScore: %q -> %d", name, id, score) + return nil +} + +func (u *Unit) IncreaseScore(id string, name string, increase int) error { + + u.scoreLock.Lock() + defer u.scoreLock.Unlock() + + var ( + scoreObj Score + ok bool + ) + + score := int(0) + if scoreObj, ok = u.ScoreMap[id]; !ok { + scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore} + u.ScoreMap[id] = scoreObj + score = increase + } else { + if value, ok := scoreObj.Values[name]; ok { + score = value + increase + } else { + score = increase + } + } + + scoreObj.Values[name] = score + scoreObj.Sum = EmptyScore + + log.V(10).Infof("%q IncreaseScore: %q -> %d", name, id, score) + return nil +} + +func (u *Unit) GetScore(id string) int { + var ( + scoreObj Score + ok bool + ) + + if scoreObj, ok = u.ScoreMap[id]; !ok { + return BaseScore + } + + if scoreObj.Sum == EmptyScore { + sum := int(0) + for _, value := range scoreObj.Values { + sum += value + } + + scoreObj.Sum = sum + } + + return scoreObj.Sum + BaseScore +} + +func (u *Unit) GetScores(id string) map[string]int { + if scores, ok := u.ScoreMap[id]; ok { + return scores.Values + } + + return EmptyScores +} +func (u *Unit) SetFiltedData(id string, name string, data interface{}) error { + u.scoreLock.Lock() + defer u.scoreLock.Unlock() + + dataItem, ok := u.DataMap[id] + if !ok { + dataItem = &SchedContextDataItem{ + Data: make(map[string]interface{}), + } + u.DataMap[id] = dataItem + } + + if name == "network" { + dataItem.Networks = data.(sync.Map) + } else { + if m, ok := data.(map[string]interface{}); ok { + for key, value := range m { + dataItem.Data[key] = value + } + } + } + + return nil +} + +func (u *Unit) GetFiltedData(id string, count int64) map[string]interface{} { + schedContextData := make(map[string]interface{}) + if data, ok := u.DataMap[id]; ok { + // deal networks + networks := make(map[string]int64) + data.Networks.Range(func(networkID, ipNumber interface{}) bool { + networkIDString := networkID.(string) + ipNumberInt64 := ipNumber.(int64) + if count > ipNumberInt64 { + networks[networkIDString] = ipNumberInt64 + count = count - ipNumberInt64 + return true + } else if count <= ipNumberInt64 { + networks[networkIDString] = count + count = 0 + return false + } + return false + }) + + schedContextData["networks"] = networks + + // others + for key, value := range data.Data { + schedContextData[key] = value + } + + return schedContextData + } + + return nil +} diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go new file mode 100644 index 0000000000..d3b73d1db0 --- /dev/null +++ b/pkg/scheduler/core/generic_scheduler.go @@ -0,0 +1,660 @@ +package core + +import ( + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/pkg/util/errors" + gp "github.com/yunionio/pkg/util/goroutine_pool" + utiltrace "github.com/yunionio/pkg/util/trace" + "github.com/yunionio/pkg/util/workqueue" +) + +const ( + NoResourceAvailableMsg = "No resource are avaliable that match all of the following predicates:" +) + +// goroutine pool is to solve the problem of go expansion in +// the stack, check goroutine pool status every minute. +var ( + pool = gp.New(60 * time.Second) +) + +type FailedPredicateMap map[string][]PredicateFailureReason + +type FitError struct { + Unit *Unit + FailedCandidateMap map[string]*FailedCandidates +} + +// Error returns detailed information of why the guest failed to fit on each host +func (fe *FitError) Error() string { + ss := []string{} + for stage, fcs := range fe.FailedCandidateMap { + ss = append(ss, fmt.Sprintf("%v(-%v)", stage, len(fcs.Candidates))) + } + reasonMsg := fmt.Sprintf("%s filter by %v, session_id=%q", NoResourceAvailableMsg, + strings.Join(ss, ", "), fe.Unit.SessionID()) + return reasonMsg +} + +type NoResourceError struct { + info string + sessionID string +} + +func (e *NoResourceError) Error() string { + return fmt.Sprintf("No resource avaliable to schedule, session_id: %q, info: %q", e.sessionID, e.info) +} + +type Scheduler interface { + BeforePredicate() error + Predicates() (map[string]FitPredicate, error) + PriorityConfigs() ([]PriorityConfig, error) + + // mark already selected candidates dirty that + // can't be use again until cleanup them + DirtySelectedCandidates([]*SelectedCandidate) +} + +type GenericScheduler struct { + Scheduler + predicates map[string]FitPredicate + priorities []PriorityConfig +} + +func NewGenericScheduler(s Scheduler) (*GenericScheduler, error) { + g := &GenericScheduler{} + predicates, err := s.Predicates() + if err != nil { + return nil, err + } + priorities, err := s.PriorityConfigs() + if err != nil { + return nil, err + } + g.Scheduler = s + g.predicates = predicates + g.priorities = priorities + return g, nil +} + +func (g *GenericScheduler) Schedule(unit *Unit, candidates []Candidater) ([]*SchedResultItem, error) { + startTime := time.Now() + defer func() { + log.V(4).Infof("Schedule cost time: %v", time.Since(startTime)) + }() + + // get schedule context and information + schedInfo := unit.SchedInfo + isSuggestion := schedInfo.IsSuggestion + + // new trace follow all steps + trace := utiltrace.New(fmt.Sprintf("SessionID: %s, schedule info: %s", + schedInfo.SessionID, unit.Info())) + + defer trace.LogIfLong(100 * time.Millisecond) + if len(candidates) == 0 { + return nil, &NoResourceError{ + sessionID: schedInfo.SessionID, + info: unit.Info(), + } + } + + // setup something before run predicates, but now there is no actions + err := g.BeforePredicate() + if err != nil { + return nil, err + } + trace.Step("Computing predicates") + + // load all predicates and find the candidate can statisfy schedule condition + filteredCandidates, err := findCandidatesThatFit(unit, candidates, g.predicates) + if err != nil { + return nil, err + } + + // if there is no candidate and not from scheduler/test api will return + if len(filteredCandidates) == 0 && !isSuggestion { + return nil, &FitError{ + Unit: unit, + FailedCandidateMap: unit.FailedCandidateMap, + } + } + + var selectedCandidates []*SelectedCandidate + if len(filteredCandidates) > 0 { + trace.Step("Prioritizing") + // load all priorities and calculate the candidate's score + priorityList, err := PrioritizeCandidates(unit, filteredCandidates, g.priorities) + if err != nil { + return nil, err + } + + trace.Step("Selecting hosts") + // select target candate hosts + selectedCandidates, err = SelectHosts(unit, priorityList) + if err != nil { + return nil, err + } + } else { + selectedCandidates = []*SelectedCandidate{} + } + + resultItems, err := generateScheduleResult(unit, selectedCandidates, candidates) + if err != nil { + return nil, err + } + + // sync schedule candidates dirty mark + if !isSuggestion && !unit.SkipDirtyMarkHost() { + g.DirtySelectedCandidates(selectedCandidates) + } + + return resultItems, nil +} + +func newSchedResultByCtx(u *Unit, count int64, c Candidater) *SchedResultItem { + showDetails := u.SchedInfo.ShowSuggestionDetails + id := c.IndexKey() + r := &SchedResultItem{ + ID: id, + Count: count, + Capacity: u.GetCapacity(id), + Name: fmt.Sprintf("%v", c.Get("Name")), + Score: u.GetScore(id), + Data: u.GetFiltedData(id, count), + } + + if showDetails { + r.CapacityDetails = GetCapacities(u, id) + r.ScoreDetails = u.GetScores(id) + } + return r +} + +func generateScheduleResult(u *Unit, scs []*SelectedCandidate, cs []Candidater) ([]*SchedResultItem, error) { + results := make([]*SchedResultItem, 0) + itemMap := make(map[string]int) + + for _, it := range scs { + cid := it.Candidate.IndexKey() + r := newSchedResultByCtx(u, it.Count, it.Candidate) + results = append(results, r) + itemMap[cid] = 1 + } + + suggestionLimit := u.SchedInfo.SuggestionLimit + for _, c := range cs { + if suggestionLimit <= int64(len(results)) { + break + } + id := c.IndexKey() + if _, ok := itemMap[id]; !ok && u.GetCapacity(id) > 0 { + itemMap[id] = 1 + r := newSchedResultByCtx(u, 0, c) + results = append(results, r) + } + } + + suggestionAll := u.SchedInfo.SuggestionAll + if suggestionAll || len(u.SchedData().Candidates) > 0 { + for _, c := range cs { + if suggestionLimit <= int64(len(results)) { + + break + } + id := c.IndexKey() + if _, ok := itemMap[id]; !ok { + itemMap[id] = 0 + r := newSchedResultByCtx(u, 0, c) + results = append(results, r) + } + } + } + + return results, nil +} + +type SchedResultItem struct { + ID string `json:"id"` + Name string `json:"name"` + Count int64 `json:"count"` + Data map[string]interface{} `json:"data"` + Capacity int64 `json:"capacity"` + Score int `json:"score"` + + CapacityDetails map[string]int64 `json:"capacity_details"` + ScoreDetails map[string]int `json:"score_details"` +} + +func GetCapacities(u *Unit, id string) (res map[string]int64) { + res = make(map[string]int64) + capacities := u.GetCapacities(id) + if len(capacities) > 0 { + for name, capacity := range capacities { + res[name] = capacity.GetCount() + } + } + return +} + +func GetScore(u *Unit, id string, details bool) string { + score := u.GetScore(id) + s := fmt.Sprintf("%v", score) + if details { + scores := u.GetScores(id) + if len(scores) > 0 { + ss := []string{} + for name, score := range scores { + ss = append(ss, fmt.Sprintf("%v:%v", name, score)) + } + s += " (" + strings.Join(ss, ", ") + ")" + } + } + return s +} + +type SchedResultItemList struct { + Unit *Unit + Data []*SchedResultItem +} + +func (its SchedResultItemList) Len() int { + return len(its.Data) +} + +func (its SchedResultItemList) Less(i, j int) bool { + it1, it2 := its.Data[i], its.Data[j] + ctx := its.Unit + + m := func(c int64) int64 { + if c > 0 { + return 1 + } + return 0 + } + + v := func(count, capacity, score int64) int64 { + return (m(count) << 42) | (m(capacity) << 21) | score + } + + count1, count2 := it1.Count, it2.Count + capacity1, capacity2 := ctx.GetCapacity(it1.ID), ctx.GetCapacity(it2.ID) + score1, score2 := int64(ctx.GetScore(it1.ID)), int64(ctx.GetScore(it2.ID)) + + return v(count1, capacity1, score1) < v(count2, capacity2, score2) +} + +type SelectedCandidate struct { + Count int64 + Candidate Candidater +} + +func (s SelectedCandidate) Index() (string, error) { + return s.Candidate.IndexKey(), nil +} + +// SelectHosts takes a prioritized list of candidates and then picks +// a group of hosts +func SelectHosts(unit *Unit, priorityList HostPriorityList) ([]*SelectedCandidate, error) { + if len(priorityList) == 0 { + return nil, fmt.Errorf("SelectHosts get empty priorityList.") + } + + sort.Sort(sort.Reverse(priorityList)) + + selectedMap := make(map[string]*SelectedCandidate) + schedData := unit.SchedData() + count := schedData.Count + isSuggestion := unit.SchedInfo.IsSuggestion + bestEffort := unit.SchedInfo.BestEffort + selectedCandidates := []*SelectedCandidate{} + plugins := unit.AllSelectPlugins() + +completed: + for len(priorityList) > 0 { + log.V(10).Debugf("PriorityList: %#v", priorityList) + priorityList0 := HostPriorityList{} + for _, it := range priorityList { + if count <= 0 { + break completed + } + hostID := it.Host + var ( + selectedItem *SelectedCandidate + ok bool + ) + if selectedItem, ok = selectedMap[hostID]; !ok { + selectedItem = &SelectedCandidate{ + Count: 0, + Candidate: it.Candidate, + } + selectedMap[hostID] = selectedItem + } + selectedItem.Count++ + count-- + doPlugins := func() bool { + r := true + for _, plugin := range plugins { + if !plugin.OnSelect(unit, selectedItem.Candidate) { + r = false + } + } + return r + } + // if no one of plugins return false or capacity of the host large than + // selected count, this host can be added to priorityList. + if doPlugins() && unit.GetCapacity(hostID) > selectedItem.Count { + priorityList0 = append(priorityList0, it) + } + } + // sort by score + priorityList = priorityList0 + sort.Sort(sort.Reverse(priorityList)) + } + + for _, sc := range selectedMap { + for _, plugin := range plugins { + plugin.OnSelectEnd(unit, sc.Candidate, sc.Count) + } + selectedCandidates = append(selectedCandidates, sc) + } + + if !isSuggestion && !bestEffort { + if count > 0 { + return nil, fmt.Errorf("No enough resource, request/capacity: %d/%d", schedData.Count, schedData.Count-count) + } + } + + return selectedCandidates, nil +} + +func findCandidatesThatFit(unit *Unit, candidates []Candidater, predicates map[string]FitPredicate) ([]Candidater, error) { + var filtered []Candidater + + ok, err, newPredicates := preExecPredicate(unit, candidates, predicates) + if !ok { + return nil, err + } + + // sort predicates by their name + predicateNames := make([]string, 0, len(newPredicates)) + for name := range newPredicates { + predicateNames = append(predicateNames, name) + } + sort.Strings(predicateNames) + predicateArray := make([]FitPredicate, 0, len(predicateNames)) + for _, name := range predicateNames { + predicateArray = append(predicateArray, newPredicates[name]) + } + + // do predicate filter + if len(predicateArray) == 0 { + filtered = candidates + } else { + // Create predicate list with enough space to avoid growing it + // and allow assigning. + filtered = make([]Candidater, len(candidates)) + errsChannel := make(chan error, len(candidates)) + var filteredLen int32 + checkUnit := func(i int) { + fits, fcs, err := unitFitsOnCandidate( + unit, candidates[i], predicateArray) + if err != nil { + errsChannel <- err + return + } + if fits { + filtered[atomic.AddInt32(&filteredLen, 1)-1] = candidates[i] + } else { + unit.AppendFailedCandidates(fcs) + } + } + workqueue.Parallelize(o.GetOptions().PredicateParallelizeSize, len(candidates), checkUnit) + filtered = filtered[:filteredLen] + if len(errsChannel) > 0 { + errs := make([]error, 0) + length := len(errsChannel) + for ; length > 0; length-- { + errs = append(errs, <-errsChannel) + } + return []Candidater{}, errors.NewAggregate(errs) + } + } + return filtered, nil +} + +func preExecPredicate(unit *Unit, candidates []Candidater, predicates map[string]FitPredicate) (bool, error, map[string]FitPredicate) { + var ( + name string + predicate FitPredicate + ok bool + err error + newPredicateFuncs map[string]FitPredicate + ) + newPredicateFuncs = make(map[string]FitPredicate) + for name, predicate = range predicates { + // generate new FitPredicates because of race condition? + newPredicate := predicate.Clone() + ok, err = newPredicate.PreExecute(unit, candidates) + if ok { + newPredicateFuncs[name] = newPredicate + } + if err != nil { + return false, err, nil + } + } + return true, err, newPredicateFuncs +} + +type WaitGroupWrapper struct { + sync.WaitGroup +} + +func (w *WaitGroupWrapper) Wrap(cb func()) { + w.Add(1) + pool.Go(func() { + cb() + w.Done() + }) +} + +func unitFitsOnCandidate( + unit *Unit, + candidate Candidater, + predicates []FitPredicate, +) (bool, []FailedCandidate, error) { + var ( + fit bool + reasons []PredicateFailureReason + err error + fcs []FailedCandidate + logs []SchedLog + ) + + isFit := true + defer func() { + if len(logs) > 0 { + unit.LogManager.Appends(logs) + } + }() + + toLog := func(fit bool, reasons []PredicateFailureReason, + err error, stage string) SchedLog { + var ( + sFit string + message string + ) + if fit { + sFit = "Success." + } else { + sFit = "Failed:" + } + if err != nil { + message = fmt.Sprintf("%v", err) + } else { + if len(reasons) == 0 { + message = "" + } else { + ss := make([]string, 0, len(reasons)) + for _, reason := range reasons { + ss = append(ss, reason.GetReason()) + } + message = strings.Join(ss, ", ") + } + } + + candidateLogIndex := fmt.Sprintf("%v:%s", candidate.Get("Name"), candidate.IndexKey()) + + return NewSchedLog(candidateLogIndex, stage, fmt.Sprintf("%v %v", sFit, message)) + } + + for _, predicate := range predicates { + fit, reasons, err = predicate.Execute(unit, candidate) + logs = append(logs, toLog(fit, reasons, err, predicate.Name())) + if err != nil { + return false, nil, err + } + if !fit { + fcs = append(fcs, FailedCandidate{ + Stage: predicate.Name(), + Candidate: candidate, + Reasons: reasons, + }) + isFit = false + // When AlwaysCheckAllPredicates is set to true, scheduler checks all + // the configured predicates even after one or more of them fails. + // When the flag is set to false, scheduler skips checking the rest + // of the predicates after it finds one predicate that failed. + if !o.GetOptions().AlwaysCheckAllPredicates { + break + } + } + } + return isFit, fcs, nil +} + +// PrioritizeCandidates by running the individual priority functions in parallel. +// Each priority function is expected to set a score of 0-10 +// 0 is the lowest priority score (least preffered node) and 10 is the highest +/// Each priority function can also have its own weight +// The resource scores returned by priority function are multiplied by the weights to get weighted scores +// All scores are finally combined (added) to get the total weighted scores of all resources +func PrioritizeCandidates( + unit *Unit, + candidates []Candidater, + priorities []PriorityConfig, +) (HostPriorityList, error) { + // If no priority configs are provided, then the EqualPriority function is applied + // This is required to generate the priority list in the required format + if len(priorities) == 0 { + result := make(HostPriorityList, 0, len(candidates)) + for _, candidate := range candidates { + hostPriority, err := EqualPriority(unit, candidate) + if err != nil { + return nil, err + } + result = append(result, hostPriority) + } + return result, nil + } + + wg := sync.WaitGroup{} + results := make([]HostPriorityList, 0, len(priorities)) + for range priorities { + results = append(results, nil) + } + newPriorities, err := preExecPriorities(priorities, unit, candidates) + if err != nil { + return nil, err + } + // Max : 3 * len(newPriorities) + errsChannel := make(chan error, 3*len(newPriorities)) + for i := range newPriorities { + results[i] = make(HostPriorityList, len(candidates)) + } + + processCandidate := func(index int) { + var err error + candidate := candidates[index] + for i := range newPriorities { + results[i][index], err = newPriorities[i].Map(unit, candidate) + if err != nil { + errsChannel <- err + return + } + } + } + workqueue.Parallelize(o.GetOptions().PriorityParallelizeSize, len(candidates), processCandidate) + + for i, p := range newPriorities { + wg.Add(1) + go func(index int, priority PriorityConfig) { + defer wg.Done() + if err := priority.Reduce(unit, candidates, results[index]); err != nil { + errsChannel <- err + } + }(i, p) + } + // Wait for all computations to be finished. + wg.Wait() + if len(errsChannel) != 0 { + errs := make([]error, 0) + length := len(errsChannel) + for ; length > 0; length-- { + errs = append(errs, <-errsChannel) + } + return HostPriorityList{}, errors.NewAggregate(errs) + } + + // Summarize all scores + result := make(HostPriorityList, 0, len(candidates)) + // TODO: Consider parallelizing it + + for i := range candidates { + result = append(result, HostPriority{Host: candidates[i].IndexKey(), Score: 0, Candidate: candidates[i]}) + for j := range newPriorities { + result[i].Score += results[j][i].Score * newPriorities[j].Weight + } + } + if log.V(10) { + for i := range result { + log.Infof("Host %s => Score %d", result[i].Host, result[i].Score) + } + } + return result, nil +} + +func preExecPriorities(priorities []PriorityConfig, unit *Unit, candidates []Candidater) ([]PriorityConfig, error) { + newPriorities := []PriorityConfig{} + for _, p := range priorities { + ok, _, err := p.Pre(unit, candidates) + if err != nil { + return nil, err + } + if ok { + newPriorities = append(newPriorities, p) + } + } + return newPriorities, nil +} + +// EqualPriority is a prioritizer function that gives an equal weight of one to all candidates +func EqualPriority(_ *Unit, candidate Candidater) (HostPriority, error) { + indexKey := candidate.IndexKey() + if indexKey == "" { + return HostPriority{}, fmt.Errorf("Candidate indexKey is empty") + } + return HostPriority{ + Host: indexKey, + Score: 1, + Candidate: candidate, + }, nil +} diff --git a/pkg/scheduler/core/resource.go b/pkg/scheduler/core/resource.go new file mode 100644 index 0000000000..36c6d55523 --- /dev/null +++ b/pkg/scheduler/core/resource.go @@ -0,0 +1,78 @@ +package core + +import ( + "github.com/yunionio/pkg/utils" +) + +type value_t interface{} + +type ResourceAlgorithm interface { + Sum(values []value_t) value_t + Sub(sum value_t, reserved value_t) value_t +} + +type DefaultResAlgorithm struct{} + +func (al *DefaultResAlgorithm) Sum(values []value_t) value_t { + var ret int64 = 0 + for _, value := range values { + if value != nil { + ret += value.(int64) + } + } + return ret +} + +func (al *DefaultResAlgorithm) Sub(sum value_t, reserved value_t) value_t { + if sum == nil { + return nil + } + + if reserved == nil { + return sum + } + + return sum.(int64) - reserved.(int64) +} + +var ( + g_defaultResAlgorithm *DefaultResAlgorithm = &DefaultResAlgorithm{} +) + +func GetResourceAlgorithm(res_name string) ResourceAlgorithm { + switch res_name { + //case "Groups": + //return g_groupResAlgorithm + case "FreeCPUCount", "FreeMemSize", "FreeLocalStorageSize", "Ports": + return g_defaultResAlgorithm + default: + if utils.HasPrefix(res_name, "FreeStorageSize:") { + return g_defaultResAlgorithm + } + + return nil + } +} + +func ReservedSub(key string, value value_t, reserved value_t) value_t { + al := GetResourceAlgorithm(key) + if al != nil { + return al.Sub(value, reserved) + } + return value +} + +func XGetCalculator(c Candidater, key string, kind Kind) value_t { + value := c.Get(key) + + switch kind { + case KindFree: + // TODO: reserved not impl by now + return ReservedSub(key, value, nil) + case KindRaw: + return value + case KindReserved: + return nil + } + return nil +} diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go new file mode 100644 index 0000000000..0cb7a93377 --- /dev/null +++ b/pkg/scheduler/core/types.go @@ -0,0 +1,112 @@ +package core + +import ( + //"sync" + //"github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" +) + +const ( + PriorityStep int = 100 +) + +type FailedCandidate struct { + Stage string + Candidate Candidater + Reasons []PredicateFailureReason +} + +type FailedCandidates struct { + Candidates []FailedCandidate +} + +type SelectPlugin interface { + OnSelect(*Unit, Candidater) bool + OnSelectEnd(u *Unit, c Candidater, count int64) +} + +type Kind int + +const ( + KindFree Kind = iota + KindRaw + KindReserved +) + +// Candidater replace host Candidate resource info +type Candidater interface { + // IndexKey return candidate cache item's ident, usually host ID + IndexKey() string + // Get return candidate cache item's value by key + Get(key string) interface{} + // XGet return candidate cache item's value by key and kind + XGet(key string, kind Kind) interface{} + Type() int + + GetAggregates() []*models.Aggregate + GetHostAggregates() []*models.Aggregate +} + +// HostPriority represents the priority of scheduling to particular host, higher priority is better. +type HostPriority struct { + // Name of the host + Host string + // Score associated with the host + Score int + // Resource wraps Candidate host info + Candidate Candidater +} + +type HostPriorityList []HostPriority + +func (h HostPriorityList) Len() int { + return len(h) +} + +func (h HostPriorityList) Less(i, j int) bool { + if h[i].Score == h[j].Score { + return h[i].Host < h[j].Host + } + return h[i].Score < h[j].Score +} + +func (h HostPriorityList) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +type FitPredicate interface { + // Get filter's name + Name() string + Clone() FitPredicate + PreExecute(*Unit, []Candidater) (bool, error) + Execute(*Unit, Candidater) (bool, []PredicateFailureReason, error) +} + +type PredicateFailureReason interface { + GetReason() string +} + +type PriorityPreFunction func(*Unit, []Candidater) (bool, []PredicateFailureReason, error) + +// PriorityMapFunction is a function that computes per-resource results for a given resource. +type PriorityMapFunction func(*Unit, Candidater) (HostPriority, error) + +// PriorityReduceFunction is a function that aggregated per-resource results and computes +// final scores for all hosts. +type PriorityReduceFunction func(*Unit, []Candidater, HostPriorityList) error + +type PriorityConfig struct { + Name string + Pre PriorityPreFunction + Map PriorityMapFunction + Reduce PriorityReduceFunction + Weight int +} + +type Priority interface { + Name() string + Clone() Priority + Map(*Unit, Candidater) (HostPriority, error) + Reduce(*Unit, []Candidater, HostPriorityList) error + PreExecute(*Unit, []Candidater) (bool, []PredicateFailureReason, error) +} diff --git a/pkg/scheduler/data_manager/candidate_manager.go b/pkg/scheduler/data_manager/candidate_manager.go new file mode 100644 index 0000000000..66f1ee4677 --- /dev/null +++ b/pkg/scheduler/data_manager/candidate_manager.go @@ -0,0 +1,368 @@ +package data_manager + +import ( + "fmt" + "time" + + "github.com/yunionio/onecloud/pkg/scheduler/cache" + candidatecache "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + dbcache "github.com/yunionio/onecloud/pkg/scheduler/cache/db" + synccache "github.com/yunionio/onecloud/pkg/scheduler/cache/sync" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/util/ttlpool" +) + +type CandidateGetArgs struct { + ResType string + ZoneID string + PoolID string + IgnorePool bool +} + +type DataManager struct { + DBCacheGroup cache.CacheGroup + SyncCacheGroup cache.CacheGroup + CandidateGroup cache.CacheGroup +} + +func NewDataManager(stopCh <-chan struct{}) *DataManager { + m := new(DataManager) + m.DBCacheGroup = dbcache.NewCacheManager(stopCh) + m.SyncCacheGroup = synccache.NewSyncManager(stopCh) + m.CandidateGroup = candidatecache.NewCandidateManager( + m.DBCacheGroup, m.SyncCacheGroup, stopCh) + + return m +} + +func (m *DataManager) Run() { + go m.DBCacheGroup.Run() + go m.SyncCacheGroup.Run() + go m.CandidateGroup.Run() +} + +type CandidateManagerImplProvider interface { + LoadCandidates() ([]interface{}, error) + ReloadCandidates(ids []string) ([]interface{}, error) + ReloadAllCandidates() ([]interface{}, error) + GetCandidate(id string) (interface{}, error) +} + +type HostCandidateManagerImplProvider struct { + dataManager *DataManager +} + +func getCache(dataManager *DataManager, name string) (cache.Cache, error) { + candidate_cache, err := dataManager.CandidateGroup.Get(name) + if err != nil { + return nil, err + } + + candidate_cache.WaitForReady() + return candidate_cache, nil +} + +func (p *HostCandidateManagerImplProvider) LoadCandidates() ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.HostCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.List(), nil +} + +func (p *HostCandidateManagerImplProvider) ReloadCandidates( + ids []string) ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.HostCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.Reload(ids) +} + +func (p *HostCandidateManagerImplProvider) ReloadAllCandidates() ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.HostCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.ReloadAll() +} + +func (p *HostCandidateManagerImplProvider) GetCandidate(id string) (interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.HostCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.Get(id) +} + +type BaremetalCandidateManagerImplProvider struct { + dataManager *DataManager +} + +func (p *BaremetalCandidateManagerImplProvider) LoadCandidates() ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.BaremetalCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.List(), nil +} + +func (p *BaremetalCandidateManagerImplProvider) ReloadCandidates( + ids []string) ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.BaremetalCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.Reload(ids) +} + +func (p *BaremetalCandidateManagerImplProvider) ReloadAllCandidates() ([]interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.BaremetalCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.ReloadAll() +} + +func (p *BaremetalCandidateManagerImplProvider) GetCandidate(id string) (interface{}, error) { + candidate_cache, err := getCache(p.dataManager, candidatecache.BaremetalCandidateCache) + if err != nil { + return nil, err + } + + return candidate_cache.Get(id) +} + +type CandidateManagerImpl struct { + provider CandidateManagerImplProvider + dataMap map[string][]interface{} + stopCh <-chan struct{} + lastLoadTime time.Time +} + +func NewCandidateManagerImpl(provider CandidateManagerImplProvider, stopCh <-chan struct{}, +) *CandidateManagerImpl { + return &CandidateManagerImpl{ + provider: provider, + dataMap: make(map[string][]interface{}), + stopCh: stopCh, + } +} + +func (impl *CandidateManagerImpl) GetCandidates() ([]interface{}, error) { + return impl.provider.LoadCandidates() +} + +func (impl *CandidateManagerImpl) GetCandidate(id string) (interface{}, error) { + return impl.provider.GetCandidate(id) +} + +func (impl *CandidateManagerImpl) Reload(ids []string) ([]interface{}, error) { + return impl.provider.ReloadCandidates(ids) +} + +func (impl *CandidateManagerImpl) ReloadAll() ([]interface{}, error) { + return impl.provider.ReloadAllCandidates() +} + +func (impl *CandidateManagerImpl) Run() { +} + +type CandidateManager struct { + stopCh <-chan struct{} + dataManager *DataManager + impls map[string]*CandidateManagerImpl + + dirtyPool *ttlpool.CountPool +} + +func (cm *CandidateManager) DirtyPoolHas(id string) bool { + ok, _ := cm.dirtyPool.HasByKey(id) + return ok +} + +func (cm *CandidateManager) GetCandidates(args CandidateGetArgs) ([]core.Candidater, error) { + impl, err := cm.getImpl(args.ResType) + if err != nil { + return nil, err + } + + candidates, err2 := impl.GetCandidates() + if err2 != nil { + return nil, err2 + } + + hasZone := len(args.ZoneID) > 0 + + result := []core.Candidater{} + + for _, c := range candidates { + r := c.(core.Candidater) + + if cm.DirtyPoolHas(r.IndexKey()) { + continue + } + + if args.IgnorePool { + result = append(result, r) + } else if (!hasZone || r.Get("ZoneID") == args.ZoneID) && r.Get("PoolID") == args.PoolID { + result = append(result, r) + } + } + + return result, nil +} + +func (cm *CandidateManager) GetCandidatesByIds(resType string, ids []string) ([]core.Candidater, error) { + impl, err := cm.getImpl(resType) + if err != nil { + return nil, err + } + + candidates := []core.Candidater{} + for _, id := range ids { + if cm.DirtyPoolHas(id) { + continue + } + + c, err2 := impl.GetCandidate(id) + if err2 != nil { + return nil, err2 + } + candidates = append(candidates, c.(core.Candidater)) + } + + return candidates, nil +} + +func (cm *CandidateManager) GetCandidate(id string, resType string) (interface{}, error) { + impl, err := cm.getImpl(resType) + if err != nil { + return nil, err + } + + if cm.DirtyPoolHas(id) { + return nil, fmt.Errorf("%s in dirtyPool", id) + } + + c, err := impl.GetCandidate(id) + if err != nil { + return nil, err + } + return c.(core.Candidater), nil +} + +func (cm *CandidateManager) getImpl(resType string) (*CandidateManagerImpl, error) { + var ( + impl *CandidateManagerImpl + ok bool + ) + + if impl, ok = cm.impls[resType]; !ok { + return nil, fmt.Errorf("Resource Type \"%v\" not supported", resType) + } + + return impl, nil +} + +func (cm *CandidateManager) AddImpl(name string, impl *CandidateManagerImpl) { + cm.impls[name] = impl +} + +func NewCandidateManager(dataManager *DataManager, stopCh <-chan struct{}) *CandidateManager { + + candidateManager := &CandidateManager{ + stopCh: stopCh, + impls: make(map[string]*CandidateManagerImpl), + dataManager: dataManager, + dirtyPool: ttlpool.NewCountPool(), + } + + candidateManager.AddImpl("host", NewCandidateManagerImpl( + &HostCandidateManagerImplProvider{dataManager: dataManager}, stopCh)) + + candidateManager.AddImpl("baremetal", NewCandidateManagerImpl( + &BaremetalCandidateManagerImplProvider{dataManager: dataManager}, stopCh)) + + return candidateManager +} + +func (cm *CandidateManager) Run() { + + for _, impl := range cm.impls { + impl.Run() + } +} + +func (cm *CandidateManager) GetData(name string) ([]interface{}, error) { + cache, err := cm.dataManager.DBCacheGroup.Get(name) + if err != nil { + return nil, err + } + + cache.WaitForReady() + return cache.List(), nil +} + +func (cm *CandidateManager) Reload(resType string, candidateIds []string) ( + []interface{}, error) { + + if len(candidateIds) == 0 { + return []interface{}{}, nil + } + + impl, err := cm.getImpl(resType) + if err != nil { + return nil, err + } + + return impl.Reload(candidateIds) +} + +func (cm *CandidateManager) ReloadAll(resType string) ([]interface{}, error) { + impl, err := cm.getImpl(resType) + if err != nil { + return nil, err + } + + return impl.ReloadAll() +} + +func (cm *CandidateManager) SetCandidatesDirty(scs []*core.SelectedCandidate) { + for _, sc := range scs { + cm.dirtyPool.Add(sc, uint64(sc.Count)) + } +} + +func (cm *CandidateManager) CleanDirtyCandidatesOnce(keys []string) { + for _, key := range keys { + cm.dirtyPool.DeleteByKey(key) + } +} + +func ToHostCandidate(c interface{}) (*candidatecache.HostDesc, error) { + h, ok := c.(*candidatecache.HostDesc) + if !ok { + return nil, fmt.Errorf("%#v can't convert to *candidatecache.HostDesc") + } + return h, nil +} + +func ToHostCandidates(cs []interface{}) ([]*candidatecache.HostDesc, error) { + hs := make([]*candidatecache.HostDesc, 0) + for _, c := range cs { + h, err := ToHostCandidate(c) + if err != nil { + return nil, err + } + hs = append(hs, h) + } + return hs, nil +} diff --git a/pkg/scheduler/data_manager/image_manager.go b/pkg/scheduler/data_manager/image_manager.go new file mode 100644 index 0000000000..272ea896df --- /dev/null +++ b/pkg/scheduler/data_manager/image_manager.go @@ -0,0 +1 @@ +package data_manager diff --git a/pkg/scheduler/data_manager/network_manager.go b/pkg/scheduler/data_manager/network_manager.go new file mode 100644 index 0000000000..338a230fb3 --- /dev/null +++ b/pkg/scheduler/data_manager/network_manager.go @@ -0,0 +1,167 @@ +package data_manager + +import ( + "sync" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/cache" + synccache "github.com/yunionio/onecloud/pkg/scheduler/cache/sync" + networks_db "github.com/yunionio/onecloud/pkg/scheduler/cache/sync/networks/db" +) + +// --------------------------------------------------- + +type CandidateIdMap map[string]int + +type VpcNetwork struct { + Data CandidateIdMap + Network *synccache.SchedNetworkBuildResult +} + +func NewVpcNetwork() *VpcNetwork { + return &VpcNetwork{Data: make(CandidateIdMap)} +} + +// --------------------------------------------------- + +type VpcNetworks struct { + Data map[string]*VpcNetwork + vpcNetworkLock sync.Mutex +} + +func NewVpcNetworks() *VpcNetworks { + return &VpcNetworks{ + Data: make(map[string]*VpcNetwork), + vpcNetworkLock: sync.Mutex{}, + } +} + +func (vns *VpcNetworks) Append(candidateId string, + networks []*synccache.SchedNetworkBuildResult) { + + vns.vpcNetworkLock.Lock() + defer vns.vpcNetworkLock.Unlock() + + for _, network := range networks { + for _, idx := range []string{network.ID, network.Name} { + vpcNetwork, ok := vns.Data[idx] + if !ok { + vpcNetwork = NewVpcNetwork() + vns.Data[idx] = vpcNetwork + } + + vpcNetwork.Network = network + vpcNetwork.Data[candidateId] = 0 + } + } +} + +func (vns *VpcNetworks) Exists(networkId, candidateId string, +) *synccache.SchedNetworkBuildResult { + vns.vpcNetworkLock.Lock() + defer vns.vpcNetworkLock.Unlock() + + if vpcNetwork, ok := vns.Data[networkId]; ok { + if _, ok := vpcNetwork.Data[candidateId]; ok { + return vpcNetwork.Network + } + } + + return nil +} + +func (vns *VpcNetworks) Get(networkId string) *VpcNetwork { + vns.vpcNetworkLock.Lock() + defer vns.vpcNetworkLock.Unlock() + + if vpcNetwork, ok := vns.Data[networkId]; ok { + return vpcNetwork + } + + return nil +} + +// --------------------------------------------------- + +type NetworkManager struct { + dataManager *DataManager + vpcNetworks *VpcNetworks + networksPool *ReservedPool +} + +func NewNetworkManager(dataManager *DataManager, reservedPoolManager *ReservedPoolManager) *NetworkManager { + networksPool, err := reservedPoolManager.GetPool("networks") + if err != nil { + log.Errorln(err) + } + + return &NetworkManager{ + dataManager: dataManager, + vpcNetworks: NewVpcNetworks(), + networksPool: networksPool, + } +} + +func (m *NetworkManager) CleanVpc() { + m.vpcNetworks = NewVpcNetworks() +} + +func (m *NetworkManager) IsUnknown(id string) bool { + return false +} + +func (m *NetworkManager) GetReservecPorts(id string) int64 { + // TODO: impl reserve network resource + //if m.networksPool.GetReservedItem(id) != nil { + //return m.networksPool.GetReservedItem(id).Get("Ports", int64(0)).(int64) + //} else { + return 0 + //} +} + +func (m *NetworkManager) LoadUnknownNetworks(ids []string) { + if len(ids) == 0 { + return + } + + builder, err := m.getHostNetworkDBDescBuilder() + if err != nil { + log.Errorf("Reload network error: %v", err) + return + } + + builder.Load(ids) +} + +func (m *NetworkManager) GetVpcNetwork(networkId string) *VpcNetwork { + return m.vpcNetworks.Get(networkId) +} + +func (m *NetworkManager) ExistsVpcNetwork(networkId, candidateId string, +) *synccache.SchedNetworkBuildResult { + return m.vpcNetworks.Exists(networkId, candidateId) +} + +func (m *NetworkManager) getHostNetworkDBDescBuilder() (networks_db.NetworkDescBuilder, error) { + cache, err := m.getNetworkCache() + if err != nil { + return nil, err + } + + builder, err := cache.Get(synccache.HostNetworkDescBuilderCache) + if err != nil { + return nil, err + } + + return builder.(networks_db.NetworkDescBuilder), nil +} + +func (m *NetworkManager) getNetworkCache() (cache.Cache, error) { + cache, err := m.dataManager.SyncCacheGroup.Get(synccache.NetworkSyncCache) + if err != nil { + return nil, err + } + + cache.WaitForReady() + return cache, nil +} diff --git a/pkg/scheduler/data_manager/reserved_algorithm.go b/pkg/scheduler/data_manager/reserved_algorithm.go new file mode 100644 index 0000000000..93c4df2c69 --- /dev/null +++ b/pkg/scheduler/data_manager/reserved_algorithm.go @@ -0,0 +1,253 @@ +package data_manager + +import ( + candidatecache "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/pkg/utils" +) + +type ResAlgorithm interface { + Sum(values []value_t) value_t + Subtract(sum value_t, value value_t) value_t +} + +//////// //////// //////// //////// //////// //////// + +type DefaultResAlgorithm struct { +} + +func (al *DefaultResAlgorithm) Sum(values []value_t) value_t { + var ret int64 = 0 + + for _, value := range values { + if value != nil { + ret += value.(int64) + } + } + + return ret +} + +func (al *DefaultResAlgorithm) Subtract(sum value_t, value value_t) value_t { + if sum == nil { + return nil + } + + if value == nil { + return sum + } + + return sum.(int64) - value.(int64) +} + +//////// //////// //////// //////// //////// //////// + +type GroupResAlgorithmResult struct { + Groups []*candidatecache.GroupCounts +} + +func NewGroupResAlgorithmResult() *GroupResAlgorithmResult { + return &GroupResAlgorithmResult{ + Groups: nil, + } +} + +func (r *GroupResAlgorithmResult) GuestCountOfGroup(groupId string) int64 { + + count := int64(0) + + for _, groupCounts := range r.Groups { + if groupCount, ok := groupCounts.Data[groupId]; ok { + count += groupCount.Count + } + } + + return count +} + +func (r *GroupResAlgorithmResult) ExistsGroup(groupId string) bool { + + for _, groupCounts := range r.Groups { + if groupCount, ok := groupCounts.Data[groupId]; ok { + if groupCount.Count > 0 { + return true + } + } + } + + return false +} + +type GroupResAlgorithm struct { +} + +func (al *GroupResAlgorithm) Sum(values []value_t) value_t { + r := NewGroupResAlgorithmResult() + + for _, value := range values { + if value != nil { + if v, ok := value.(*candidatecache.GroupCounts); ok { + r.Groups = append(r.Groups, v) + } else if v, ok := value.(*GroupResAlgorithmResult); ok { + r.Groups = append(r.Groups, v.Groups...) + } + } + } + + return r +} + +func (al *GroupResAlgorithm) Subtract(sum value_t, value value_t) value_t { + + r := NewGroupResAlgorithmResult() + + if sum != nil { + groups, _ := sum.(*candidatecache.GroupCounts) + if groups != nil && len(groups.Data) > 0 { + r.Groups = append(r.Groups, groups) + } + } + + if value != nil { + grar, _ := value.(*GroupResAlgorithmResult) + if grar != nil && len(grar.Groups) > 0 { + r.Groups = append(r.Groups, grar.Groups...) + } + } + + return r +} + +//////// //////// //////// //////// //////// //////// + +type NetworksResAlgorithmResult struct { + Networks map[string]int +} + +type NetworksResAlgorithm struct { +} + +func (a *NetworksResAlgorithm) Sum(values []value_t) value_t { + countOfNetworks := 0 + for _, value := range values { + if value != nil { + if v, ok := value.(int); ok { + countOfNetworks += v + } + } + } + + return countOfNetworks +} + +func (a *NetworksResAlgorithm) Subtract(sum value_t, value value_t) value_t { + + r := make(map[string]int) + + return r +} + +type GroupGuestRelation struct { + Data map[string]*candidatecache.GroupCount +} + +//////// //////// //////// //////// //////// //////// + +type IsolatedDeviceResAlgorithmResult struct { + IDs map[string]int +} + +func newIsolatedDeviceResAlgorithmResult() *IsolatedDeviceResAlgorithmResult { + return &IsolatedDeviceResAlgorithmResult{ + IDs: make(map[string]int), + } +} + +func (r *IsolatedDeviceResAlgorithmResult) appendDevices(value value_t) { + if devices, ok := value.([]*candidatecache.IsolatedDeviceDesc); ok { + for _, dev := range devices { + r.IDs[dev.ID] = 0 + } + } else if r2, ok := value.(*IsolatedDeviceResAlgorithmResult); ok { + for id := range r2.IDs { + r.IDs[id] = 0 + } + } else if ids, ok := value.([]string); ok { + for _, id := range ids { + r.IDs[id] = 0 + } + } +} + +func (r *IsolatedDeviceResAlgorithmResult) removeDevices(value value_t) { + if devices, ok := value.([]*candidatecache.IsolatedDeviceDesc); ok { + for _, dev := range devices { + delete(r.IDs, dev.ID) + } + } else if r2, ok := value.(*IsolatedDeviceResAlgorithmResult); ok { + for id := range r2.IDs { + delete(r.IDs, id) + } + } else if ids, ok := value.([]string); ok { + for _, id := range ids { + delete(r.IDs, id) + } + } +} + +type IsolatedDeviceResAlgorithm struct { +} + +func (a *IsolatedDeviceResAlgorithm) Sum(values []value_t) value_t { + r := newIsolatedDeviceResAlgorithmResult() + + for _, v := range values { + r.appendDevices(v) + } + + return r +} + +func (al *IsolatedDeviceResAlgorithm) Subtract(sum value_t, value value_t) value_t { + + if sum == nil || value == nil { + return sum + } + + reserved := value.(*IsolatedDeviceResAlgorithmResult) + var isolatedDevices []*candidatecache.IsolatedDeviceDesc + for _, dev := range sum.([]*candidatecache.IsolatedDeviceDesc) { + if _, ok := reserved.IDs[dev.ID]; !ok { + isolatedDevices = append(isolatedDevices, dev) + } + } + + return isolatedDevices +} + +//////// //////// //////// //////// //////// //////// + +var ( + g_defaultResAlgorithm *DefaultResAlgorithm = &DefaultResAlgorithm{} + g_groupResAlgorithm *GroupResAlgorithm = &GroupResAlgorithm{} + g_networksResAlgorithm *NetworksResAlgorithm = &NetworksResAlgorithm{} + g_isolatedDeviceResAlgorithm *IsolatedDeviceResAlgorithm = &IsolatedDeviceResAlgorithm{} +) + +func GetResAlgorithm(res_name string) ResAlgorithm { + switch res_name { + case "Groups": + return g_groupResAlgorithm + case "IsolatedDevices": + return g_isolatedDeviceResAlgorithm + case "FreeCPUCount", "FreeMemSize", "FreeLocalStorageSize": + return g_defaultResAlgorithm + case "Ports": + return g_defaultResAlgorithm + default: + if utils.HasPrefix(res_name, "FreeStorageSize:") { + return g_defaultResAlgorithm + } + + return nil + } +} diff --git a/pkg/scheduler/data_manager/reserved_manager.go b/pkg/scheduler/data_manager/reserved_manager.go new file mode 100644 index 0000000000..4eac40d302 --- /dev/null +++ b/pkg/scheduler/data_manager/reserved_manager.go @@ -0,0 +1,88 @@ +package data_manager + +import ( + "fmt" + "sync" +) + +// Reserved pool manager is mainly to cloud resources to do state +// updates, currently divided into host, baemetal there are three +// network resources management. +type ReservedPoolManager struct { + // store all reserved data + pools map[string]*ReservedPool + stopCh <-chan struct{} + sync.RWMutex +} + +func (pm *ReservedPoolManager) GetPool(name string) (*ReservedPool, error) { + pm.RLock() + pool, ok := pm.pools[name] + if !ok { + return nil, fmt.Errorf("reserved pool %v not found", name) + } + pm.RUnlock() + return pool, nil +} + +func (pm *ReservedPoolManager) addPool(pool *ReservedPool) { + pm.Lock() + // add or update + pm.pools[pool.Name] = pool + pm.Unlock() + + pool.Start() +} + +func NewReservedPoolManager(stopCh <-chan struct{}) *ReservedPoolManager { + pm := &ReservedPoolManager{ + pools: make(map[string]*ReservedPool), + stopCh: stopCh, + } + pm.addPool(NewReservedPool("host", stopCh)) + pm.addPool(NewReservedPool("baremetal", stopCh)) + pm.addPool(NewReservedPool("networks", stopCh)) + return pm +} + +func (pm *ReservedPoolManager) SearchReservedPoolBySessionID(sessionId string) ( + *ReservedPool, error) { + for _, pool := range pm.pools { + if pool.GetSessionItem(sessionId) != nil { + return pool, nil + } + } + return nil, fmt.Errorf("session id: %v not found", sessionId) +} + +func (pm *ReservedPoolManager) InSession(resType string, candidateId string) bool { + if pool, err := pm.GetPool(resType); err == nil { + return pool.InSession(candidateId) + } + return false +} + +func (pm *ReservedPoolManager) RemoveSession(sessionId string) bool { + for _, pool := range pm.pools { + if pool.RemoveSession(sessionId) { + return true + } + } + return false +} + +func ReservedSubtract(key string, value value_t, reserved value_t) value_t { + var al ResAlgorithm = GetResAlgorithm(key) + if al != nil { + return al.Subtract(value, reserved) + } + return value +} + +func ReservedSum(key string, values []value_t) value_t { + var al ResAlgorithm = GetResAlgorithm(key) + if al != nil { + return al.Sum(values) + } + return nil +} diff --git a/pkg/scheduler/data_manager/reserved_pool.go b/pkg/scheduler/data_manager/reserved_pool.go new file mode 100644 index 0000000000..d8fad90b78 --- /dev/null +++ b/pkg/scheduler/data_manager/reserved_pool.go @@ -0,0 +1,306 @@ +package data_manager + +import ( + "sync" + "time" + + "github.com/yunionio/pkg/util/wait" +) + +const ( + SessionExpiredTime = 30 * 60 // Seconds +) + +type value_t interface{} + +type KeyValue interface { + Get(key string) interface{} +} + +// ReservedItem +type ReservedItem struct { + CandidateId string + data map[string]value_t + sync.RWMutex +} + +func NewReservedItem(candidateID string) *ReservedItem { + return &ReservedItem{ + CandidateId: candidateID, + data: make(map[string]value_t), + } +} + +func (item *ReservedItem) Get(key string, def_value value_t) value_t { + item.RLock() + defer item.RUnlock() + value, ok := item.data[key] + if ok { + return value + } + return def_value +} + +func (item *ReservedItem) Set(key string, value value_t) { + item.set(key, value) +} + +func (item *ReservedItem) set(key string, value value_t) { + item.Lock() + item.Unlock() + + item.data[key] = value +} + +func (item *ReservedItem) SetAll(values map[string]interface{}) { + for key, value := range values { + if value != nil { + item.set(key, value) + } + } +} + +func (item *ReservedItem) GetAll() (values map[string]interface{}) { + item.RLock() + defer item.RUnlock() + values = make(map[string]interface{}) + for key, value := range item.data { + values[key] = value + } + return values +} + +func (item *ReservedItem) ToDict() map[string]interface{} { + item.RLock() + defer item.RUnlock() + + dict := make(map[string]interface{}) + + for id, value := range item.data { + dict[id] = value + } + + return dict +} + +type SessionItem struct { + Time time.Time + data map[string]*ReservedItem // candidateID -> ReservedItem + sync.RWMutex +} + +func (si *SessionItem) get(candidateID string) *ReservedItem { + si.RLock() + defer si.RUnlock() + reservedItem := si.data[candidateID] // *ReservedItem + return reservedItem +} + +func (si *SessionItem) set(candidateID string, reservedItem *ReservedItem) { + si.Lock() + defer si.Unlock() + si.data[candidateID] = reservedItem +} + +func (si *SessionItem) AllCandidateIDs() []string { + si.RLock() + defer si.RUnlock() + candidateIds := []string{} + for candidateID := range si.data { + candidateIds = append(candidateIds, candidateID) + } + return candidateIds +} + +func NewSessionItem() *SessionItem { + return &SessionItem{ + Time: time.Now(), + data: make(map[string]*ReservedItem), + } +} + +func (si *SessionItem) ToDict() map[string]interface{} { + si.RLock() + defer si.RUnlock() + + dict := make(map[string]interface{}) + + dict["time"] = si.Time.Local().Format("2006-01-02 15:04:05") + for key, rItem := range si.data { + dict[key] = rItem.ToDict() + } + + return dict +} + +type CandidateItem struct { + candidateID string + data map[string]*ReservedItem // sessionID -> ReservedItem + result *ReservedItem + dirty bool + sync.RWMutex +} + +func (ci *CandidateItem) get(sessionID string) *ReservedItem { + ci.RLock() + defer ci.RUnlock() + reservedItem := ci.data[sessionID] + return reservedItem +} + +func (ci *CandidateItem) set(sessionID string, reservedItem *ReservedItem) { + ci.Lock() + defer ci.Unlock() + ci.data[sessionID] = reservedItem + ci.dirty = true +} + +func NewCandidateItem(candidateID string) *CandidateItem { + return &CandidateItem{ + candidateID: candidateID, + data: make(map[string]*ReservedItem), + dirty: true, + } +} + +func (ci *CandidateItem) caculate() *ReservedItem { + ci.RLock() + defer ci.RUnlock() + data := make(map[string][]value_t) + for _, reservedItem := range ci.data { + for key, value := range reservedItem.data { + values, _ := data[key] + data[key] = append(values, value) + } + } + reservedItem := NewReservedItem(ci.candidateID) + for key, values := range data { + reservedItem.Set(key, ReservedSum(key, values)) + } + return reservedItem +} + +type ReservedPool struct { + Name string + sessionDict map[string]*SessionItem + candidateDict map[string]*CandidateItem + data map[string]value_t + + stopCh <-chan struct{} + sync.RWMutex +} + +func NewReservedPool(name string, stopCh <-chan struct{}) *ReservedPool { + return &ReservedPool{ + Name: name, + sessionDict: make(map[string]*SessionItem), + candidateDict: make(map[string]*CandidateItem), + data: make(map[string]value_t), + stopCh: stopCh, + } +} + +func (pool *ReservedPool) Start() { + go wait.Until(pool.checkSessionExpires, time.Duration(10)*time.Second, pool.stopCh) +} + +func (pool *ReservedPool) checkSessionExpires() { + pool.Lock() + defer pool.Unlock() + now := time.Now() + for sessionID, sessionItem := range pool.sessionDict { + if now.Sub(sessionItem.Time).Seconds() > SessionExpiredTime { + pool.removeSession(sessionID) + } + } +} + +func (pool *ReservedPool) Add(sessionID string, candidateID string, + reservedItem *ReservedItem) { + pool.Lock() + defer pool.Unlock() + session_item, ok := pool.sessionDict[sessionID] + if !ok { + session_item = NewSessionItem() + pool.sessionDict[sessionID] = session_item + } + session_item.set(candidateID, reservedItem) + candidateItem, ok := pool.candidateDict[candidateID] + if !ok { + candidateItem = NewCandidateItem(candidateID) + pool.candidateDict[candidateID] = candidateItem + } + candidateItem.set(sessionID, reservedItem) +} + +func (pool *ReservedPool) GetReservedItem(candidateID string) *ReservedItem { + pool.RLock() + defer pool.RUnlock() + candidateItem, ok := pool.candidateDict[candidateID] + if !ok { + return nil + } + + if candidateItem.dirty { + candidateItem.result = candidateItem.caculate() + candidateItem.dirty = false + } + return candidateItem.result +} + +func (pool *ReservedPool) GetSessionItem(sessionID string) *SessionItem { + pool.RLock() + defer pool.RUnlock() + if sessionItem, ok := pool.sessionDict[sessionID]; ok { + return sessionItem + } + return nil +} + +func (pool *ReservedPool) RemoveSession(sessionID string) bool { + pool.Lock() + defer pool.Unlock() + + return pool.removeSession(sessionID) +} + +func (pool *ReservedPool) removeSession(sessionID string) bool { + if sessionItem, ok := pool.sessionDict[sessionID]; ok { + delete(pool.sessionDict, sessionID) + if len(pool.sessionDict) == 0 { + pool.candidateDict = make(map[string]*CandidateItem) + } else { + for _, candidateId := range sessionItem.AllCandidateIDs() { + if candidateItem, ok := pool.candidateDict[candidateId]; ok { + if _, ok := candidateItem.data[sessionID]; ok { + delete(candidateItem.data, sessionID) + } + } + } + } + return true + } + return false +} + +func (pool *ReservedPool) InSession(candidateId string) bool { + pool.RLock() + defer pool.RUnlock() + if candidateItem, ok := pool.candidateDict[candidateId]; ok { + return len(candidateItem.data) > 0 + } + return false +} + +func (pool *ReservedPool) ToDict() interface{} { + pool.RLock() + defer pool.RUnlock() + + data := make(map[string]interface{}) + + for sessionId, sessionItem := range pool.sessionDict { + data[sessionId] = sessionItem.ToDict() + } + + return data +} diff --git a/pkg/scheduler/db/db.go b/pkg/scheduler/db/db.go new file mode 100644 index 0000000000..73d0c160a6 --- /dev/null +++ b/pkg/scheduler/db/db.go @@ -0,0 +1,23 @@ +package db + +import ( + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/mysql" + + "github.com/yunionio/log" +) + +var DB *gorm.DB + +func Init(dialect string, args ...interface{}) error { + if DB == nil { + db, err := gorm.Open(dialect, args...) + if err != nil { + return err + } + DB = db + return nil + } + log.Warningf("DB: %s , Conn: %v already connected...", dialect, args) + return nil +} diff --git a/pkg/scheduler/db/models/aggregate.go b/pkg/scheduler/db/models/aggregate.go new file mode 100644 index 0000000000..acee9bfdd5 --- /dev/null +++ b/pkg/scheduler/db/models/aggregate.go @@ -0,0 +1,33 @@ +package models + +import ( + "encoding/json" + + "github.com/jinzhu/gorm" +) + +type Aggregate struct { + StandaloneModel + DefaultStrategy string `json:"default_strategy" gorm:"not null"` +} + +func (c Aggregate) TableName() string { + return aggregatesTable +} + +func (c Aggregate) String() string { + s, _ := json.Marshal(c) + return string(s) +} + +func NewAggregateResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Aggregate{} + } + models := func() interface{} { + aggregates := []Aggregate{} + return &aggregates + } + + return newResource(db, aggregatesTable, model, models) +} diff --git a/pkg/scheduler/db/models/aggregate_host.go b/pkg/scheduler/db/models/aggregate_host.go new file mode 100644 index 0000000000..7ba8fecf7d --- /dev/null +++ b/pkg/scheduler/db/models/aggregate_host.go @@ -0,0 +1,42 @@ +package models + +import ( + "encoding/json" + + "github.com/jinzhu/gorm" +) + +type AggregateHost struct { + JointBaseModel + HostID string `json:"host_id" gorm:"column:host_id;not null"` + AggregateID string `json:"schedtag_id" gorm:"column:schedtag_id;not null"` +} + +func (c AggregateHost) TableName() string { + return aggregateHostsTable +} + +func (c AggregateHost) String() string { + s, _ := json.Marshal(c) + return string(s) +} + +func (c AggregateHost) Aggregate() (*Aggregate, error) { + a, err := FetchByID(Aggregates, c.AggregateID) + if err != nil { + return nil, err + } + return a.(*Aggregate), nil +} + +func NewAggregateHostResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &AggregateHost{} + } + models := func() interface{} { + aggregate_hosts := []AggregateHost{} + return &aggregate_hosts + } + + return newResource(db, aggregateHostsTable, model, models) +} diff --git a/pkg/scheduler/db/models/baremetal.go b/pkg/scheduler/db/models/baremetal.go new file mode 100644 index 0000000000..1d5b6e3381 --- /dev/null +++ b/pkg/scheduler/db/models/baremetal.go @@ -0,0 +1,54 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type Baremetal struct { + StandaloneModel + Status string `json:"status" gorm:"not null"` + Enabled bool `json:"enabled" gorm:"not null"` + CliGUID string `json:"cli_guid,omitempty"` + AgentID string `json:"agent_id,omitempty"` + CPUCount int `json:"cpu_count,omitempty"` + NodeCount int `json:"node_count,omitempty"` + CPUDesc string `json:"cpu_desc,omitempty"` + CPUMHZ int `json:"cpu_mhz,omitempty"` + MemSize int `json:"mem_size,omitempty"` + + StorageSize int `json:"storage_size,omitempty"` + StorageType string `json:"storage_type,omitempty"` + StorageDriver string `json:"storage_driver,omitempty"` + StorageInfo string `json:"storage_info,omitempty"` + + IpmiInfo string `json:"ipmi_info,omitempty" gorm:"type:text"` + + Rack string `json:"rack,omitempty"` + Slots string `json:"slots,omitempty"` + + ServerID string `json:"server_id,omitempty"` + UseCount int `json:"use_count,omitempty"` + + PoolID string `json:"pool_id,omitempty"` +} + +func (b Baremetal) TableName() string { + return baremetalsTable +} + +func (b Baremetal) String() string { + str, _ := JsonString(b) + return str +} + +func NewBaremetalResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Baremetal{} + } + models := func() interface{} { + baremetals := []Baremetal{} + return &baremetals + } + + return newResource(db, baremetalsTable, model, models) +} diff --git a/pkg/scheduler/db/models/baremetal_networks.go b/pkg/scheduler/db/models/baremetal_networks.go new file mode 100644 index 0000000000..1f61089ab9 --- /dev/null +++ b/pkg/scheduler/db/models/baremetal_networks.go @@ -0,0 +1,85 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + BaremetalNetworksResourceName = "baremetalnetworks" +) + +type BaremetalNetwork struct { + StandaloneModel + BaremetalID string `json:"baremetal_id,omitempty" gorm:"column:Baremetal_id;not null"` + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + MacAddr string `json:"mac_addr" gorm:"column:mac_addr;not null"` + IpAddr string `json:"ip_addr,omitempty" gorm:"column:ip_addr"` + Ip6Addr string `json:"ip6_addr" gorm:"column:ip6_addr"` + Driver string `json:"driver" gorm:"column:driver"` + BwLimit int64 `json:"bw_limit" gorm:"column:bw_limit;not null"` + Index int `json:"index" gorm:"column:index;not null"` + Virtual int `json:"virtual" gorm:"column:virtual"` + IfName string `json:"if_name,omitempty" gorm:"column:if_name"` + MappingIpAddr string `json:"mapping_ip_addr" gorm:"column:mapping_ip_addr"` +} + +func (n BaremetalNetwork) TableName() string { + return baremetalNetworksTable +} + +func (n BaremetalNetwork) String() string { + s, _ := JsonString(n) + return string(s) +} + +func NewBaremetalNetworksResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &BaremetalNetwork{} + } + models := func() interface{} { + baremetalNetworks := []BaremetalNetwork{} + return &baremetalNetworks + } + + return newResource(db, baremetalNetworksTable, model, models) +} + +type BaremetalNicCount struct { + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c BaremetalNicCount) First() string { + return c.NetworkID +} + +func (c BaremetalNicCount) Second() int { + return c.Count +} +func BaremetalNicCounts() ([]BaremetalNicCount, error) { + counts := []BaremetalNicCount{} + err := BaremetalNetworks.DB().Table(baremetalNetworksTable). + Select("network_id,count(*) as count"). + Where("deleted=0"). + Group("network_id"). + Scan(&counts).Error + return counts, err +} + +type BaremetalNicCounti struct { + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c BaremetalNicCounti) First() int { + return c.Count +} +func BaremetalNicCountsWithNetworkID(networkID string) (BaremetalNicCounti, error) { + counts := BaremetalNicCounti{0} + err := BaremetalNetworks.DB().Table(baremetalNetworksTable). + Select("count(*) as count"). + Where(fmt.Sprintf("network_id = '%s' and deleted=0", networkID)). + Scan(&counts).Error + return counts, err +} diff --git a/pkg/scheduler/db/models/baremetalagent.go b/pkg/scheduler/db/models/baremetalagent.go new file mode 100644 index 0000000000..31bec2e0c4 --- /dev/null +++ b/pkg/scheduler/db/models/baremetalagent.go @@ -0,0 +1,35 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type BaremetalAgent struct { + StandaloneModel + AccessIP string `json:"access_ip" gorm:"not null"` + ManagerURI string `json:"manager_uri,omitempty"` + Status string `json:"status" gorm:"not null"` + ZoneID string `json:"zone_id,omitempty"` + Version string `json:"version,omitempty"` +} + +func (b BaremetalAgent) TableName() string { + return baremetalAgentsTable +} + +func (b BaremetalAgent) String() string { + s, _ := JsonString(b) + return s +} + +func NewBaremetalAgentResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &BaremetalAgent{} + } + models := func() interface{} { + agents := []BaremetalAgent{} + return &agents + } + + return newResource(db, baremetalAgentsTable, model, models) +} diff --git a/pkg/scheduler/db/models/base.go b/pkg/scheduler/db/models/base.go new file mode 100644 index 0000000000..403a924e3a --- /dev/null +++ b/pkg/scheduler/db/models/base.go @@ -0,0 +1,307 @@ +package models + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/jinzhu/gorm" +) + +type objectFunc func() interface{} + +type BaseModel struct { + CreatedAt time.Time `json:"created_at" gorm:"column:created_at;type:datetime" sql:"DEFAULT:NULL"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at;type:datetime" sql:"DEFAULT:NULL"` + DeletedAt time.Time `json:"deleted_at" gorm:"column:deleted_at;type:datetime" sql:"DEFAULT:NULL"` + Deleted bool `json:"deleted" gorm:"column:deleted;not null;index" sql:"DEFAULT:false"` +} + +type StandaloneModel struct { + BaseModel + ID string `json:"id" gorm:"primary_key;column:id;type:varchar(36) CHARACTER SET ascii"` + Name string `json:"name" gorm:"column:name;type:varchar(128) CHARACTER SET utf8"` + Description string `json:"description,omitempty" gorm:"column:description"` +} + +func (m *StandaloneModel) UUID() string { + return m.ID +} + +type JointBaseModel struct { + BaseModel + RowID string `json:"row_id" gorm:"primary_key;column:row_id"` +} + +func (m *JointBaseModel) UUID() string { + return m.RowID +} + +type VirtualResourceModel struct { + StandaloneModel + Status string `json:"status" gorm:"column:status;not null"` + TenantID string `json:"tenant_id" gorm:"column:tenant_id;not null"` + UserID string `json:"user_id" gorm:"column:user_id;not null"` + BillingType string `json:"billing_type" gorm:"column:billing_type"` + IsSystem bool `json:"is_system" gorm:"column:is_system"` + PendingDeletedAt time.Time `json:"pending_deleted_at" gorm:"column:pending_deleted_at;type:datetime" sql:"DEFAULT:NULL"` + PendingDeleted bool `json:"pending_deleted" gorm:"column:pending_deleted;not null;index" sql:"DEFAULT:false"` +} + +type SharableVirtualResourceModel struct { + VirtualResourceModel + IsPublic bool `json:"is_public" gorm:"column:is_public;not null"` +} + +type resource struct { + db *gorm.DB + tableName string + getModel objectFunc + getModels objectFunc +} + +func newResource(db *gorm.DB, tbl string, model, models objectFunc) (Resourcer, error) { + if db == nil { + return nil, fmt.Errorf("gorm db is nil") + } + r := new(resource) + r.db = db + r.tableName = tbl + r.getModel = model + r.getModels = models + return r, nil +} + +func (r *resource) DB() *gorm.DB { + return r.db +} + +func (r *resource) TableName() string { + return r.tableName +} + +func (r *resource) Model() interface{} { + return r.getModel() +} + +func (r *resource) Models() interface{} { + return r.getModels() +} + +func JsonString(obj interface{}) (string, error) { + bytes, err := json.Marshal(obj) + if err != nil { + return "", err + } + return string(bytes), nil +} + +func FetchByID(r Resourcer, id string) (interface{}, error) { + cond := map[string]interface{}{ + "deleted": 0, + "id": id, + } + obj := r.Model() + if err := r.DB().Where(condition2String(cond)).First(obj).Error; err != nil { + return nil, err + } + return obj, nil +} + +func rowsWithCondIn(r Resourcer, key string, set []string, cond map[string]interface{}) (*sql.Rows, error) { + return r.DB().Table(r.TableName()).Where(condition2String(cond)). + Where(fmt.Sprintf("%s in ('%s')", key, strings.Join(set, "','"))).Rows() +} + +func rowsNotDeletedIn(r Resourcer, key string, set []string) (*sql.Rows, error) { + cond := map[string]interface{}{ + "deleted": false, + } + return rowsWithCondIn(r, key, set, cond) +} + +func rowsNotDeletedInWithCond(r Resourcer, key string, set []string, cond map[string]interface{}) (*sql.Rows, error) { + cond["deleted"] = false + return rowsWithCondIn(r, key, set, cond) +} + +func virtualResourceRowsNotDeletedIn(r Resourcer, key string, set []string) (*sql.Rows, error) { + cond := map[string]interface{}{ + "deleted": false, + "pending_deleted": false, + } + return rowsWithCondIn(r, key, set, cond) +} + +func rowsToArray(r Resourcer, rows *sql.Rows) ([]interface{}, error) { + columns, _ := rows.Columns() + + objs := make([]interface{}, 0, len(columns)) + for rows.Next() { + obj := r.Model() + err := r.DB().ScanRows(rows, obj) + if err != nil { + return nil, err + } + objs = append(objs, obj) + } + return objs, nil +} + +func FetchByIDsWithKey(r Resourcer, key string, ids []string) ([]interface{}, error) { + rows, err := rowsNotDeletedIn(r, key, ids) + if err != nil { + return nil, err + } + return rowsToArray(r, rows) +} + +func FetchByIDsWithKeyAndCond(r Resourcer, key string, ids []string, cond map[string]interface{}) ([]interface{}, error) { + rows, err := rowsNotDeletedInWithCond(r, key, ids, cond) + if err != nil { + return nil, err + } + return rowsToArray(r, rows) +} + +func FetchByIDs(r Resourcer, ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(r, "id", ids) +} + +func FetchByHostIDs(r Resourcer, ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(r, "host_id", ids) +} + +func FetchGuestByIDs(ids []string) ([]interface{}, error) { + return FetchByIDs(Guests, ids) +} + +func FetchGuestByHostIDs(ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(Guests, "host_id", ids) +} + +func FetchGuestByHostIDsWithCond(ids []string, cond map[string]interface{}) ([]interface{}, error) { + return FetchByIDsWithKeyAndCond(Guests, "host_id", ids, cond) +} + +func FetchHostByIDs(ids []string) ([]interface{}, error) { + return FetchByIDs(Hosts, ids) +} + +func FetchDiskByIDs(ids []string) ([]interface{}, error) { + return FetchByIDs(Disks, ids) +} + +func FetchGroupByIDs(ids []string) ([]interface{}, error) { + return FetchByIDs(Groups, ids) +} + +func FetchByWireIDs(r Resourcer, ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(r, "wire_id", ids) +} + +func FetchByBaremetalIDs(r Resourcer, ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(r, "baremetal_id", ids) +} + +func FetchByGuestIDs(r Resourcer, ids []string) ([]interface{}, error) { + return FetchByIDsWithKey(r, "guest_id", ids) +} + +func AllWithDeleted(r Resourcer) ([]interface{}, error) { + return AllWithCond(r, map[string]interface{}{}) +} + +func All(r Resourcer) ([]interface{}, error) { + cond := map[string]interface{}{ + "deleted": 0, + } + return AllWithCond(r, cond) +} + +func AllWithCond(r Resourcer, cond map[string]interface{}) ([]interface{}, error) { + rows, err := r.DB().Model(r.Model()).Where(condition2String(cond)).Rows() + if err != nil { + return nil, err + } + return rowsToArray(r, rows) +} + +func AllIDs(r Resourcer) ([]string, error) { + cond := map[string]interface{}{ + "deleted": 0, + } + return AllIDsWithCond(r, cond) +} + +func AllHostIDs() ([]string, error) { + cond := map[string]interface{}{ + "deleted": 0, + "host_type!": "baremetal", + } + return AllIDsWithCond(Hosts, cond) +} + +func AllBaremetalIDs() ([]string, error) { + cond := map[string]interface{}{ + "deleted": 0, + "host_type": "baremetal", + } + return AllIDsWithCond(Hosts, cond) +} + +func condition2String(cond map[string]interface{}) string { + result := make([]string, 0) + for key, value := range cond { + if _, ok := value.(string); ok { + result = append(result, fmt.Sprintf("%s='%s'", key, value.(string))) + } else if _, ok := value.(int); ok { + result = append(result, fmt.Sprintf("%s=%d", key, value.(int))) + } else if _, ok := value.(int64); ok { + result = append(result, fmt.Sprintf("%s=%d", key, value.(int64))) + } else if _, ok := value.(bool); ok { + result = append(result, fmt.Sprintf("%s=%v", key, value.(bool))) + } + } + return strings.Join(result, " and ") +} + +func AllIDsWithCond(r Resourcer, cond map[string]interface{}) ([]string, error) { + rows, err := r.DB().Table(r.TableName()).Where(condition2String(cond)).Select("id").Rows() + if err != nil { + return nil, err + } + + defer rows.Close() + + ids := []string{} + for rows.Next() { + var id string + rows.Scan(&id) + ids = append(ids, id) + } + return ids, nil +} + +type StatusOfHost struct { + ID string `json:"id" gorm:"primary_key;column:id;type:varchar(36) CHARACTER SET ascii"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at;type:datetime" sql:"DEFAULT:NULL"` +} + +func AllHostStatus(isBaremetal bool) ([]StatusOfHost, error) { + whereState := "deleted=0 and host_type%s" + if isBaremetal { + whereState = fmt.Sprintf(whereState, "='baremetal'") + } else { + whereState = fmt.Sprintf(whereState, "!='baremtal'") + } + + status := make([]StatusOfHost, 0) + err := Hosts.DB().Table(Hosts.TableName()). + Select("id, updated_at"). + Where(whereState). + Scan(&status).Error + return status, err +} diff --git a/pkg/scheduler/db/models/base_test.go b/pkg/scheduler/db/models/base_test.go new file mode 100644 index 0000000000..4a2deaf646 --- /dev/null +++ b/pkg/scheduler/db/models/base_test.go @@ -0,0 +1,192 @@ +package models + +import ( + "flag" + "fmt" + "testing" +) + +var ( + dialect = flag.String("db-dialect", "mysql", "db dialect") + dbURL = flag.String("db-url", "root:root@tcp(127.0.0.1:3306)/yunioncloud?charset=utf8&parseTime=True", "db url") +) + +func init() { + flag.Parse() + err := Init(*dialect, *dbURL) + if err != nil { + panic(fmt.Errorf("Test init error: %v", err)) + } +} + +func TestQuery(t *testing.T) { + ids, err := AllIDs(Guests) + if err != nil { + t.Fatal(err) + } + t.Logf("%v: , length: %d", ids, len(ids)) + + objs, err := All(Guests) + if err != nil { + t.Fatal(err) + } + t.Logf("%v: , length: %d", objs[1], len(objs)) +} + +func TestQueryIn(t *testing.T) { + ids := []string{"000ea33f-f751-4f7f-85ef-958676a5e78b", "000f5af0-ee2b-4678-a7ce-a93987f2a87d"} + bms, err := FetchByIDs(Baremetals, ids) + if err != nil { + t.Fatal(err) + } + t.Logf("Bms: %v, length: %d", bms, len(bms)) +} + +func TestFetchByHostIDs(t *testing.T) { + ids := []string{"7916bd54-40b5-4465-842c-832e4e42313f"} + objs, err := FetchByHostIDs(Guests, ids) + if err != nil { + t.Fatal(err) + } + t.Logf("Guests: %v, length: %d", objs, len(objs)) +} + +func TestHostStorage(t *testing.T) { + ss, err := All(HostStorages) + if err != nil { + t.Fatal(err) + } + t.Logf("HostStorages: %v, length: %d", ss, len(ss)) +} + +func TestStorage(t *testing.T) { + ss, err := All(Storages) + if err != nil { + t.Fatal(err) + } + for _, s := range ss { + storage := s.(*Storage) + if storage.ZoneID != "" { + t.Logf("Storage: %#v", storage) + } + } +} + +func TestGroup(t *testing.T) { + groups, err := All(Groups) + if err != nil { + t.Fatal(err) + } + t.Logf("groups: %v, len: %d", groups, len(groups)) +} + +func TestGroupGuest(t *testing.T) { + groups, err := All(GroupGuests) + if err != nil { + t.Fatal(err) + } + t.Logf("group guests: %v, len: %d", groups[0], len(groups)) +} + +func TestMetadata(t *testing.T) { + metadatas, err := AllWithDeleted(Metadatas) + if err != nil { + t.Fatal(err) + } + t.Logf("metadata: %v, len: %d", metadatas[0], len(metadatas)) +} + +func TestIsolatedDev(t *testing.T) { + devs, err := All(IsolatedDevices) + if err != nil { + t.Fatal(err) + } + t.Logf("IsolatedDevices: %+v, len: %d", devs[0], len(devs)) +} + +func TestDisk(t *testing.T) { + disks, err := All(Disks) + if err != nil { + t.Fatal(err) + } + t.Logf("Disks: %v, len: %d", disks[0], len(disks)) + capas, err := GetStorageCapacities([]string{"d0205a6a-b8aa-4365-ba5e-1003104006a8"}) + if err != nil { + t.Fatal(err) + } + t.Logf("Capacities: %v, len: %d", capas, len(capas)) +} + +func TestGuestTenant(t *testing.T) { + hostids := []string{"7916bd54-40b5-4465-842c-832e4e42313f"} + ts, err := ResidentTenantsInHosts(hostids) + if err != nil { + t.Fatal(err) + } + t.Logf("tenants: %v, len: %d", ts, len(ts)) +} + +func TestFetchMetadatas(t *testing.T) { + hostids := []string{"7916bd54-40b5-4465-842c-832e4e42313f"} + serverids := []string{"fffd63c7-b0ef-446b-bfbf-ad05e1cefe2a"} + hostMetadataNames := []string{"dynamic_load_cpu_percent", "dynamic_load_io_util", + "enable_sriov", "bridge_driver"} + hostMetadataNames = append(hostMetadataNames, HostExtraFeature...) + hostMetadatas, err := FetchMetadatas(HostResourceName, hostids, hostMetadataNames) + if err != nil { + t.Fatal(err) + } + t.Logf("hostMetadatas: %v", hostMetadatas) + guestMetadataNames := []string{"app_tags"} + guestMetadatas, err := FetchMetadatas(GuestResourceName, serverids, guestMetadataNames) + if err != nil { + t.Fatal(err) + } + t.Logf("guestMetadatas: %v", guestMetadatas) +} + +func TestGuestDisk(t *testing.T) { + disks, err := All(GuestDisks) + if err != nil { + t.Fatal(err) + } + t.Logf("Disks: %v, len: %d", disks[0], len(disks)) + gst, err := FetchByID(Guests, "b4438e03-c6c2-4f88-8b95-11efea0300c4") + if err != nil { + t.Fatal(err) + } + size, err := gst.(*Guest).DiskSize() + if err != nil { + t.Fatal(err) + } + t.Logf("DiskSize: %d", size) +} + +func BenchmarkQueryUseScanRow(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := All(Guests) + if err != nil { + b.Fatal(err) + } + } +} + +func allTest(r Resourcer) (interface{}, error) { + cond := map[string]interface{}{ + "deleted": false, + } + objs := r.Models() + if err := r.DB().Where(cond).Find(objs).Error; err != nil { + return nil, err + } + return objs, nil +} + +func BenchmarkQueryUseSlice(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := allTest(Guests) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/scheduler/db/models/cluster.go b/pkg/scheduler/db/models/cluster.go new file mode 100644 index 0000000000..736dc865f7 --- /dev/null +++ b/pkg/scheduler/db/models/cluster.go @@ -0,0 +1,39 @@ +package models + +import ( + "encoding/json" + + "github.com/jinzhu/gorm" +) + +type Cluster struct { + StandaloneModel + HostIPStart string `json:"host_ip_start" gorm:"not null"` + HostIPEnd string `json:"host_ip_end" gorm:"not null"` + HostNetmask int `json:"host_netmask,omitempty"` + HostGateway string `json:"host_gateway,omitempty"` + HostDNS string `json:"host_dns,omitempty"` + ScheduleRank int `json:"schedule_rank,omitempty"` + ZoneID string `json:"zone_id" gorm:"not null"` +} + +func (c Cluster) TableName() string { + return clustersTable +} + +func (c Cluster) String() string { + s, _ := json.Marshal(c) + return string(s) +} + +func NewClusterResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Cluster{} + } + models := func() interface{} { + clusters := []Cluster{} + return &clusters + } + + return newResource(db, clustersTable, model, models) +} diff --git a/pkg/scheduler/db/models/default.go b/pkg/scheduler/db/models/default.go new file mode 100644 index 0000000000..48c0d06325 --- /dev/null +++ b/pkg/scheduler/db/models/default.go @@ -0,0 +1,124 @@ +package models + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/db" +) + +const ( + hostsTable = "hosts_tbl" + clustersTable = "clusters_tbl" + guestsTable = "guests_tbl" + + baremetalsTable = "baremetals_tbl" + baremetalAgentsTable = "baremetalagents_tbl" + baremetalNetworksTable = "baremetalnetworks_tbl" + + storageTable = "storages_tbl" + hostStorageTable = "hoststorages_tbl" + + groupGuestTable = "guestgroups_tbl" + groupTable = "groups_tbl" + groupNetworksTable = "groupnetworks_tbl" + + metadataTable = "metadata_tbl" + + isolatedDeviceTable = "isolated_devices_tbl" + + disksTable = "disks_tbl" + guestDiskTable = "guestdisks_tbl" + guestNetworksTable = "guestnetworks_tbl" + + aggregatesTable = "aggregates_tbl" + aggregateHostsTable = "aggregate_hosts_tbl" + + networksTable = "networks_tbl" + netinterfacesTable = "netinterfaces_tbl" + + wiresTable = "wires_tbl" + hostWiresTable = "hostwires_tbl" + + reserveDipsTable = "reservedips_tbl" +) + +var ( + Hosts Resourcer + HostWires Resourcer + + Clusters Resourcer + Guests Resourcer + + Baremetals Resourcer + BaremetalAgents Resourcer + BaremetalNetworks Resourcer + + Storages Resourcer + HostStorages Resourcer + + GroupGuests Resourcer + Groups Resourcer + GroupNetworks Resourcer + + Metadatas Resourcer + + IsolatedDevices Resourcer + Disks Resourcer + + GuestDisks Resourcer + GuestNetworks Resourcer + + Aggregates Resourcer + AggregateHosts Resourcer + + Networks Resourcer + NetInterfaces Resourcer + + Wires Resourcer + + ReserveDipsNerworks Resourcer +) + +func Init(dialect string, args ...interface{}) error { + err := db.Init(dialect, args...) + if err != nil { + return err + } + Hosts, _ = NewHostResource(db.DB) + Clusters, _ = NewClusterResource(db.DB) + Guests, _ = NewGuestResource(db.DB) + + Baremetals, _ = NewBaremetalResource(db.DB) + BaremetalAgents, _ = NewBaremetalAgentResource(db.DB) + + Storages, _ = NewStorageResource(db.DB) + HostStorages, _ = NewHostStorageResource(db.DB) + + Groups, _ = NewGroupResource(db.DB) + GroupGuests, _ = NewGroupGuestResource(db.DB) + + Metadatas, _ = NewMetadataResource(db.DB) + + IsolatedDevices, _ = NewIsolatedDeviceResource(db.DB) + + Disks, _ = NewDiskResource(db.DB) + GuestDisks, _ = NewGuestDiskResource(db.DB) + + Aggregates, _ = NewAggregateResource(db.DB) + AggregateHosts, _ = NewAggregateHostResource(db.DB) + + Networks, _ = NewNetworksResource(db.DB) + NetInterfaces, _ = NewNetInterfacesResource(db.DB) + Wires, _ = NewWiresResource(db.DB) + HostWires, _ = NewHostWiresResource(db.DB) + GuestNetworks, _ = NewGuestNetworksResource(db.DB) + GroupNetworks, _ = NewGroupNetworksResource(db.DB) + BaremetalNetworks, _ = NewBaremetalNetworksResource(db.DB) + ReserveDipsNerworks, _ = NewReserveDipsNetworksResource(db.DB) + return nil +} + +func DBValid() bool { + if db.DB == nil { + return false + } + return true +} diff --git a/pkg/scheduler/db/models/disk.go b/pkg/scheduler/db/models/disk.go new file mode 100644 index 0000000000..7b73f19d4c --- /dev/null +++ b/pkg/scheduler/db/models/disk.go @@ -0,0 +1,93 @@ +package models + +import ( + "github.com/jinzhu/gorm" + + "github.com/yunionio/pkg/utils" +) + +const ( + DiskResourceName = "disk" + + DiskInit = "init" + DiskRebuild = "rebuild" + DiskAllocFailed = "alloc_failed" + DiskStartAlloc = "start_alloc" + DiskAllocating = "allocating" + DiskReady = "ready" + DiskFrozen = "frozen" + DiskDealloc = "deallocating" + DiskDeallocFailed = "dealloc_failed" + + DiskStartSave = "start_save" + DiskSaving = "saving" + + DiskStartResize = "start_resize" + DiskResizing = "resizing" + + DiskStartMigrate = "start_migrate" + DiskPostMigrate = "post_migrate" + DiskMigrating = "migrating" + + TakeMebsSnapshot = "take_mebs_snapshot" + TakeMebsSnapshotFailed = "take_mebs_snapshot_failed" + ApplyMebsSnapshot = "apply_mebs_snapshot" + ApplyMebsSnapshotFailed = "apply_mebs_snapshot_failed" + CloneMebsSnapshot = "clone_mebs_snapshot" + CloneMebsSnapshotFailed = "clone_mebs_snapshot_failed" + PerformMebsBackup = "perform_mebs_backup" + PerformMebsBackupFailed = "perform_mebs_backup_failed" + RestoreMebsBackup = "restore_mebs_backup" + RestoreMebsBackupFailed = "restore_mebs_backup_failed" + SaveMebsTemplate = "save_mebs_template" + SaveMebsTemplateFailed = "save_mebs_template_failed" + + ActionThrottle = "throttle" + ActionFreeze = "freeze" + ActionUnfreeze = "unfreeze" +) + +var ( + IOThrottleActions = []string{ActionThrottle, ActionFreeze, ActionUnfreeze} +) + +type Disk struct { + SharableVirtualResourceModel + DiskFormat string `json:"disk_format" gorm:"column:disk_format;not null"` + DiskSize int64 `json:"disk_size" gorm:"column:disk_size;not null"` + AccessPath string `json:"access_path" gorm:"column:access_path;not null"` + AutoDelete bool `json:"auto_delete" gorm:"column:auto_delete;not null"` + StorageID string `json:"storage_id" gorm:"column:storage_id;not null"` + MebsInfo string `json:"mebs_info" gorm:"column:mebs_info;type:text"` +} + +func (d Disk) TableName() string { + return disksTable +} + +func (d Disk) String() string { + str, _ := JsonString(d) + return str +} + +func NewDiskResource(db *gorm.DB) (Resourcer, error) { + return newResource(db, disksTable, + func() interface{} { return &Disk{} }, + func() interface{} { return &([]Disk{}) }) +} + +func (d Disk) Storage() (*Storage, error) { + s, err := FetchByID(Storages, d.StorageID) + if err != nil { + return nil, err + } + return s.(*Storage), nil +} + +func (d Disk) IsLocal() (bool, error) { + s, err := d.Storage() + if err != nil { + return false, err + } + return utils.IsLocalStorage(s.StorageType), nil +} diff --git a/pkg/scheduler/db/models/group.go b/pkg/scheduler/db/models/group.go new file mode 100644 index 0000000000..3b64954b60 --- /dev/null +++ b/pkg/scheduler/db/models/group.go @@ -0,0 +1,33 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type Group struct { + VirtualResourceModel + ServiceType string `json:"service_type" gorm:"column:service_type"` + ParentID string `json:"parent_id" gorm:"column:parent_id"` + ZoneID string `json:"zone_id" gorm:"column:zone_id"` + SchedStrategy string `json:"sched_strategy" gorm:"column:sched_strategy"` +} + +func (g Group) TableName() string { + return groupTable +} + +func (g Group) String() string { + str, _ := JsonString(g) + return str +} + +func NewGroupResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Group{} + } + models := func() interface{} { + groups := []Group{} + return &groups + } + return newResource(db, groupTable, model, models) +} diff --git a/pkg/scheduler/db/models/group_guest.go b/pkg/scheduler/db/models/group_guest.go new file mode 100644 index 0000000000..2f0df66e58 --- /dev/null +++ b/pkg/scheduler/db/models/group_guest.go @@ -0,0 +1,32 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type GroupGuest struct { + GroupJointModel + Tag *string `json:"tag" gorm:"column:tag"` + GuestID *string `json:"guest_id" gorm:"column:guest_id"` +} + +func (g GroupGuest) TableName() string { + return groupGuestTable +} + +func (g GroupGuest) String() string { + str, _ := JsonString(g) + return str +} + +func NewGroupGuestResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &GroupGuest{} + } + models := func() interface{} { + groupGuests := []GroupGuest{} + return &groupGuests + } + + return newResource(db, groupGuestTable, model, models) +} diff --git a/pkg/scheduler/db/models/group_joint.go b/pkg/scheduler/db/models/group_joint.go new file mode 100644 index 0000000000..0d086c0ddd --- /dev/null +++ b/pkg/scheduler/db/models/group_joint.go @@ -0,0 +1,6 @@ +package models + +type GroupJointModel struct { + JointBaseModel + GroupID string `json:"group_id" gorm:"not null"` +} diff --git a/pkg/scheduler/db/models/group_networks.go b/pkg/scheduler/db/models/group_networks.go new file mode 100644 index 0000000000..5d556662e0 --- /dev/null +++ b/pkg/scheduler/db/models/group_networks.go @@ -0,0 +1,82 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + GroupNetworksResourceName = "groupnetworks" +) + +type GroupNetwork struct { + StandaloneModel + GroupID string `json:"group_id,omitempty" gorm:"column:group_id;not null"` + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + IpAddr string `json:"ip_addr,omitempty" gorm:"column:ip_addr"` + Index int `json:"index" gorm:"column:index;not null"` + EipID string `json:"eip_id,omitempty" gorm:"column:if_name"` + MappingIpAddr string `json:"mapping_ip_addr" gorm:"column:mapping_ip_addr"` +} + +func (n GroupNetwork) TableName() string { + return groupNetworksTable +} + +func (n GroupNetwork) String() string { + s, _ := JsonString(n) + return string(s) +} + +func NewGroupNetworksResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &GroupNetwork{} + } + models := func() interface{} { + groupNetworks := []GroupNetwork{} + return &groupNetworks + } + + return newResource(db, groupNetworksTable, model, models) +} + +type GroupNicCount struct { + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c GroupNicCount) First() string { + return c.NetworkID +} + +func (c GroupNicCount) Second() int { + return c.Count +} +func GroupNicCounts() ([]GroupNicCount, error) { + counts := []GroupNicCount{} + + err := Groups.DB().Table(groupNetworksTable). + Select("network_id,count(*) as count"). + Where("deleted=0"). + Group("network_id"). + Scan(&counts).Error + return counts, err +} + +type GroupNicCounti struct { + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c GroupNicCounti) First() int { + return c.Count +} +func GroupNicCountsWithNetworkID(networkID string) (GroupNicCounti, error) { + counts := GroupNicCounti{0} + + err := Groups.DB().Table(groupNetworksTable). + Select("count(*) as count"). + Where(fmt.Sprintf("network_id = '%s' and deleted=0", networkID)). + Scan(&counts).Error + return counts, err +} diff --git a/pkg/scheduler/db/models/guest.go b/pkg/scheduler/db/models/guest.go new file mode 100644 index 0000000000..346b8e7b2e --- /dev/null +++ b/pkg/scheduler/db/models/guest.go @@ -0,0 +1,209 @@ +package models + +import ( + "fmt" + "strings" + + "github.com/jinzhu/gorm" + + "github.com/yunionio/pkg/util/sets" +) + +const ( + GuestResourceName = "server" + + VmInit = "init" + VmUnknown = "unknown" + VmSchedule = "schedule" + VmScheduleFailed = "schedule_fail" + VmCreateNetwork = "network" + VmNetworkFailed = "net_fail" + VmCreateDisk = "disk" + VmDiskFailed = "disk_fail" + VmStartDeploy = "start_deploy" + VmDeploying = "deploying" + VmDeployFailed = "deploy_fail" + VmReady = "ready" + VmStartStart = "start_start" + VmStarting = "starting" + VmStartFailed = "start_fail" + VmStartRestart = "start_restart" + VmRunning = "running" + VmStartStop = "start_stop" + VmStopping = "stopping" + VmStopFailed = "stop_fail" + + VmStartSuspend = "start_suspend" + VmSuspending = "suspending" + VmSuspend = "suspend" + VmSuspendFailed = "suspend_failed" + + VmReset = "reset" + VmStartDelete = "start_delete" + VmDeleteFail = "delete_fail" + VmDeleting = "deleting" + + VmStartMigrate = "start_migrate" + VmMigrating = "migrating" + VmMigrateFailed = "migrate_failed" + + VmDiskMigrating = "disk_migrating" + VmDiskMigrateFailed = "disk_migrate_fail" + + VmChangeFlavor = "change_flavor" + VmChangeFlavorFail = "change_flavor_fail" + + VmRebuildRoot = "rebuild_root" + VmRebuildRootFail = "rebuild_root_fail" + + VmRebuildDisk = "rebuild_disk" + VmRebuildDiskFail = "rebuild_disk_fail" + + VmBlockStream = "block_stream" + + VmStartSnapshot = "snapshot_start" + VmSnapshot = "snapshot" + VmSnapshotSucc = "snapshot_succ" + VmSnapshotFailed = "snapshot_failed" + + VmSyncConfig = "sync_config" + VmSyncConfigFail = "sync_config_failed" + + VmResizeDisk = "resize_disk" + VmStartSaveDisk = "start_save_disk" + VmSaveDisk = "save_disk" + VmSaveDiskFailed = "save_disk_failed" + + VmRestoringSnapshot = "restoring_snapshot" + + VmRestoreDisk = "restore_disk" + VmRestoreState = "restore_state" + VmRestoreFailed = "restore_failed" + + VmRemoveStatefile = "remove_state" + + VmHotplugCPUMEM = "hotplug_cpu_mem" + + VmAdmin = "admin" + + ShutdownStop = "stop" + ShutdownTerminate = "terminate" + + HostTypeHost = "host" + HostTypeBaremetal = "baremetal" + + GuestTypeVm = "vm" + GuestTypeContainer = "container" + + QGAStatusUnknown = "unknown" + QGAStatusStop = "stop" + QGAStatusStarting = "starting" + QGAStatusStartFailed = "start_failed" + QGAStatusRunning = "running" + QGAStatusCrashed = "crashed" +) + +var ( + VmRunningStatus = sets.NewString(VmStartStart, VmStarting, VmRunning, VmStopFailed, VmBlockStream) + VmCreatingStatus = sets.NewString(VmCreateNetwork, VmCreateDisk, VmStartDeploy, VmDeploying) + GuestExtraFeature = sets.NewString("kvm", "storage_type") +) + +type Guest struct { + VirtualResourceModel + VCPUCount int64 `json:"vcpu_count" gorm:"column:vcpu_count;type:tinyint64(4);not null"` + VMemSize int64 `json:"vmem_size" gorm:"column:vmem_size;type:int64(11);not null"` + DimmSlots string `json:"dimm_slots,omitempty" gorm:"column:dimm_slots;type:text"` + BootOrder string `json:"boot_order,omitempty" gorm:"column:boot_order"` + DisableDelete bool `json:"disable_delete" gorm:"column:disable_delete"` + ShutdownBehavior string `json:"shutdown_behavior,omitempty" gorm:"column:shutdown_behavior"` + KeypairID string `json:"keypair_id,omitempty" gorm:"column:keypair_id"` + HostID string `json:"host_id,omitempty" gorm:"column:host_id"` + VNCPort int64 `json:"vnc_port,omitempty" gorm:"column:vnc_port"` + VGA string `json:"vga" gorm:"column:vga"` + FlavorID string `json:"flavor_id,omitempty" gorm:"column:flavor_id"` + SecgrpID string `json:"secgrp_id,omitempty" gorm:"column:secgrp_id"` + AdminSecgrpID string `json:"admin_secgrp_id,omitempty" gorm:"column:admin_secgrp_id"` + VrouterID string `json:"vrouter_id,omitempty" gorm:"column:vrouter_id"` + HostType string `json:"host_type,omitempty" gorm:"column:host_type"` + PreferZoneID string `json:"prefer_zone_id,omitempty" gorm:"column:prefer_zone_id"` + GuestType string `json:"guest_type,omitempty" gorm:"column:guest_type"` + QGAStatus string `json:"qga_status" gorm:"column:qga_status"` +} + +func (g Guest) TableName() string { + return guestsTable +} + +func (g Guest) String() string { + s, _ := JsonString(g) + return s +} + +func (g Guest) DisksQuery(diskFormat ...string) *gorm.DB { + q := GuestDisks.DB().Table(guestDiskTable). + Where(map[string]interface{}{ + "deleted": false, + "guest_id": g.ID, + }) + if len(diskFormat) != 0 { + joinStr := fmt.Sprintf("JOIN %s on %s.id = %s.disk_id AND %s.disk_format = ?", disksTable, disksTable, guestDiskTable, disksTable) + q.Joins(joinStr, diskFormat[0]) + } + + return q +} + +func (g Guest) Disks(diskFormat ...string) *gorm.DB { + q := g.DisksQuery(diskFormat...) + q.Order(fmt.Sprintf("%s.index", g.TableName())) + return q +} + +func (g Guest) DiskSize(onlyLocal bool) (int64, error) { + var size int64 + q := g.Disks() + disks := []GuestDisk{} + err := q.Scan(&disks).Error + if err != nil { + return 0, err + } + for _, gstDisk := range disks { + disk, err := gstDisk.Disk() + if err != nil { + return 0, err + } + isLocal, err := disk.IsLocal() + if err != nil { + return 0, err + } + if !onlyLocal || isLocal { + size += disk.DiskSize + } + } + return size, nil +} + +func (g Guest) IsRunning() bool { + return VmRunningStatus.Has(g.Status) +} + +func (g Guest) IsCreating() bool { + return VmCreatingStatus.Has(g.Status) +} + +func (g Guest) IsGuestFakeDeleted() bool { + return strings.HasSuffix(g.Name, "_deleted") +} + +func NewGuestResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Guest{} + } + models := func() interface{} { + guests := []Guest{} + return &guests + } + + return newResource(db, guestsTable, model, models) +} diff --git a/pkg/scheduler/db/models/guest_disk.go b/pkg/scheduler/db/models/guest_disk.go new file mode 100644 index 0000000000..0cd47ea79b --- /dev/null +++ b/pkg/scheduler/db/models/guest_disk.go @@ -0,0 +1,38 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type GuestDisk struct { + GuestJointModel + DiskID string `json:"disk_id" gorm:"column:disk_id;not null;index"` + ImagePath string `json:"image_path" gorm:"column:image_path;not null"` + Driver string `json:"driver" gorm:"column:driver"` + CacheMode string `json:"cache_mode" gorm:"column:cache_mode"` + AioMode string `json:"aio_mode" gorm:"column:aio_mode"` + Index int `json:"index" gorm:"column:index;not null"` +} + +func (d GuestDisk) TableName() string { + return guestDiskTable +} + +func (d GuestDisk) String() string { + str, _ := JsonString(d) + return str +} + +func (d GuestDisk) Disk() (*Disk, error) { + disk, err := FetchByID(Disks, d.DiskID) + if err != nil { + return nil, err + } + return disk.(*Disk), nil +} + +func NewGuestDiskResource(db *gorm.DB) (Resourcer, error) { + return newResource(db, guestDiskTable, + func() interface{} { return &GuestDisk{} }, + func() interface{} { return &([]GuestDisk{}) }) +} diff --git a/pkg/scheduler/db/models/guest_joint.go b/pkg/scheduler/db/models/guest_joint.go new file mode 100644 index 0000000000..6039de9499 --- /dev/null +++ b/pkg/scheduler/db/models/guest_joint.go @@ -0,0 +1,6 @@ +package models + +type GuestJointModel struct { + JointBaseModel + GuestID string `json:"guest_id" gorm:"column:guest_id;not null;index"` +} diff --git a/pkg/scheduler/db/models/guest_networks.go b/pkg/scheduler/db/models/guest_networks.go new file mode 100644 index 0000000000..cd458fffc8 --- /dev/null +++ b/pkg/scheduler/db/models/guest_networks.go @@ -0,0 +1,86 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + GuestNetworksResourceName = "guestnetworks" +) + +type GuestNetwork struct { + StandaloneModel + GuestID string `json:"guest_id,omitempty" gorm:"column:guest_id;not null"` + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + MacAddr string `json:"mac_addr" gorm:"column:mac_addr;not null"` + IpAddr string `json:"ip_addr,omitempty" gorm:"column:ip_addr"` + Ip6Addr string `json:"ip6_addr" gorm:"column:ip6_addr"` + Driver string `json:"driver" gorm:"column:driver"` + BwLimit int64 `json:"bw_limit" gorm:"column:bw_limit;not null"` + Index int `json:"index" gorm:"column:index;not null"` + Virtual int `json:"virtual" gorm:"column:virtual"` + IfName string `json:"if_name,omitempty" gorm:"column:if_name"` + MappingIpAddr string `json:"mapping_ip_addr" gorm:"column:mapping_ip_addr"` +} + +func (n GuestNetwork) TableName() string { + return guestNetworksTable +} + +func (n GuestNetwork) String() string { + s, _ := JsonString(n) + return string(s) +} + +func NewGuestNetworksResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &GuestNetwork{} + } + models := func() interface{} { + guestNetworks := []GuestNetwork{} + return &guestNetworks + } + + return newResource(db, guestNetworksTable, model, models) +} + +type GuestNicCount struct { + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c GuestNicCount) First() string { + return c.NetworkID +} + +func (c GuestNicCount) Second() int { + return c.Count +} + +func GuestNicCounts() ([]GuestNicCount, error) { + counts := []GuestNicCount{} + err := GuestNetworks.DB().Table(guestNetworksTable). + Select("network_id,count(*) as count"). + Where("deleted=0"). + Group("network_id"). + Scan(&counts).Error + return counts, err +} + +type GuestNicCounti struct { + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c GuestNicCounti) First() int { + return c.Count +} +func GuestNicCountsWithNetworkID(networkID string) (GuestNicCounti, error) { + counts := GuestNicCounti{0} + err := GuestNetworks.DB().Table(guestNetworksTable). + Select("count(*) as count"). + Where(fmt.Sprintf("network_id = '%s' and deleted=0", networkID)). + Scan(&counts).Error + return counts, err +} diff --git a/pkg/scheduler/db/models/host.go b/pkg/scheduler/db/models/host.go new file mode 100644 index 0000000000..20615fa1ab --- /dev/null +++ b/pkg/scheduler/db/models/host.go @@ -0,0 +1,170 @@ +package models + +import ( + "fmt" + "strings" + + "github.com/jinzhu/gorm" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/onecloud/pkg/scheduler/api" +) + +const ( + HostResourceName = "host" +) + +var ( + HostExtraFeature = []string{"nest", "storage_type", "vip_reserved"} +) + +type Host struct { + StandaloneModel + + Rack string `json:"rack,omitempty" gorm:"column:rack"` + Slots string `json:"slots,omitempty" gorm:"column:slots"` + AccessMAC string `json:"access_mac" gorm:"not null"` + AccessIP string `json:"access_ip" gorm:"column:access_ip"` + ManagerURI string `json:"manager_uri,omitempty" gorm:"column:manager_uri"` + SysInfo string `json:"sys_info,omitempty" gorm:"type:text"` + Sn string `json:"sn,omitempty" gorm:"column:sn"` + + CPUCount int64 `json:"cpu_count" gorm:"column:cpu_count"` + NodeCount int64 `json:"node_count" gorm:"column:node_count"` + CPUDesc string `json:"cpu_desc" gorm:"column:cpu_desc"` + CPUMHZ int64 `json:"cpu_mhz" gorm:"column:cpu_mhz"` + CPUCache int64 `json:"cpu_cache" gorm:"column:cpu_cache"` + CPUReserved int64 `json:"cpu_reserved" gorm:"column:cpu_reserved"` + CPUCmtbound *float64 `json:"cpu_cmtbound" gorm:"column:cpu_cmtbound"` + + MemSize int64 `json:"mem_size" gorm:"column:mem_size"` + MemReserved int64 `json:"mem_reserved" gorm:"column:mem_reserved"` + MemCmtbound *float64 `json:"mem_cmtbound" gorm:"column:mem_cmtbound"` + + StorageSize int `json:"storage_size,omitempty" gorm:"column:storage_size"` + StorageType string `json:"storage_type,omitempty" gorm:"column:storage_type"` + StorageDriver string `json:"storage_driver,omitempty" gorm:"column:storage_driver"` + StorageInfo string `json:"storage_info,omitempty" gorm:"column:storage_info"` + IpmiInfo string `json:"ipmi_info,omitempty" gorm:"type:text"` + + Status string `json:"status" gorm:"column:status;not null"` + HostStatus string `json:"host_status" gorm:"column:host_status;not null"` + Enabled bool `json:"enabled" gorm:"column:enabled;not null"` + ZoneID string `json:"zone_id" gorm:"column:zone_id;not null"` + HostType string `json:"host_type" gorm:"column:host_type"` + Version string `json:"version" gorm:"column:version"` + IsBaremetal bool `json:"is_baremetal" gorm:"column:is_baremetal"` + ManagerID *string `json:"manager_id" gorm:"column:manager_id"` + IsMaintenance bool `json:"is_maintenance" gorm:"column:is_maintenance"` + + // DECAPITATE + ClusterID string `json:"cluster_id" gorm:"column:cluster_id"` + PoolID string `json:"pool_id,omitempty" gorm:"column:pool_id"` +} + +func (h Host) TableName() string { + return hostsTable +} + +func (h Host) String() string { + s, _ := JsonString(h) + return string(s) +} + +func (h Host) IsHypervisor() bool { + if h.HostType == api.HostTypeBaremetal { + return false + } + return true +} + +func NewHostResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Host{} + } + models := func() interface{} { + hosts := []Host{} + return &hosts + } + + return newResource(db, hostsTable, model, models) +} + +func (h Host) CPUOverCommitBound() float64 { + if h.CPUCmtbound != nil { + return *h.CPUCmtbound + } + return float64(o.GetOptions().DefaultCpuOvercommitBound) +} + +func (h Host) MemOverCommitBound() float64 { + if h.MemCmtbound != nil { + return *h.MemCmtbound + } + return float64(o.GetOptions().DefaultMemoryOvercommitBound) +} + +func HostAggregates(hostID string) ([]*Aggregate, error) { + hAggs, err := FetchByHostIDs(AggregateHosts, []string{hostID}) + if err != nil { + return nil, err + } + aggs := make([]*Aggregate, 0) + for _, obj := range hAggs { + ha := obj.(*AggregateHost) + agg, err := ha.Aggregate() + if err != nil { + return nil, err + } + aggs = append(aggs, agg) + } + return aggs, nil +} + +type ResidentTenant struct { + HostID string `json:"host_id" gorm:"column:host_id;not null"` + TenantID string `json:"tenant_id" gorm:"column:tenant_id;not null"` + TenantCount int64 `json:"tenant_count" gorm:"column:tenant_count"` +} + +func (t ResidentTenant) First() string { + return t.HostID +} +func (t ResidentTenant) Second() string { + return t.TenantID +} + +func (t ResidentTenant) Third() interface{} { + return t.TenantCount +} + +func ResidentTenantsInHosts(hostIDs []string) ([]ResidentTenant, error) { + tenants := []ResidentTenant{} + err := Guests.DB().Table(guestsTable). + Select("host_id, tenant_id, count(tenant_id) as tenant_count"). + Where(fmt.Sprintf("host_id in ('%s') and deleted=0", strings.Join(hostIDs, "','"))). + Group("tenant_id, host_id").Scan(&tenants).Error + return tenants, err +} + +func FetchHypervisorHostByIDs(ids []string) ([]interface{}, error) { + rows, err := rowsNotDeletedInWithCond(Hosts, "id", ids, + map[string]interface{}{ + "host_type!": "baremetal", + }) + if err != nil { + return nil, err + } + return rowsToArray(Hosts, rows) +} + +func FetchBaremetalHostByIDs(ids []string) ([]interface{}, error) { + rows, err := rowsNotDeletedInWithCond(Hosts, "id", ids, + map[string]interface{}{ + "host_type": "baremetal", + }) + if err != nil { + return nil, err + } + return rowsToArray(Hosts, rows) +} diff --git a/pkg/scheduler/db/models/host_joint.go b/pkg/scheduler/db/models/host_joint.go new file mode 100644 index 0000000000..5fe05ce177 --- /dev/null +++ b/pkg/scheduler/db/models/host_joint.go @@ -0,0 +1,6 @@ +package models + +type HostJointModel struct { + JointBaseModel + HostID string `json:"host_id" gorm:"not null"` +} diff --git a/pkg/scheduler/db/models/host_storage.go b/pkg/scheduler/db/models/host_storage.go new file mode 100644 index 0000000000..a0855c43de --- /dev/null +++ b/pkg/scheduler/db/models/host_storage.go @@ -0,0 +1,32 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type HostStorage struct { + HostJointModel + MountPoint string `json:"mount_point" gorm:"not null"` + StorageID string `json:"storage_id" gorm:"not null"` +} + +func (s HostStorage) TableName() string { + return hostStorageTable +} + +func (s HostStorage) String() string { + str, _ := JsonString(s) + return str +} + +func NewHostStorageResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &HostStorage{} + } + models := func() interface{} { + storages := []HostStorage{} + return &storages + } + + return newResource(db, hostStorageTable, model, models) +} diff --git a/pkg/scheduler/db/models/host_wires.go b/pkg/scheduler/db/models/host_wires.go new file mode 100644 index 0000000000..806d73c35e --- /dev/null +++ b/pkg/scheduler/db/models/host_wires.go @@ -0,0 +1,68 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + HostWireResourceName = "hostwires" +) + +type HostWire struct { + StandaloneModel + Bridge string `json:"bridge,omitempty" gorm:"not null"` + Interface string `json:"interface,omitempty" gorm:"not null"` + HostID string `json:"host_id,omitempty" gorm:"not null"` + WireID string `json:"wire_id,omitempty" gorm:"not null"` +} + +func (w HostWire) TableName() string { + return hostWiresTable +} + +func (w HostWire) String() string { + str, _ := JsonString(w) + return str +} + +func NewHostWiresResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &HostWire{} + } + models := func() interface{} { + hostWires := []HostWire{} + return &hostWires + } + + return newResource(db, hostWiresTable, model, models) +} + +type Host2Wire struct { + HostID string `json:"host_id" gorm:"column:host_id;not null"` + WireID string `json:"wire_id" gorm:"column:wire_id;not null"` +} + +func (c Host2Wire) First() string { + return c.WireID +} + +func SelectWiresWithHostID(hostID string) ([]Host2Wire, error) { + wires := []Host2Wire{} + err := HostWires.DB().Table(hostWiresTable). + Select("distinct wire_id"). + Where(fmt.Sprintf("host_id = '%s' and deleted=0", hostID)). + Scan(&wires).Error + + return wires, err +} +func SelectHostHasWires() ([]Host2Wire, error) { + wires := []Host2Wire{} + err := HostWires.DB().Table(hostWiresTable). + Select("host_id,wire_id"). + Where("deleted=0"). + Scan(&wires).Error + + return wires, err +} diff --git a/pkg/scheduler/db/models/interface.go b/pkg/scheduler/db/models/interface.go new file mode 100644 index 0000000000..4ee3d34531 --- /dev/null +++ b/pkg/scheduler/db/models/interface.go @@ -0,0 +1,16 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type Resourcer interface { + DB() *gorm.DB + TableName() string + Model() interface{} + Models() interface{} +} + +type Modeler interface { + UUID() string +} diff --git a/pkg/scheduler/db/models/isolated_device.go b/pkg/scheduler/db/models/isolated_device.go new file mode 100644 index 0000000000..e945ca9f67 --- /dev/null +++ b/pkg/scheduler/db/models/isolated_device.go @@ -0,0 +1,30 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +type IsolatedDevice struct { + StandaloneModel + HostID string `json:"host_id,omitempty" gorm:"column:host_id;not null"` + DevType string `json:"dev_type" gorm:"column:dev_type;not null"` + Model string `json:"model" gorm:"column:model;not null"` + GuestID string `json:"guest_id" gorm:"column:guest_id"` + Addr string `json:"addr" gorm:"column:addr"` + VendorDeviceID string `json:"vendor_device_id" gorm:"column:vendor_device_id"` +} + +func (d IsolatedDevice) TableName() string { + return isolatedDeviceTable +} + +func (d IsolatedDevice) String() string { + str, _ := JsonString(d) + return str +} + +func NewIsolatedDeviceResource(db *gorm.DB) (Resourcer, error) { + return newResource(db, isolatedDeviceTable, + func() interface{} { return &IsolatedDevice{} }, + func() interface{} { return &([]IsolatedDevice{}) }) +} diff --git a/pkg/scheduler/db/models/metadata.go b/pkg/scheduler/db/models/metadata.go new file mode 100644 index 0000000000..1bc3fc4559 --- /dev/null +++ b/pkg/scheduler/db/models/metadata.go @@ -0,0 +1,44 @@ +package models + +import ( + "fmt" + "strings" + "time" + + "github.com/jinzhu/gorm" +) + +type Metadata struct { + ID string `json:"id" gorm:"primary_key;column:id;type:varchar(128) CHARACTER SET ascii"` + Key string `json:"key" gorm:"primary_key;column:key;type:varchar(64) CHARACTER SET ascii"` + Value string `json:"value" gorm:"type:text CHARACTER SET utf8"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at;type:datetime" sql:"DEFAULT:NULL"` +} + +func (m Metadata) TableName() string { + return metadataTable +} + +func (m Metadata) String() string { + str, _ := JsonString(m) + return str +} + +func NewMetadataResource(db *gorm.DB) (Resourcer, error) { + return newResource(db, metadataTable, + func() interface{} { return &Metadata{} }, + func() interface{} { return &([]Metadata{}) }) +} + +func FetchMetadatas(resourceName string, ids, keys []string) ([]interface{}, error) { + idsWithRes := make([]string, len(ids)) + for i, id := range ids { + idsWithRes[i] = fmt.Sprintf("%s::%s", resourceName, id) + } + rows, err := Metadatas.DB().Table(metadataTable). + Where(fmt.Sprintf("id in ('%s') AND `key` in ('%s')", strings.Join(idsWithRes, "','"), strings.Join(keys, "','"))).Rows() + if err != nil { + return nil, err + } + return rowsToArray(Metadatas, rows) +} diff --git a/pkg/scheduler/db/models/netinterfaces.go b/pkg/scheduler/db/models/netinterfaces.go new file mode 100644 index 0000000000..a0a7bf1c65 --- /dev/null +++ b/pkg/scheduler/db/models/netinterfaces.go @@ -0,0 +1,71 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + NetInterfaceResourceName = "netinterface" +) + +type NetInterface struct { + Mac string `json:"mac,omitempty" gorm:"not null"` + BaremetalId string `json:"baremetal_id,omitempty"` + WireId string `json:"wire_id,omitempty"` + Rate int64 `json:"rate,omitempty"` + NicType string `json:"nic_type,omitempty"` + Index int `json:"index,omitempty"` + LinkUp int `json:"link_up,omitempty"` + Mtu int64 `json:"mtu,omitempty"` +} + +func (n NetInterface) TableName() string { + return netinterfacesTable +} + +func (n NetInterface) String() string { + str, _ := JsonString(n) + return str +} + +func NewNetInterfacesResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &NetInterface{} + } + models := func() interface{} { + netInterfaces := []NetInterface{} + return &netInterfaces + } + + return newResource(db, netinterfacesTable, model, models) +} + +type BaremetalWire struct { + BaremetalID string `json:"baremetal_id" gorm:"column:baremetal_id;not null"` + WireID string `json:"wire_id" gorm:"column:wire_id;not null"` +} + +func (c BaremetalWire) First() string { + return c.WireID +} + +func SelectWiresWithBaremetalID(baremetalID string) ([]BaremetalWire, error) { + baremetalWires := []BaremetalWire{} + err := NetInterfaces.DB().Table(netinterfacesTable). + Select("distinct wire_id"). + Where(fmt.Sprintf("baremetal_id = '%s'", baremetalID)). + Scan(&baremetalWires).Error + + return baremetalWires, err +} + +func SelectWiresAndBaremetals() ([]BaremetalWire, error) { + baremetalWires := []BaremetalWire{} + err := NetInterfaces.DB().Table(netinterfacesTable). + Select("baremetal_id,wire_id"). + Scan(&baremetalWires).Error + + return baremetalWires, err +} diff --git a/pkg/scheduler/db/models/networks.go b/pkg/scheduler/db/models/networks.go new file mode 100644 index 0000000000..aa4baa6a87 --- /dev/null +++ b/pkg/scheduler/db/models/networks.go @@ -0,0 +1,256 @@ +package models + +import ( + "fmt" + "strings" + + "github.com/jinzhu/gorm" + + "github.com/yunionio/pkg/utils" +) + +const ( + NetworkResourceName = "network" + + GuestNicCountC = "GuestNiCount" + GroupNicCountC = "GroupNicCount" + BaremetalNicCountC = "BaremetalNicCount" + ReserveDipNicCountC = "ReserveDipNicCount" +) + +type Network struct { + StandaloneModel + Status string `json:"status,omitempty" gorm:"not null"` + TenantID string `json:"tenant_id,omitempty" gorm:"not null"` + UserId string `json:"user_id,omitempty" gorm:"not null"` + IsPublic int `json:"is_public,omitempty" gorm:"not null"` + GuestIpStart string `json:"guest_ip_start,omitempty" gorm:"not null"` + GuestIpEnd string `json:"guest_ip_end,omitempty" gorm:"not null"` + GuestIpMask int `json:"guest_ip_mask,omitempty" gorm:"not null"` + GuestGateway string `json:"guest_gateway,omitempty"` + GuestDns string `json:"guest_dns,omitempty"` + GuestIp6Start string `json:"guest_ip6_start,omitempty"` + GuestIp6End string `json:"guest_ip6_end,omitempty"` + GuestIp6Mask int `json:"guest_ip6_mask,omitempty"` + GuestGateway6 string `json:"guest_gateway6,omitempty"` + GuestDns6 string `json:"guest_dns6,omitempty"` + GuestDomain6 string `json:"guest_domain6,omitempty"` + VlanId int64 `json:"vlan_id,omitempty" gorm:"not null"` + DhcpHostId string `json:"dhcp_host_id,omitempty"` + WireID string `json:"wire_id,omitempty"` + IsChanged int `json:"is_changed,omitempty" gorm:"not null"` + IsSystem int `json:"is_system,omitempty"` + GuestDhcp string `json:"guest_dhcp,omitempty"` + BillingType string `json:"billing_type,omitempty"` + ServerType string `json:"server_type,omitempty"` + VpcId string `json:"vpc_id,omitempty"` + ZoneId string `json:"zone_id,omitempty"` + AcSubnetId string `json:"ac_subnet_id,omitempty"` +} + +func (n Network) TableName() string { + return networksTable +} + +func (n Network) String() string { + str, _ := JsonString(n) + return str +} + +func NewNetworksResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Network{} + } + models := func() interface{} { + networks := []Network{} + return &networks + } + + return newResource(db, networksTable, model, models) +} + +func SelectNetworksWithByWireIDs(wireIDs []string) ([]WireNetwork, error) { + networks := []WireNetwork{} + err := Networks.DB().Table(networksTable). + Select("distinct id"). + Where(fmt.Sprintf("wire_id in ('%s') and deleted=0", strings.Join(wireIDs, "','"))). + Scan(&networks).Error + + return networks, err +} + +func SelectWireIDsHasNetworks() ([]WireNetwork, error) { + networks := []WireNetwork{} + err := Networks.DB().Table(networksTable). + Select("id,wire_id"). + Where("deleted=0"). + Scan(&networks).Error + + return networks, err +} + +type WireNetwork struct { + ID string `json:"id,omitempty" gorm:"not null"` + TenantID string `json:"tenant_id,omitempty" gorm:"not null"` + GuestIpStart string `json:"guest_ip_start,omitempty" gorm:"not null"` + GuestIpEnd string `json:"guest_ip_end,omitempty" gorm:"not null"` + IsPublic int `json:"is_public,omitempty" gorm:"not null"` + WireID string `json:"wire_id,omitempty"` + ServerType string `json:"server_type,omitempty"` +} + +func (c WireNetwork) First() string { + return c.ID +} + +func SelectNetworksWithByWireIDsi(wireIDs []string) ([]WireNetwork, error) { + networks := []WireNetwork{} + err := Networks.DB().Table(networksTable). + Select("distinct id,wire_id,tenant_id,is_public,server_type,guest_ip_start,guest_ip_end"). + Where(fmt.Sprintf("wire_id in ('%s') and deleted=0", strings.Join(wireIDs, "','"))). + Scan(&networks).Error + return networks, err +} + +type NetworkSchedResult struct { + ID string `json:"id"` + Name string `json:"name"` + TenantID string `json:"tenant_id"` + IsPublic bool `json:"is_public"` + ServerType string `json:"server_type"` + Ports int `json:"ports"` + IsExit bool `json:"is_exit"` + Wire string `json:"wire_name"` + WireID string `json:"wire_id"` +} + +func HostNetworkSchedResults(hostID string) ([]*NetworkSchedResult, error) { + hostAndWires, err := SelectWiresWithHostID(hostID) + if err != nil { + return nil, err + } + + if len(hostAndWires) == 0 { + return nil, fmt.Errorf("Host %q not in wire.", hostID) + } + + wireIDs := []string{} + for _, hostWire := range hostAndWires { + wireIDs = append(wireIDs, hostWire.WireID) + } + + hostNets, err := FetchByWireIDs(Networks, wireIDs) + if err != nil { + return nil, err + } + + netRes := []*NetworkSchedResult{} + for _, n := range hostNets { + r, err := NewNetworkSchedResult(n.(*Network)) + if err != nil { + return nil, fmt.Errorf("NewNetworkBuildResult err: %v", err) + } + netRes = append(netRes, r) + } + return netRes, nil +} + +func NewNetworkSchedResult(net *Network) (*NetworkSchedResult, error) { + if net == nil { + return nil, fmt.Errorf("empty network model resource") + } + + wire, err := FetchByID(Wires, net.WireID) + if err != nil { + return nil, fmt.Errorf("fetch wire %q err: %v", net.WireID, err) + } + + res := &NetworkSchedResult{ + ID: net.ID, + WireID: net.WireID, + Name: net.Name, + Wire: wire.(*Wire).Name, + TenantID: net.TenantID, + ServerType: net.ServerType, + IsExit: utils.IsExitAddress(net.GuestIpStart), + } + res.IsPublic = net.IsPublic == 1 + ports, err := NetworkAvaliableAddress(net) + if err != nil { + return nil, err + } + res.Ports = ports + return res, nil +} + +func NetworkAvaliableAddress(net *Network) (ports int, err error) { + totalAddress := utils.IpRangeCount(net.GuestIpStart, net.GuestIpEnd) + guestNicCount, err := NicCount(GuestNicCountC) + if err != nil { + return + } + + groupNicCount, err := NicCount(GroupNicCountC) + if err != nil { + return + } + + baremetalNicCount, err := NicCount(BaremetalNicCountC) + if err != nil { + return + } + + reserveDipNicCount, err := NicCount(ReserveDipNicCountC) + if err != nil { + return + } + + ports = totalAddress - guestNicCount[net.ID] - groupNicCount[net.ID] - baremetalNicCount[net.ID] - reserveDipNicCount[net.ID] + return +} + +func NicCount(nicName string) (map[string]int, error) { + countsMap := make(map[string]int) + switch nicName { + case GuestNicCountC: + counts, err := GuestNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + case GroupNicCountC: + counts, err := GroupNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + + case BaremetalNicCountC: + counts, err := BaremetalNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + + case ReserveDipNicCountC: + counts, err := ReserveNicCounts() + if err != nil { + return nil, err + } + + for _, count := range counts { + countsMap[count.NetworkID] = count.Count + } + } + + return countsMap, nil +} diff --git a/pkg/scheduler/db/models/networks_test.go b/pkg/scheduler/db/models/networks_test.go new file mode 100644 index 0000000000..37513f7b84 --- /dev/null +++ b/pkg/scheduler/db/models/networks_test.go @@ -0,0 +1,16 @@ +package models + +import ( + "encoding/json" + "testing" +) + +func TestHostNetworkSchedResults(t *testing.T) { + id := "dd11e175-c2b3-403b-8fed-47a17cd72b79" + result, err := HostNetworkSchedResults(id) + if err != nil { + t.Fatal(err) + } + js, _ := json.MarshalIndent(result, "", " ") + t.Logf("NetworkSchedResult: %s", string(js)) +} diff --git a/pkg/scheduler/db/models/reservedips.go b/pkg/scheduler/db/models/reservedips.go new file mode 100644 index 0000000000..5ab1d3701a --- /dev/null +++ b/pkg/scheduler/db/models/reservedips.go @@ -0,0 +1,77 @@ +package models + +import ( + "fmt" + + "github.com/jinzhu/gorm" +) + +const ( + ReserveDipsResourceName = "reservedips" +) + +type ReserveDipNetwork struct { + StandaloneModel + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + IpAddr string `json:"ip_addr,omitempty" gorm:"column:ip_addr"` + Notes string `json:"notes" gorm:"column:notes"` +} + +func (n ReserveDipNetwork) TableName() string { + return reserveDipsTable +} + +func (n ReserveDipNetwork) String() string { + s, _ := JsonString(n) + return string(s) +} + +func NewReserveDipsNetworksResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &ReserveDipNetwork{} + } + models := func() interface{} { + reserveDips := []ReserveDipNetwork{} + return &reserveDips + } + + return newResource(db, reserveDipsTable, model, models) +} + +type ReserveNicCount struct { + NetworkID string `json:"network_id,omitempty" gorm:"column:network_id;not null"` + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c ReserveNicCount) First() string { + return c.NetworkID +} + +func (c ReserveNicCount) Second() int { + return c.Count +} +func ReserveNicCounts() ([]ReserveNicCount, error) { + counts := []ReserveNicCount{} + err := ReserveDipsNerworks.DB().Table(reserveDipsTable). + Select("network_id,count(*) as count"). + Where("deleted=0"). + Group("network_id"). + Scan(&counts).Error + return counts, err +} + +type ReserveNicCounti struct { + Count int `json:"count" gorm:"column:count;not null"` +} + +func (c ReserveNicCounti) First() int { + return c.Count +} +func ReserveNicCountsWithNetworkID(networkID string) (ReserveNicCounti, error) { + counts := ReserveNicCounti{0} + err := ReserveDipsNerworks.DB().Table(reserveDipsTable). + Select("count(*) as count"). + Where(fmt.Sprintf("network_id = '%s' and deleted=0", networkID)). + Scan(&counts).Error + return counts, err +} diff --git a/pkg/scheduler/db/models/storage.go b/pkg/scheduler/db/models/storage.go new file mode 100644 index 0000000000..99cf65be4b --- /dev/null +++ b/pkg/scheduler/db/models/storage.go @@ -0,0 +1,76 @@ +package models + +import ( + "fmt" + "strings" + + "github.com/jinzhu/gorm" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" +) + +type Storage struct { + StandaloneModel + Capacity int64 `json:"capacity" gorm:"not null"` + StorageType string `json:"storage_type" gorm:"not null"` + MediumType string `json:"medium_type" gorm:"not null"` + Cmtbound *float64 `json:"cmtbound"` + Status string `json:"status" gorm:"not null"` + StorageConf string `json:"storage_conf" gorm:"type:text"` + ZoneID string `json:"zone_id"` +} + +func (s Storage) TableName() string { + return storageTable +} + +func (s Storage) String() string { + str, _ := JsonString(s) + return str +} + +func NewStorageResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Storage{} + } + models := func() interface{} { + storages := []Storage{} + return &storages + } + + return newResource(db, storageTable, model, models) +} + +type StorageCapacity struct { + StorageID string `json:"storage_id" gorm:"column:storage_id;not null"` + Status string `json:"status" gorm:"not null"` + TotalSize int64 `json:"total_size" gorm:"column:total_size"` +} + +func (s StorageCapacity) First() string { + return s.StorageID +} + +func (s StorageCapacity) Second() string { + return s.Status +} + +func (s StorageCapacity) Third() interface{} { + return s.TotalSize +} + +func GetStorageCapacities(storageIDs []string) ([]StorageCapacity, error) { + results := make([]StorageCapacity, 0) + err := Disks.DB().Table(disksTable). + Select("storage_id, status, sum(disk_size) as total_size"). + Where(fmt.Sprintf("storage_id in ('%s') and deleted=0", strings.Join(storageIDs, "','"))). + Group("storage_id, status").Scan(&results).Error + return results, err +} + +func (s Storage) OverCommitBound() float64 { + if s.Cmtbound != nil { + return *s.Cmtbound + } + return float64(o.GetOptions().DefaultStorageOvercommitBound) +} diff --git a/pkg/scheduler/db/models/wires.go b/pkg/scheduler/db/models/wires.go new file mode 100644 index 0000000000..cfc7fe757d --- /dev/null +++ b/pkg/scheduler/db/models/wires.go @@ -0,0 +1,60 @@ +package models + +import ( + "github.com/jinzhu/gorm" +) + +const ( + NetWireResourceName = "wire" +) + +type Wire struct { + StandaloneModel + Bandwidth int64 `json:"bandwidth,omitempty" gorm:"not null"` + NetDns string `json:"net_dns,omitempty"` + NetDomain string `json:"net_domain,omitempty"` + VpcVersion int64 `json:"vpc_version,omitempty" gorm:"not null"` +} + +func (w Wire) TableName() string { + return wiresTable +} + +func (w Wire) String() string { + str, _ := JsonString(w) + return str +} + +func NewWiresResource(db *gorm.DB) (Resourcer, error) { + model := func() interface{} { + return &Wire{} + } + models := func() interface{} { + wires := []Wire{} + return &wires + } + + return newResource(db, wiresTable, model, models) +} + +type WireInfo struct { + ID string `json:"id" gorm:"column:id;not null"` + Name string `json:"name" gorm:"column:name;not null"` +} + +func (i WireInfo) First() string { + return i.ID +} + +func (i WireInfo) Second() string { + return i.Name +} +func LoadAllWires() ([]WireInfo, error) { + wires := []WireInfo{} + err := Wires.DB().Table(wiresTable). + Select("id,name"). + Where("deleted=0"). + Scan(&wires).Error + + return wires, err +} diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go new file mode 100644 index 0000000000..dea801d573 --- /dev/null +++ b/pkg/scheduler/factory/factory.go @@ -0,0 +1,27 @@ +package factory + +import ( + "fmt" + + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/util/sets" +) + +func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + + provider, ok := algorithmProviderMap[name] + if !ok { + return nil, fmt.Errorf("AlgorithmProvider plugin %q has not been registered", name) + } + return &provider, nil +} + +func GetPredicates(predicatesKeys sets.String) (map[string]core.FitPredicate, error) { + return getFitPredites(predicatesKeys) +} + +func GetPriorityConfigs(priorityKeys sets.String) ([]core.PriorityConfig, error) { + return getPriorityConfigs(priorityKeys) +} diff --git a/pkg/scheduler/factory/plugins.go b/pkg/scheduler/factory/plugins.go new file mode 100644 index 0000000000..562a95b428 --- /dev/null +++ b/pkg/scheduler/factory/plugins.go @@ -0,0 +1,129 @@ +package factory + +import ( + "fmt" + "regexp" + "sync" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/pkg/util/sets" +) + +type AlgorithmProviderConfig struct { + FitPredicateKeys sets.String + PriorityKeys sets.String +} + +type FitPredicateFactory func() core.FitPredicate + +type PriorityFunctionFactory func() (core.PriorityPreFunction, core.PriorityMapFunction, core.PriorityReduceFunction) + +type PriorityConfigFactory struct { + MapReduceFunction PriorityFunctionFactory + Weight int +} + +var ( + schedulerFactoryMutex sync.Mutex + + // maps that hold registered algorithm types + fitPredicateMap = make(map[string]FitPredicateFactory) + priorityConfigMap = make(map[string]PriorityConfigFactory) + algorithmProviderMap = make(map[string]AlgorithmProviderConfig) + + validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$") +) + +const ( + DefaultProvider = "DefaultProvider" + BaremetalProvider = "BaremetalProvider" +) + +// RegisterAlgorithmProvider registers a new algorithm provider with the +// algorithm registry. This shoud be called from the init function in a +// provider plugin. +func RegisterAlgorithmProvider(name string, predicatesKeys, priorityKeys sets.String) string { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + validateAlgorithmNameOrDie(name) + algorithmProviderMap[name] = AlgorithmProviderConfig{ + FitPredicateKeys: predicatesKeys, + PriorityKeys: priorityKeys, + } + return name +} + +func validateAlgorithmNameOrDie(name string) { + if !validName.MatchString(name) { + log.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName) + } + +} + +// RegisterFitPredicate registers a fit predicate with the algorithm +// registry. Returns the name with which the predicates was registerd. +func RegisterFitPredicate(name string, predicate core.FitPredicate) string { + return RegisterFitPredicateFactory(name, func() core.FitPredicate { return predicate }) +} + +// RegisterFitPredicateFactory registers a fit predicate factory with the +// algorithm registry. Returns the name with which the predicate was registered. +func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + validateAlgorithmNameOrDie(name) + fitPredicateMap[name] = predicateFactory + return name +} + +func getFitPredites(names sets.String) (map[string]core.FitPredicate, error) { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + + predicates := map[string]core.FitPredicate{} + for _, name := range names.List() { + factory, ok := fitPredicateMap[name] + if !ok { + return nil, fmt.Errorf("Invalid predicate name %q specified - no corresponding predicate found", name) + } + predicates[name] = factory() + } + return predicates, nil +} + +// RegisterPriority registers a priority with the algorithm registry. +func RegisterPriority(name string, priority core.Priority, weight int) string { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + validateAlgorithmNameOrDie(name) + priorityConfigMap[name] = PriorityConfigFactory{ + MapReduceFunction: func() (core.PriorityPreFunction, core.PriorityMapFunction, core.PriorityReduceFunction) { + p := priority.Clone() + return p.PreExecute, p.Map, p.Reduce + }, + Weight: weight, + } + return name +} + +func getPriorityConfigs(names sets.String) ([]core.PriorityConfig, error) { + schedulerFactoryMutex.Lock() + defer schedulerFactoryMutex.Unlock() + + configs := []core.PriorityConfig{} + for _, name := range names.List() { + factory, ok := priorityConfigMap[name] + if !ok { + return nil, fmt.Errorf("Invalid priority name %q specified - no corresponding priority found", name) + } + preFunc, mapFunc, reduceFunc := factory.MapReduceFunction() + configs = append(configs, core.PriorityConfig{ + Pre: preFunc, + Map: mapFunc, + Reduce: reduceFunc, + Weight: factory.Weight, + }) + } + return configs, nil +} diff --git a/pkg/scheduler/factory/plugins_test.go b/pkg/scheduler/factory/plugins_test.go new file mode 100644 index 0000000000..47000311a4 --- /dev/null +++ b/pkg/scheduler/factory/plugins_test.go @@ -0,0 +1,27 @@ +package factory + +import ( + "testing" +) + +func TestAlgorithmNameValidation(t *testing.T) { + namesShouldValidate := []string{ + "1SomeAlgo1rithm", + "someAlgor-ithm1", + } + namesShouldNotValidate := []string{ + "-SomeAlgorithm", + "SomeAlgorithm-", + "Some,Alg:orithm", + } + for _, name := range namesShouldValidate { + if !validName.MatchString(name) { + t.Errorf("%v should be a valid algorithm name but is not valid.", name) + } + } + for _, name := range namesShouldNotValidate { + if validName.MatchString(name) { + t.Errorf("%v should be an invalid algorithm name but is valid", name) + } + } +} diff --git a/pkg/scheduler/handler/handler.go b/pkg/scheduler/handler/handler.go new file mode 100644 index 0000000000..737728a03c --- /dev/null +++ b/pkg/scheduler/handler/handler.go @@ -0,0 +1,381 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httputil" + "time" + + "github.com/bitly/go-simplejson" + "gopkg.in/gin-gonic/gin.v1" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/db/models" + schedman "github.com/yunionio/onecloud/pkg/scheduler/manager" +) + +// InstallHandler is an interface that registes route and +// handles scheduler's services. +func InstallHandler(r *gin.Engine) { + r.POST("/scheduler", timer(scheduleHandler)) + r.POST("/scheduler/:action", timer(schedulerActionHandler)) + r.POST("/scheduler/:action/:ident", timer(schedulerActionIdentHandler)) + InstallPingHandler(r) + InstallVersionHandler(r) +} + +func timer(f gin.HandlerFunc) gin.HandlerFunc { + return func(c *gin.Context) { + startTime := time.Now() + bytes, _ := httputil.DumpRequest(c.Request, true) + log.V(10).Debugf(` +>>>>>>>>>>>>> +HTTP Request: +%s +>>>>>>>>>>>>>`, string(bytes)) + f(c) + log.Infof("Handler %q cost: %v", c.Request.URL.Path, time.Since(startTime)) + } +} + +func scheduleHandler(c *gin.Context) { + doSyncSchedule(c) +} + +func schedulerActionHandler(c *gin.Context) { + act := c.Param("action") + switch act { + case "test": + doSchedulerTest(c) + case "candidate-list": + doCandidateList(c) + case "cleanup": + doCleanup(c) + case "history-list": + doHistoryList(c) + case "clean-cache": + doCleanAllHostCache(c) + //case "reserved-resources": + //doReservedResources(c) + default: + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("action: %s not support", act)) + } +} + +func schedulerActionIdentHandler(c *gin.Context) { + act := c.Param("action") + id := c.Param("ident") + switch act { + case "clean-cache": + doCleanHostCache(c, id) + case "candidate-detail": + doCandidateDetail(c, id) + case "history-detail": + doHistoryDetail(c, id) + case "completed": + doCompleted(c, id) + default: + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("action: %s not support", act)) + } +} + +func doSchedulerTest(c *gin.Context) { + if !schedman.IsReady() { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) + return + } + + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + schedInfo, err := api.NewSchedInfo(sjson, true) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + schedInfo.IsSuggestion = true + result, err := schedman.Schedule(schedInfo) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + c.JSON(http.StatusOK, transToSchedTestResult(result, schedInfo.SuggestionLimit)) +} + +func transToSchedTestResult(result []*core.SchedResultItem, limit int64) interface{} { + return &api.SchedTestResult{ + Data: result, + Total: int64(len(result)), + Limit: limit, + Offset: 0, + } +} + +func doCandidateList(c *gin.Context) { + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + args, err := api.NewCandidateListArgs(sjson) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.GetCandidateList(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} + +func doCandidateDetail(c *gin.Context, id string) { + hs, err := models.FetchHostByIDs([]string{id}) + if err != nil { + c.AbortWithError(http.StatusInternalServerError, err) + return + } + + if len(hs) == 0 { + c.AbortWithError(http.StatusNotFound, fmt.Errorf("Candidate %s not found.", id)) + return + } + + host := hs[0].(*models.Host) + + args := new(api.CandidateDetailArgs) + args.ID = id + if !host.IsHypervisor() { + args.Type = api.HostTypeBaremetal + } else { + args.Type = api.HostTypeHost + } + + result, err := schedman.GetCandidateDetail(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} + +func doCleanup(c *gin.Context) { + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + args, err := api.NewCleanupArgs(sjson) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.Cleanup(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} + +func doHistoryList(c *gin.Context) { + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + args, err := api.NewHistoryArgs(sjson) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.GetHistoryList(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} + +func doHistoryDetail(c *gin.Context, id string) { + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + args, err := api.NewHistoryDetailArgs(sjson, id) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.GetHistoryDetail(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} + +func doSyncSchedule(c *gin.Context) { + if !schedman.IsReady() { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) + return + } + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + schedInfo, err := api.NewSchedInfo(sjson, false) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.Schedule(schedInfo) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, transToRegionSchedResult(result, schedInfo.Data.Count)) +} + +func transToRegionSchedResult(result []*core.SchedResultItem, count int64) interface{} { + apiResults := make([]api.SchedResultItem, 0) + succCount := 0 + for _, nr := range result { + for { + if nr.Count <= 0 { + break + } + tr := api.SchedSuccItem{ + Candidate: api.SchedNormalResultItem{ + ID: nr.ID, + Name: nr.Name, + Data: nr.Data, + }, + } + apiResults = append(apiResults, tr) + nr.Count-- + succCount++ + } + } + + for { + if int64(succCount) >= count { + break + } + er := api.SchedErrItem{Error: "Out of resource"} + apiResults = append(apiResults, er) + succCount++ + } + + return regionResponse(apiResults) +} + +func regionResponse(v interface{}) interface{} { + return struct { + Result interface{} `json:"scheduler"` + }{Result: v} +} + +func newExpireArgsByHostIDs(ids []string) (*api.ExpireArgs, error) { + hs, err := models.FetchHostByIDs(ids) + if err != nil { + return nil, err + } + if len(hs) == 0 { + return nil, fmt.Errorf("Hostscache %v not found", ids) + } + + expireArgs := &api.ExpireArgs{ + DirtyBaremetals: []string{}, + DirtyHosts: []string{}, + } + for _, obj := range hs { + host := obj.(*models.Host) + if !host.IsHypervisor() { + expireArgs.DirtyBaremetals = append(expireArgs.DirtyBaremetals, host.ID) + } else { + expireArgs.DirtyHosts = append(expireArgs.DirtyHosts, host.ID) + } + } + return expireArgs, nil +} + +func doCleanAllHostCache(c *gin.Context) { + ids, err := models.AllIDs(models.Hosts) + if err != nil { + c.AbortWithError(http.StatusInternalServerError, err) + return + } + args, err := newExpireArgsByHostIDs(ids) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + doCleanHostCacheByArgs(c, args) +} + +func doCleanHostCache(c *gin.Context, hostID string) { + args, err := newExpireArgsByHostIDs([]string{hostID}) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + doCleanHostCacheByArgs(c, args) +} + +func doCleanHostCacheByArgs(c *gin.Context, args *api.ExpireArgs) { + result, err := schedman.Expire(args) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, regionResponse(result)) +} + +func doCompleted(c *gin.Context, id string) { + sjson, err := simplejson.NewFromReader(c.Request.Body) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + completedNotifyArgs, err := api.NewCompletedNotifyArgs(sjson, id) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + result, err := schedman.CompletedNotify(completedNotifyArgs) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + c.JSON(http.StatusOK, result) +} diff --git a/pkg/scheduler/handler/ping.go b/pkg/scheduler/handler/ping.go new file mode 100644 index 0000000000..205cde490d --- /dev/null +++ b/pkg/scheduler/handler/ping.go @@ -0,0 +1,52 @@ +package handler + +import ( + "fmt" + "net/http" + + "gopkg.in/gin-gonic/gin.v1" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + schedman "github.com/yunionio/onecloud/pkg/scheduler/manager" +) + +var counter = 0 + +func InstallPingHandler(r *gin.Engine) { + r.GET("/ping", pingHandler) + r.GET("/switch", switchHandler) +} + +// pingHandler is a handler of ping-pong service that just for +// testing scheduler serevice. +func pingHandler(c *gin.Context) { + if !schedman.IsReady() { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) + return + } + log.Infof("%v", c.Request.Body) + + c.JSON(http.StatusOK, "pong") +} + +// switchHandler is a handler of switch service that just for +// testing scheduler serevice. +func switchHandler(c *gin.Context) { + if !schedman.IsReady() { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("Global scheduler not init")) + return + } + log.Infof("%v", c.Request.Body) + counter++ + var result string + if counter%2 == 1 { + o.GetOptions().DisableBaremetalPredicates = true + result = fmt.Sprintf("ignore_baremetal_filter_switch is true") + } else { + o.GetOptions().DisableBaremetalPredicates = false + result = fmt.Sprintf("ignore_baremetal_filter_switch is false") + } + + c.JSON(http.StatusOK, result) +} diff --git a/pkg/scheduler/handler/version.go b/pkg/scheduler/handler/version.go new file mode 100644 index 0000000000..5720607590 --- /dev/null +++ b/pkg/scheduler/handler/version.go @@ -0,0 +1,17 @@ +package handler + +import ( + "net/http" + + "gopkg.in/gin-gonic/gin.v1" + + "github.com/yunionio/pkg/util/version" +) + +func InstallVersionHandler(r *gin.Engine) { + r.GET("/version", versionHandler) +} + +func versionHandler(c *gin.Context) { + c.String(http.StatusOK, version.GetShortString()) +} diff --git a/pkg/scheduler/manager/completed_queue.go b/pkg/scheduler/manager/completed_queue.go new file mode 100644 index 0000000000..abda1adaed --- /dev/null +++ b/pkg/scheduler/manager/completed_queue.go @@ -0,0 +1,93 @@ +package manager + +import ( + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/pkg/utils" +) + +type CompletedManager struct { + completedChannel chan *api.CompletedNotifyArgs + stopCh <-chan struct{} +} + +func NewCompletedManager(stopCh <-chan struct{}) *CompletedManager { + return &CompletedManager{ + completedChannel: make(chan *api.CompletedNotifyArgs, o.GetOptions().CompletedQueueMaxLength), + stopCh: stopCh, + } +} + +func (c *CompletedManager) Add(completedNotifyArgs *api.CompletedNotifyArgs) { + c.completedChannel <- completedNotifyArgs +} + +func (c *CompletedManager) Run() { + t := time.Tick(utils.ToDuration(o.GetOptions().CompletedQueueConsumptionPeriod)) + + removeSession := func() { + //completedNotifyArgs := <-c.completedChannel + //pool, err := schedManager.ReservedPoolManager.SearchReservedPoolBySessionID(completedNotifyArgs.SessionID) + //if err != nil { + //log.Errorln(err) + //return + //} + + //sessionItem := pool.GetSessionItem(completedNotifyArgs.SessionID) + //if sessionItem == nil { + //log.Errorln(fmt.Errorf("session %v not found\n", completedNotifyArgs.SessionID)) + //return + //} + //candidateIds := sessionItem.AllCandidateIDs() + + // load candidates + //if len(candidateIds) > 0 { + //schedManager.CandidateManager.Reload(pool.Name, candidateIds) + //} + + // remove session + //pool.RemoveSession(completedNotifyArgs.SessionID) + } + + reloadAndRemoveSessions := func() { + completedRequestNumber := len(c.completedChannel) + // If the completedRequestNumber then return right now. + if completedRequestNumber <= 0 { + return + } + + wg := &utils.WaitGroupWrapper{} + for i := 0; i < completedRequestNumber; i++ { + wg.Wrap(removeSession) + } + + if ok := utils.WaitTimeOut(wg, time.Duration(completedRequestNumber)*utils.ToDuration(o.GetOptions().CompletedQueueConsumptionTimeout)); !ok { + log.Errorln("time out reload data in completed when remove sessions.") + } + } + + // Watching the completed sessions. + for { + select { + case <-t: + reloadAndRemoveSessions() + case <-c.stopCh: + // update all the sessions before return. + reloadAndRemoveSessions() + close(c.completedChannel) + c.completedChannel = nil + log.Errorln("completed manager EXIT!") + return + default: + // if sessions' number is bigger then 10 then reload and remove. + if len(c.completedChannel) >= o.GetOptions().CompletedQueueDealLength { + reloadAndRemoveSessions() + } else { + time.Sleep(10 * time.Second) + } + } + } +} diff --git a/pkg/scheduler/manager/expire_queue.go b/pkg/scheduler/manager/expire_queue.go new file mode 100644 index 0000000000..0732caa333 --- /dev/null +++ b/pkg/scheduler/manager/expire_queue.go @@ -0,0 +1,138 @@ +package manager + +import ( + "sync" + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + u "github.com/yunionio/pkg/utils" +) + +type ExpireManager struct { + expireChannel chan *api.ExpireArgs + stopCh <-chan struct{} +} + +func NewExpireManager(stopCh <-chan struct{}) *ExpireManager { + return &ExpireManager{ + expireChannel: make(chan *api.ExpireArgs, o.GetOptions().ExpireQueueMaxLength), + stopCh: stopCh, + } +} + +func (e *ExpireManager) Add(expireArgs *api.ExpireArgs) { + e.expireChannel <- expireArgs +} + +func (e *ExpireManager) Run() { + t := time.Tick(u.ToDuration(o.GetOptions().ExpireQueueConsumptionPeriod)) + + notInSession := func(ids []string, resType string) []string { + //var newIds []string + //for _, id := range ids { + //if !schedManager.ReservedPoolManager.InSession(resType, id) { + //newIds = append(newIds, id) + //} + //} + //return newIds + return ids + } + + waitTimeOut := func(wg *sync.WaitGroup, timeout time.Duration) bool { + ch := make(chan struct{}) + go func() { + wg.Wait() + close(ch) + }() + select { + case <-ch: + return true + case <-time.After(timeout): + return false + } + } + + batchMergeExpire := func() { + expireRequestNumber := len(e.expireChannel) + // If the expireRequestNumber then return right now. + if expireRequestNumber <= 0 { + return + } + dirtyHostMap := make(map[string]int, expireRequestNumber) + dirtyBaremetalMap := make(map[string]int, expireRequestNumber) + dirtyHosts := make([]string, 0) + dirtyBaremetals := make([]string, 0) + // Merge all same host. + for i := 0; i < expireRequestNumber; i++ { + expireArgs := <-e.expireChannel + log.V(4).Infof("Get expireArgs from channel: %#v", expireArgs) + for _, host := range expireArgs.DirtyHosts { + if _, ok := dirtyHostMap[host]; !ok { + dirtyHostMap[host] = len(dirtyHosts) + dirtyHosts = append(dirtyHosts, host) + } + } + for _, baremetal := range expireArgs.DirtyBaremetals { + if _, ok := dirtyBaremetalMap[baremetal]; !ok { + dirtyBaremetalMap[baremetal] = len(dirtyBaremetals) + dirtyBaremetals = append(dirtyBaremetals, baremetal) + } + } + } + log.V(4).Infof("batchMergeExpire dirtyHosts: %v, dirtyBaremetals: %v", dirtyHosts, dirtyBaremetals) + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + dirtyHosts = notInSession(dirtyHosts, "host") + if len(dirtyHosts) > 0 { + log.V(10).Debugf("CleanDirty Hosts: %v\n", dirtyHosts) + _, err := schedManager.CandidateManager.Reload("host", dirtyHosts) + schedManager.CandidateManager.CleanDirtyCandidatesOnce(dirtyHosts) + if err != nil { + log.Errorf("%v", err) + } + } + }() + + go func() { + defer wg.Done() + dirtyBaremetals = notInSession(dirtyBaremetals, "baremetal") + if len(dirtyBaremetals) > 0 { + log.V(10).Debugf("CleanDirty Baremetals: %v\n", dirtyBaremetals) + _, err := schedManager.CandidateManager.Reload("baremetal", dirtyBaremetals) + schedManager.CandidateManager.CleanDirtyCandidatesOnce(dirtyBaremetals) + if err != nil { + log.Errorf("%v", err) + } + } + }() + if ok := waitTimeOut(wg, u.ToDuration(o.GetOptions().ExpireQueueConsumptionTimeout)); !ok { + log.Errorln("time out reload data.") + } + } + + // Watching the expires. + for { + select { + case <-t: + batchMergeExpire() + case <-e.stopCh: + // update all the expire before return + batchMergeExpire() + close(e.expireChannel) + e.expireChannel = nil + log.Errorln("expire manager EXIT!") + return + default: + // if expire number is bigger then 80 then update + if len(e.expireChannel) >= o.GetOptions().ExpireQueueDealLength { + batchMergeExpire() + } else { + time.Sleep(1 * time.Second) + } + } + } +} diff --git a/pkg/scheduler/manager/manager.go b/pkg/scheduler/manager/manager.go new file mode 100644 index 0000000000..8296c02e7e --- /dev/null +++ b/pkg/scheduler/manager/manager.go @@ -0,0 +1,453 @@ +package manager + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/cache/candidate" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/data_manager" + "github.com/yunionio/pkg/utils" +) + +const defaultIgnorePool = true + +var schedManager *SchedulerManager + +type SchedulerManager struct { + ExpireManager *ExpireManager + CompletedManager *CompletedManager + HistoryManager *HistoryManager + TaskManager *TaskManager + + DataManager *data_manager.DataManager + CandidateManager *data_manager.CandidateManager + //ReservedPoolManager *data_manager.ReservedPoolManager + //NetworkManager *data_manager.NetworkManager +} + +func NewSchedulerManager(stopCh <-chan struct{}) *SchedulerManager { + sm := &SchedulerManager{} + sm.DataManager = data_manager.NewDataManager(stopCh) + sm.CandidateManager = data_manager.NewCandidateManager(sm.DataManager, stopCh) + sm.ExpireManager = NewExpireManager(stopCh) + sm.CompletedManager = NewCompletedManager(stopCh) + sm.HistoryManager = NewHistoryManager(stopCh) + sm.TaskManager = NewTaskManager(stopCh) + //sm.ReservedPoolManager = data_manager.NewReservedPoolManager(stopCh) + //sm.NetworkManager = data_manager.NewNetworkManager(sm.DataManager, sm.ReservedPoolManager) + + return sm +} + +func InitAndStart(stopCh <-chan struct{}) { + if schedManager != nil { + log.Warningf("Global scheduler already init.") + return + } + schedManager = NewSchedulerManager(stopCh) + go schedManager.start() + log.Infof("InitAndStart ok") +} + +func (sm *SchedulerManager) start() { + startFuncs := []func(){ + sm.ExpireManager.Run, + sm.CompletedManager.Run, + sm.HistoryManager.Run, + sm.TaskManager.Run, + sm.DataManager.Run, + sm.CandidateManager.Run, + //sm.ReservedPoolManager.Run, + //sm.NetworkManager.Run, + } + for _, f := range startFuncs { + go f() + } +} + +func (sm *SchedulerManager) schedule(info *api.SchedInfo) ([]*core.SchedResultItem, error) { + log.V(10).Infof("SchedulerManager do schedule, input: %#v", info) + task, err := sm.TaskManager.AddTask(sm, info) + if err != nil { + return nil, err + } + + sm.HistoryManager.NewHistoryItem(task) + results, err := task.Wait() + if err != nil { + return nil, err + } + log.V(10).Infof("SchedulerManager finish schedule, selected candidates: %#v", results) + return results, nil +} + +// NewSessionID returns the current timestamp of a string type with precision of +// milliseconds. And it should be consistent with the format of Region. +// just like: 1509699887616 +func NewSessionID() string { + return fmt.Sprintf("%v", time.Now().UnixNano()/1000000) +} + +// Schedule process the request data that is scheduled for dispatch and complements +// the session information. +func Schedule(info *api.SchedInfo) ([]*core.SchedResultItem, error) { + if len(info.SessionID) == 0 { + info.SessionID = NewSessionID() + } + return schedManager.schedule(info) +} + +func IsReady() bool { + return schedManager != nil +} + +func GetCandidateManager() *data_manager.CandidateManager { + return schedManager.CandidateManager +} + +func Expire(expireArgs *api.ExpireArgs) (*api.ExpireResult, error) { + schedManager.ExpireManager.Add(expireArgs) + return &api.ExpireResult{}, nil +} + +func CompletedNotify(completedNotifyArgs *api.CompletedNotifyArgs) (*api.CompletedNotifyResult, error) { + schedManager.CompletedManager.Add(completedNotifyArgs) + return &api.CompletedNotifyResult{}, nil +} + +func getHostCandidatesList(args *api.CandidateListArgs) (*api.CandidateListResult, error) { + r := new(api.CandidateListResult) + r.Limit = args.Limit + r.Offset = args.Offset + cs, err := GetCandidateManager().GetCandidates(data_manager.CandidateGetArgs{ + ResType: "host", + ZoneID: args.Zone, + PoolID: args.Pool, + IgnorePool: defaultIgnorePool, + }) + if err != nil { + return nil, fmt.Errorf("Get host candidates err: %v", err) + } + return GetCandidateHostList(cs, args, r) +} + +func getBaremetalCandidatesList(args *api.CandidateListArgs) (*api.CandidateListResult, error) { + r := new(api.CandidateListResult) + r.Limit = args.Limit + r.Offset = args.Offset + cs, err := GetCandidateManager().GetCandidates(data_manager.CandidateGetArgs{ + ResType: "baremetal", + ZoneID: args.Zone, + PoolID: args.Pool, + IgnorePool: defaultIgnorePool, + }) + if err != nil { + return nil, fmt.Errorf("Get baremetal candidates err: %v", err) + } + return GetCandidateBaremetalList(cs, args, r) +} + +func mergeAllCandidateList(host, baremetal *api.CandidateListResult, args *api.CandidateListArgs) (*api.CandidateListResult, error) { + res := make([]api.CandidateListResultItem, 0) + if host.Total == args.Limit { + return host, nil + } else { + // must < args.Limit + res = append(res, host.Data...) + } + + for _, bm := range baremetal.Data { + if int64(len(res)) >= args.Limit { + break + } + res = append(res, bm) + } + + r := new(api.CandidateListResult) + r.Limit = args.Limit + r.Offset = args.Offset + r.Total = int64(len(res)) + r.Data = res + + return r, nil +} + +func GetCandidateList(args *api.CandidateListArgs) (*api.CandidateListResult, error) { + var ( + hostRes, bmRes *api.CandidateListResult + err error + ) + switch args.Type { + case "all": + hostRes, err = getHostCandidatesList(args) + if err != nil { + return nil, err + } + + bmRes, err = getBaremetalCandidatesList(args) + if err != nil { + return nil, err + } + return mergeAllCandidateList(hostRes, bmRes, args) + + case "host": + return getHostCandidatesList(args) + + case "baremetal": + return getBaremetalCandidatesList(args) + + default: + return nil, fmt.Errorf("Unsupport candidate type %q", args.Type) + } +} + +func GetCandidateHostList( + candidates []core.Candidater, + args *api.CandidateListArgs, + r *api.CandidateListResult, +) (*api.CandidateListResult, error) { + r.Total = int64(len(candidates)) + + for _, cc := range candidates { + if int64(len(r.Data)) >= args.Limit { + break + } + c := cc.(*candidate.HostDesc) + mem := api.NewResultResourceInt64( + c.GetFreeMemSize(false), + c.GetReservedMemSize(), + c.GetTotalMemSize(false)) + + cpu := api.NewResultResourceInt64( + c.GetFreeCPUCount(false), + c.GetReservedCPUCount(), + c.GetTotalCPUCount(false)) + + storage := api.NewResultResourceInt64( + c.GetFreeLocalStorageSize(false), + c.GetReservedStorageSize(), + c.GetTotalLocalStorageSize(false)) + + item := api.CandidateListResultItem{ + ID: c.IndexKey(), + Name: c.Name, + Mem: *mem, + Cpu: *cpu, + Storage: *storage, + + Status: c.Status, + HostStatus: c.HostStatus, + HostType: c.GetHostType(), + EnableStatus: c.GetEnableStatus(), + } + r.Data = append(r.Data, item) + } + return r, nil +} + +func GetCandidateBaremetalList( + candidates []core.Candidater, + args *api.CandidateListArgs, + r *api.CandidateListResult, +) (*api.CandidateListResult, error) { + r.Total = int64(len(candidates)) + + for _, cc := range candidates { + if int64(len(r.Data)) >= args.Limit { + break + } + c := cc.(*candidate.BaremetalDesc) + + mem := api.NewResultResourceInt64( + c.FreeMemSize(), + 0, + c.MemSize) + + cpu := api.NewResultResourceInt64( + c.FreeCPUCount(), + 0, + c.CPUCount) + + storage := api.NewResultResourceInt64( + c.FreeStorageSize(), + 0, + c.StorageSize) + + item := api.CandidateListResultItem{ + ID: c.IndexKey(), + Name: c.Name, + Mem: *mem, + Cpu: *cpu, + Storage: *storage, + + Status: c.Status, + HostStatus: c.HostStatus, + HostType: c.GetHostType(), + EnableStatus: c.GetEnableStatus(), + } + r.Data = append(r.Data, item) + } + return r, nil +} + +func GetCandidateDetail(args *api.CandidateDetailArgs) (*api.CandidateDetailResult, error) { + r := new(api.CandidateDetailResult) + candidate, err := GetCandidateManager().GetCandidate(args.ID, args.Type) + if err != nil { + return nil, err + } + + r.Candidate = candidate + return r, nil +} + +func Cleanup(cleanupArgs *api.CleanupArgs) (*api.CleanupResult, error) { + r := new(api.CleanupResult) + cm := GetCandidateManager() + + if cleanupArgs.ResType != "" { + cm.ReloadAll(cleanupArgs.ResType) + } else { + cm.ReloadAll("host") + cm.ReloadAll("baremetal") + } + + return r, nil +} + +func GetHistoryList(historyArgs *api.HistoryArgs) (*api.HistoryResult, error) { + offset, limit, all := historyArgs.Offset, historyArgs.Limit, historyArgs.All + if limit == int64(0) { + limit = int64(50) + } + + historyItems, total := schedManager.HistoryManager.GetHistoryList(offset, limit, all) + items := []*api.HistoryItem{} + + for _, hi := range historyItems { + items = append(items, newHistoryItem(hi)) + } + + return &api.HistoryResult{ + Items: items, + Offset: offset, + Limit: int64(len(items)), + Total: total, + }, nil +} + +func newHistoryItem(historyItem *HistoryItem) *api.HistoryItem { + task := historyItem.Task + schedInfo := task.SchedInfo + + tenants := []string{} + forGuests := []string{} + countDict := make(map[string]int64) + + data := schedInfo.Data + tenants = append(tenants, data.OwnerTenantID) + + for _, forGuest := range data.ForGuests { + forGuests = append(forGuests, fmt.Sprintf("%v(%v)", forGuest.ID, forGuest.Name)) + } + + guestType := data.Hypervisor + if c, ok := countDict[guestType]; !ok { + countDict[guestType] = data.Count + } else { + countDict[guestType] = c + data.Count + } + + counts := []string{} + for guestType, count := range countDict { + s := "" + if count > 1 { + s = "s" + } + + counts = append(counts, fmt.Sprintf("%v %v%v", count, guestType, s)) + } + + countStr := strings.Join(counts, ", ") + + return &api.HistoryItem{ + Time: historyItem.Time.Local().Format("2006-01-02 15:04:05"), + Consuming: fmt.Sprintf("%s", task.Consuming), + SessionID: task.GetSessionID(), + Status: task.GetStatus(), + Tenants: utils.Distinct(tenants), + Guests: forGuests, + Count: countStr, + IsSuggestion: schedInfo.IsSuggestion, + } +} + +func GetHistoryDetail(historyDetailArgs *api.HistoryDetailArgs) (*api.HistoryDetailResult, error) { + historyItem := schedManager.HistoryManager.GetHistory(historyDetailArgs.ID) + if historyItem == nil { + return nil, fmt.Errorf("History '%v' not found", historyDetailArgs.ID) + } + + task := historyItem.Task + schedInfo := task.SchedInfo + historyTasks := []api.HistoryTask{} + + data := schedInfo.Data + taskExecutor := task.GetTaskExecutor(data.Tag) + historyTask := api.HistoryTask{ + Type: data.Hypervisor, + Data: data, + } + + if taskExecutor != nil { + historyTask.Status = taskExecutor.Status + historyTask.Time = taskExecutor.Time.Local().Format("2006-01-02 15:04:05") + historyTask.Consuming = fmt.Sprintf("%s", taskExecutor.Consuming) + + resultItems, err := taskExecutor.GetResult() + historyTask.Result = resultItems + if err != nil { + historyTask.Error = fmt.Sprintf("%v", err) + } + + if historyDetailArgs.Log { + historyTask.Logs = taskExecutor.GetLogs() + } + } + + historyTasks = append(historyTasks, historyTask) + + var inputStr, outputStr, errStr string + result, err := task.GetResult() + + if err != nil { + errStr = fmt.Sprintf("%v", err) + } else { + if bytes, err0 := json.MarshalIndent(result, "", " "); err0 == nil { + outputStr = string(bytes) + } + } + + if historyDetailArgs.Raw { + inputStr = schedInfo.Raw + } + + historyDetail := &api.HistoryDetail{ + Time: historyItem.Time.Local().Format("2006-01-02 15:04:05"), + Consuming: fmt.Sprintf("%s", task.Consuming), + SessionID: task.GetSessionID(), + Tasks: historyTasks, + Input: inputStr, + Output: outputStr, + Error: errStr, + } + + return &api.HistoryDetailResult{ + Detail: historyDetail, + }, nil +} diff --git a/pkg/scheduler/manager/scheduler.go b/pkg/scheduler/manager/scheduler.go new file mode 100644 index 0000000000..6e9b200330 --- /dev/null +++ b/pkg/scheduler/manager/scheduler.go @@ -0,0 +1,222 @@ +package manager + +import ( + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" + "github.com/yunionio/onecloud/pkg/scheduler/data_manager" + "github.com/yunionio/onecloud/pkg/scheduler/factory" +) + +type CandidatesProvider interface { + ProviderType() string + CandidateType() string + Candidates() ([]core.Candidater, error) + CandidateManager() *data_manager.CandidateManager +} + +func candidatesByProvider(provider CandidatesProvider, schedData *api.SchedData) ([]core.Candidater, error) { + var hosts []core.Candidater + var err error + + candidateManager := provider.CandidateManager() + if len(schedData.Candidates) > 0 { + hosts, err = candidateManager.GetCandidatesByIds(provider.CandidateType(), schedData.Candidates) + } else { + args := data_manager.CandidateGetArgs{ + ResType: provider.CandidateType(), + ZoneID: schedData.ZoneID, + PoolID: schedData.PoolID, + } + hosts, err = candidateManager.GetCandidates(args) + } + if err != nil { + return nil, err + } + return hosts, nil +} + +type BaseCandidateProvider struct { + scheduler Scheduler +} + +func (b *BaseCandidateProvider) CandidateManager() *data_manager.CandidateManager { + return b.scheduler.CandidateManager() +} + +type HostCandidatesProvider struct { + *BaseCandidateProvider +} + +func NewHostCandidatesProvider(s Scheduler) *HostCandidatesProvider { + return &HostCandidatesProvider{ + BaseCandidateProvider: &BaseCandidateProvider{scheduler: s}, + } +} + +func (h *HostCandidatesProvider) ProviderType() string { + return factory.DefaultProvider +} + +func (h *HostCandidatesProvider) CandidateType() string { + return api.HostTypeHost +} + +func (h *HostCandidatesProvider) Candidates() ([]core.Candidater, error) { + return candidatesByProvider(h, h.scheduler.SchedData()) +} + +type BaremetalCandidatesProvider struct { + *BaseCandidateProvider +} + +func NewBaremetalCandidatesProvider(s Scheduler) *BaremetalCandidatesProvider { + return &BaremetalCandidatesProvider{ + BaseCandidateProvider: &BaseCandidateProvider{scheduler: s}, + } +} + +func (b *BaremetalCandidatesProvider) ProviderType() string { + return factory.BaremetalProvider +} + +func (b *BaremetalCandidatesProvider) CandidateType() string { + return api.SchedTypeBaremetal +} + +func (b *BaremetalCandidatesProvider) Candidates() ([]core.Candidater, error) { + return candidatesByProvider(b, b.scheduler.SchedData()) +} + +type Scheduler interface { + SchedData() *api.SchedData + CandidateManager() *data_manager.CandidateManager + + // Schedule process + BeforePredicate() error + Predicates() (map[string]core.FitPredicate, error) + PriorityConfigs() ([]core.PriorityConfig, error) + + // Schedule input get function + Unit() *core.Unit + Candidates() ([]core.Candidater, error) + + DirtySelectedCandidates([]*core.SelectedCandidate) +} + +type BaseScheduler struct { + schedManager *SchedulerManager + schedInfo *api.SchedInfo +} + +func newBaseScheduler(manager *SchedulerManager, info *api.SchedInfo) (*BaseScheduler, error) { + s := &BaseScheduler{ + schedManager: manager, + schedInfo: info, + } + return s, nil +} + +func (s *BaseScheduler) NewSchedUnit() *core.Unit { + return core.NewScheduleUnit(s.schedInfo, s.schedManager) +} + +func (s *BaseScheduler) CandidateManager() *data_manager.CandidateManager { + return s.schedManager.CandidateManager +} + +func (s *BaseScheduler) SchedData() *api.SchedData { + return s.schedInfo.Data +} + +func (s *BaseScheduler) Unit() *core.Unit { + return s.NewSchedUnit() +} + +func (s *BaseScheduler) BeforePredicate() error { + return nil +} + +func (s *BaseScheduler) DirtySelectedCandidates(scs []*core.SelectedCandidate) { + s.CandidateManager().SetCandidatesDirty(scs) +} + +// GuestScheduler for guest type schedule +type GuestScheduler struct { + *BaseScheduler + algorithmProvider *factory.AlgorithmProviderConfig + candidatesProvider *HostCandidatesProvider +} + +func newGuestScheduler(manager *SchedulerManager, info *api.SchedInfo) (*GuestScheduler, error) { + bs, err := newBaseScheduler(manager, info) + if err != nil { + return nil, err + } + + algorithmProvider, err := factory.GetAlgorithmProvider(factory.DefaultProvider) + if err != nil { + return nil, err + } + + gs := &GuestScheduler{ + BaseScheduler: bs, + algorithmProvider: algorithmProvider, + } + candidatesProvider := NewHostCandidatesProvider(gs) + gs.candidatesProvider = candidatesProvider + + return gs, nil +} + +func (gs *GuestScheduler) Candidates() ([]core.Candidater, error) { + return gs.candidatesProvider.Candidates() +} + +func (gs *GuestScheduler) Predicates() (map[string]core.FitPredicate, error) { + return factory.GetPredicates(gs.algorithmProvider.FitPredicateKeys) +} + +func (gs *GuestScheduler) PriorityConfigs() ([]core.PriorityConfig, error) { + return factory.GetPriorityConfigs(gs.algorithmProvider.PriorityKeys) +} + +// BaremetalScheduler for baremetal type schedule +type BaremetalScheduler struct { + *BaseScheduler + algorithmProvider *factory.AlgorithmProviderConfig + candidatesProvider *BaremetalCandidatesProvider +} + +func newBaremetalScheduler(manager *SchedulerManager, info *api.SchedInfo) (*BaremetalScheduler, error) { + bs, err := newBaseScheduler(manager, info) + if err != nil { + return nil, err + } + + algorithmProvider, err := factory.GetAlgorithmProvider(factory.BaremetalProvider) + if err != nil { + return nil, err + } + + bms := &BaremetalScheduler{ + BaseScheduler: bs, + algorithmProvider: algorithmProvider, + } + + cp := NewBaremetalCandidatesProvider(bms) + bms.candidatesProvider = cp + + return bms, nil +} + +func (bs *BaremetalScheduler) Candidates() ([]core.Candidater, error) { + return bs.candidatesProvider.Candidates() +} + +func (bs *BaremetalScheduler) Predicates() (map[string]core.FitPredicate, error) { + return factory.GetPredicates(bs.algorithmProvider.FitPredicateKeys) +} + +func (bs *BaremetalScheduler) PriorityConfigs() ([]core.PriorityConfig, error) { + return factory.GetPriorityConfigs(bs.algorithmProvider.PriorityKeys) +} diff --git a/pkg/scheduler/manager/task_history.go b/pkg/scheduler/manager/task_history.go new file mode 100644 index 0000000000..eb9faed5ec --- /dev/null +++ b/pkg/scheduler/manager/task_history.go @@ -0,0 +1,145 @@ +package manager + +import ( + "container/list" + "sync" + "time" + + o "github.com/yunionio/onecloud/cmd/scheduler/options" + u "github.com/yunionio/pkg/utils" + "github.com/yunionio/pkg/util/wait" +) + +type HistoryItem struct { + Task *Task + Time time.Time +} + +func NewHistoryItem(task *Task) *HistoryItem { + return &HistoryItem{ + Task: task, + Time: time.Now(), + } +} + +func (h *HistoryItem) ToMap() map[string]string { + ret := make(map[string]string) + ret["SessionID"] = h.Task.GetSessionID() + return ret +} + +func (h *HistoryItem) IsSuggestion() bool { + return h.Task.SchedInfo.IsSuggestion +} + +type HistoryManager struct { + capacity int + historyMap map[string]*HistoryItem + historyList *list.List + normalHistoryList *list.List // exclude scheduler-test + lock sync.Mutex + stopCh <-chan struct{} +} + +func NewHistoryManager(stopCh <-chan struct{}) *HistoryManager { + return &HistoryManager{ + capacity: o.GetOptions().SchedulerHistoryLimit, + historyMap: make(map[string]*HistoryItem), + historyList: list.New(), + normalHistoryList: list.New(), + lock: sync.Mutex{}, + stopCh: stopCh, + } +} + +func (m *HistoryManager) NewHistoryItem(task *Task) *HistoryItem { + m.lock.Lock() + defer m.lock.Unlock() + + for _, ls := range []*list.List{m.historyList, m.normalHistoryList} { + for ls.Len() > m.capacity { + h := ls.Back() + ls.Remove(h) + } + } + + historyItem := NewHistoryItem(task) + m.historyList.PushFront(historyItem) + if !historyItem.IsSuggestion() { + m.normalHistoryList.PushFront(historyItem) + } + m.historyMap[task.GetSessionID()] = historyItem + + return historyItem +} + +func (m *HistoryManager) cleanHistoryMap() { + m.lock.Lock() + defer m.lock.Unlock() + + if len(m.historyMap) <= m.capacity { + return + } + oldHistoryMap := m.historyMap + newHistoryMap := make(map[string]*HistoryItem) + for _, ls := range []*list.List{m.historyList, m.normalHistoryList} { + for element := ls.Front(); element != nil; element = element.Next() { + sessionId := (element.Value.(*HistoryItem)).Task.GetSessionID() + if h, ok := oldHistoryMap[sessionId]; ok { + newHistoryMap[sessionId] = h + } + } + } + oldHistoryMap = nil + + m.historyMap = newHistoryMap +} + +func (m *HistoryManager) Run() { + go wait.Until(m.cleanHistoryMap, u.ToDuration(o.GetOptions().SchedulerHistoryCleanPeriod), m.stopCh) +} + +func (m *HistoryManager) GetHistoryList(offset int64, limit int64, all bool) ([]*HistoryItem, int64) { + m.lock.Lock() + defer m.lock.Unlock() + + var hList *list.List + if all { + hList = m.historyList + } else { + hList = m.normalHistoryList + } + + total := int64(hList.Len()) + historyItems := []*HistoryItem{} + element := hList.Front() + for index := int64(0); index < offset; index++ { + if element != nil { + element = element.Next() + } else { + return historyItems, total + } + } + + for index := int64(0); index < limit; index++ { + if element != nil { + historyItems = append(historyItems, element.Value.(*HistoryItem)) + element = element.Next() + } else { + break + } + } + + return historyItems, total +} + +func (m *HistoryManager) GetHistory(sessionId string) *HistoryItem { + m.lock.Lock() + defer m.lock.Unlock() + + if historyItem, ok := m.historyMap[sessionId]; ok { + return historyItem + } + + return nil +} diff --git a/pkg/scheduler/manager/task_queue.go b/pkg/scheduler/manager/task_queue.go new file mode 100644 index 0000000000..5a4decf06a --- /dev/null +++ b/pkg/scheduler/manager/task_queue.go @@ -0,0 +1,360 @@ +package manager + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/onecloud/pkg/scheduler/core" +) + +const ( + TaskExecutorStatusWaiting string = "waiting" + TaskExecutorStatusRunning string = "running" + TaskExecutorStatusFailed string = "failed" + TaskExecutorStatusKilled string = "killed" + TaskExecutorStatusSuccess string = "success" +) + +type TaskExecuteCallback func(task *TaskExecutor) + +type TaskExecutor struct { + Tag string + Status string + Time time.Time + Consuming time.Duration + scheduler Scheduler + callback TaskExecuteCallback + unit *core.Unit + + resultItems []*core.SchedResultItem + resultError error + logs []string + completed bool +} + +func NewTaskExecutor(scheduler Scheduler, taskExecuteCallback TaskExecuteCallback) *TaskExecutor { + return &TaskExecutor{ + Tag: scheduler.SchedData().Tag, + Status: TaskExecutorStatusWaiting, + Time: time.Now(), + scheduler: scheduler, + callback: taskExecuteCallback, + completed: false, + } +} + +func (te *TaskExecutor) Execute() { + te.Status = TaskExecutorStatusRunning + te.resultItems, te.resultError = te.execute() + te.completed = true + + if te.resultError != nil { + te.Status = TaskExecutorStatusFailed + } else { + te.Status = TaskExecutorStatusSuccess + } + + if te.callback != nil { + te.callback(te) + } +} + +func (te *TaskExecutor) execute() ([]*core.SchedResultItem, error) { + scheduler := te.scheduler + genericScheduler, err := core.NewGenericScheduler(scheduler.(core.Scheduler)) + if err != nil { + return nil, err + } + + // Get current resources from DB. + candidates, err := scheduler.Candidates() + if err != nil { + return nil, err + } + + te.unit = scheduler.Unit() + return genericScheduler.Schedule(te.unit, candidates) +} + +func (te *TaskExecutor) cleanup() { + te.unit = nil + te.scheduler = nil + te.callback = nil +} + +func (te *TaskExecutor) Kill() { + if te.Status == TaskExecutorStatusWaiting { + te.Status = TaskExecutorStatusKilled + } +} + +func (te *TaskExecutor) GetResult() ([]*core.SchedResultItem, error) { + return te.resultItems, te.resultError +} + +func (te *TaskExecutor) GetLogs() []string { + return te.logs +} + +type TaskExecutorQueue struct { + schedType string + poolId string + queue chan *TaskExecutor + running bool +} + +func (teq *TaskExecutorQueue) AddTaskExecutor(scheduler Scheduler, + callback TaskExecuteCallback) *TaskExecutor { + taskExecutor := NewTaskExecutor(scheduler, callback) + teq.queue <- taskExecutor + return taskExecutor +} + +func NewTaskExecutorQueue(schedType string, poolId string, stopCh <-chan struct{}) *TaskExecutorQueue { + taskExecutorQueue := &TaskExecutorQueue{ + schedType: schedType, + poolId: poolId, + running: false, + } + + taskExecutorQueue.Start(stopCh) + return taskExecutorQueue +} + +func (teq *TaskExecutorQueue) Start(stopCh <-chan struct{}) { + if !teq.running { + teq.running = true + teq.queue = make(chan *TaskExecutor, 5000) + + go func() { + defer close(teq.queue) + + var taskExecutor *TaskExecutor + for taskExecutor = <-teq.queue; teq.running; taskExecutor = <-teq.queue { + if taskExecutor.Status == TaskExecutorStatusWaiting { + taskExecutor.Execute() + } + } + }() + + go func() { + <-stopCh + teq.running = false + teq.queue <- nil + }() + } +} + +type TaskExecutorQueueManager struct { + taskExecutorMap map[string]*TaskExecutorQueue + lock sync.Mutex + stopCh <-chan struct{} +} + +func NewTaskExecutorQueueManager(stopCh <-chan struct{}) *TaskExecutorQueueManager { + return &TaskExecutorQueueManager{ + taskExecutorMap: make(map[string]*TaskExecutorQueue), + lock: sync.Mutex{}, + stopCh: stopCh, + } +} + +func (teqm *TaskExecutorQueueManager) GetQueue(schedType string, poolId string, +) *TaskExecutorQueue { + teqm.lock.Lock() + defer teqm.lock.Unlock() + + var ( + key string + taskExecutorQueue *TaskExecutorQueue + ok bool + ) + + key = fmt.Sprintf("%v:%v", schedType, poolId) + if taskExecutorQueue, ok = teqm.taskExecutorMap[key]; !ok { + taskExecutorQueue = NewTaskExecutorQueue(schedType, poolId, teqm.stopCh) + teqm.taskExecutorMap[key] = taskExecutorQueue + } + + return taskExecutorQueue +} + +func (teqm *TaskExecutorQueueManager) AddTaskExecutor( + scheduler Scheduler, callback TaskExecuteCallback) *TaskExecutor { + schedData := scheduler.SchedData() + log.V(10).Infof("AddTaskExecutor schedData: %#v", schedData) + taskQueue := teqm.GetQueue(schedData.Type, schedData.PoolID) + return taskQueue.AddTaskExecutor(scheduler, callback) +} + +type TaskManager struct { + taskExecutorQueueManager *TaskExecutorQueueManager + stopCh <-chan struct{} + lock sync.Mutex +} + +func NewTaskManager(stopCh <-chan struct{}) *TaskManager { + return &TaskManager{ + taskExecutorQueueManager: NewTaskExecutorQueueManager(stopCh), + stopCh: stopCh, + lock: sync.Mutex{}, + } +} + +func (tm *TaskManager) Run() { + // Do nothing +} + +// AddTask provides an interface to increase the scheduling task, +// it will be a scheduling request by the host specification type +// split into multiple scheduling tasks, added to the scheduling +// task manager. +func (tm *TaskManager) AddTask(schedulerManager *SchedulerManager, schedInfo *api.SchedInfo) (*Task, error) { + var ( + scheduler Scheduler + err error + ) + + task := NewTask(schedulerManager, schedInfo) + // Split into multiple scheduling tasks by host specification type. + if schedInfo.Data.Hypervisor == api.SchedTypeBaremetal { + scheduler, err = newBaremetalScheduler(schedulerManager, schedInfo) + } else { + scheduler, err = newGuestScheduler(schedulerManager, schedInfo) + } + if err != nil { + return nil, err + } + + taskExecutorCallback := func(taskExecutor *TaskExecutor) { + taskExecutor.Consuming = time.Since(taskExecutor.Time) + task.onTaskCompleted(taskExecutor) + } + + tm.lock.Lock() + defer tm.lock.Unlock() + + taskExecutor := tm.taskExecutorQueueManager.AddTaskExecutor(scheduler, taskExecutorCallback) + task.taskExecutors = append(task.taskExecutors, taskExecutor) + + return task, nil +} + +type Task struct { + Time time.Time + SchedInfo *api.SchedInfo + Consuming time.Duration + taskExecutors []*TaskExecutor + manager *SchedulerManager + lock sync.Mutex + waitCh chan struct{} + + completedCount int + resultItems []*core.SchedResultItem + resultError error +} + +func NewTask(manager *SchedulerManager, schedInfo *api.SchedInfo) *Task { + return &Task{ + Time: time.Now(), + SchedInfo: schedInfo, + manager: manager, + taskExecutors: []*TaskExecutor{}, + lock: sync.Mutex{}, + waitCh: make(chan struct{}), + resultItems: []*core.SchedResultItem{}, + resultError: nil, + } +} + +func (t *Task) GetTaskExecutor(tag string) *TaskExecutor { + for _, executor := range t.taskExecutors { + if executor.Tag == tag { + return executor + } + } + + return nil +} + +func (t *Task) GetSessionID() string { + return t.SchedInfo.SessionID +} + +func (t *Task) GetStatus() string { + statusMap := make(map[string]int) + for _, executor := range t.taskExecutors { + if count, ok := statusMap[executor.Status]; ok { + statusMap[executor.Status] = count + 1 + } else { + statusMap[executor.Status] = 1 + } + } + + ss := []string{} + for status, count := range statusMap { + ss = append(ss, fmt.Sprintf("%v %v", count, status)) + } + + return strings.Join(ss, ", ") +} + +func (t *Task) onTaskCompleted(taskExecutor *TaskExecutor) { + t.lock.Lock() + defer t.lock.Unlock() + + log.V(10).Infof("onTaskCompleted executor: %#v", taskExecutor) + if taskExecutor.resultError != nil { + t.resultError = taskExecutor.resultError + t.onError() + } else { + t.resultItems = append(t.resultItems, taskExecutor.resultItems...) + t.completedCount += 1 + if t.completedCount >= len(t.taskExecutors) { + t.onCompleted() + } + } + + go func() { + t.readLog(taskExecutor) + taskExecutor.cleanup() + }() +} + +func (t *Task) readLog(taskExecutor *TaskExecutor) { + u := taskExecutor.unit + if u != nil { + logs := u.LogManager.Read() + taskExecutor.logs = logs + } +} + +func (t *Task) onError() { + for _, taskExecutor := range t.taskExecutors { + taskExecutor.Kill() + } + + log.Errorf("Remove Session on error: %v\n", t.SchedInfo.SessionID) + //t.manager.ReservedPoolManager.RemoveSession(t.SchedInfo.SessionID) + + close(t.waitCh) +} + +func (t *Task) onCompleted() { + t.Consuming = time.Since(t.Time) + close(t.waitCh) +} + +func (t *Task) Wait() ([]*core.SchedResultItem, error) { + log.V(10).Infof("Task wait...") + <-t.waitCh + return t.GetResult() +} + +func (t *Task) GetResult() ([]*core.SchedResultItem, error) { + return t.resultItems, t.resultError +} diff --git a/pkg/scheduler/util/baremetal/diskconfig.go b/pkg/scheduler/util/baremetal/diskconfig.go new file mode 100644 index 0000000000..1b9305b517 --- /dev/null +++ b/pkg/scheduler/util/baremetal/diskconfig.go @@ -0,0 +1,660 @@ +package baremetal + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" + "github.com/yunionio/pkg/util/sets" + "github.com/yunionio/pkg/utils" +) + +const ( + DISK_CONF_RAID0 = api.DISK_CONF_RAID0 + DISK_CONF_RAID1 = api.DISK_CONF_RAID1 + DISK_CONF_RAID5 = api.DISK_CONF_RAID5 + DISK_CONF_RAID10 = api.DISK_CONF_RAID10 + DISK_CONF_NONE = api.DISK_CONF_NONE + + DEFAULT_DISK_CONF = DISK_CONF_NONE + + DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE + DISK_TYPE_SSD = api.DISK_TYPE_SSD + DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID + + DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + + DISK_DRIVER_MEGARAID = api.DISK_DRIVER_MEGARAID + DISK_DRIVER_LINUX = api.DISK_DRIVER_LINUX + DISK_DRIVER_HPSARAID = api.DISK_DRIVER_HPSARAID + DISK_DRIVER_MPT2SAS = api.DISK_DRIVER_MPT2SAS + DISK_DRIVER_MARVELRAID = api.DISK_DRIVER_MARVELRAID + DISK_DRIVER_PCIE = api.DISK_DRIVER_PCIE + + HDD_DISK_SPEC_TYPE = api.HDD_DISK_SPEC_TYPE + SSD_DISK_SPEC_TYPE = api.SSD_DISK_SPEC_TYPE + + ErrMoreThanOneSizeUnspecificSplit = `more than 1 size unspecific split` + ErrNoMoreSpaceForUnspecificSplit = `no more space for an unspecific split` + ErrSubtotalOfSplitExceedsDiskSize = `subtotal of split exceeds disk size` +) + +var ( + DISK_CONFS = sets.NewString( + DISK_CONF_RAID0, + DISK_CONF_RAID1, + DISK_CONF_RAID5, + DISK_CONF_RAID10, + DISK_CONF_NONE, + ) + + DISK_TYPES = sets.NewString( + DISK_TYPE_ROTATE, + DISK_TYPE_SSD, + DISK_TYPE_HYBRID, + ) + + DISK_DRIVERS_RAID = sets.NewString( + DISK_DRIVER_MEGARAID, + DISK_DRIVER_HPSARAID, + DISK_DRIVER_MPT2SAS, + DISK_DRIVER_MARVELRAID, + ) + + DISK_DRIVERS = sets.NewString( + DISK_DRIVER_LINUX, + DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) +) + +// return bytes +func parseStrip(stripStr string, defaultSize string) int64 { + size, _ := utils.GetSize(stripStr, defaultSize, 1024) + return size / 1024 +} + +func parseRangeStr(str string) (ret []int64, err error) { + im := utils.IsMatchInteger + errGen := func(e string) error { + return fmt.Errorf("Incorrect range str: %q", e) + } + rs := strings.Split(str, "-") + if len(rs) != 2 { + err = errGen(str) + return + } + + bs, es := rs[0], rs[1] + if !im(bs) { + err = errGen(str) + return + } + if !im(es) { + err = errGen(str) + return + } + + begin, _ := strconv.ParseInt(bs, 10, 64) + end, _ := strconv.ParseInt(es, 10, 64) + + if begin > end { + begin, end = end, begin + } + + for i := begin; i <= end; i++ { + ret = append(ret, i) + } + return +} + +// range string should be: "1-3", "3" +func _parseRange(str string) (ret []int64, err error) { + if len(str) == 0 { + return + } + + // exclude "," symbol + if len(str) == 1 && !utils.IsMatchInteger(str) { + return + } + + // add int string + if utils.IsMatchInteger(str) { + i, _ := strconv.ParseInt(str, 10, 64) + ret = append(ret, i) + return + } + + // add rang like string, "2-10" etc. + ret, err = parseRangeStr(str) + return +} + +func ParseRange(rangeStr string) (ret []int64, err error) { + rss := regexp.MustCompile(`[\s,]+`).Split(rangeStr, -1) + intSet := sets.NewInt64() + + for _, rs := range rss { + r, err1 := _parseRange(rs) + if err1 != nil { + err = err1 + return + } + intSet.Insert(r...) + } + ret = intSet.List() + return +} + +func ParseDiskConfig(desc string) (bdc api.BaremetalDiskConfig, err error) { + bdc.Type = DISK_TYPE_HYBRID + bdc.Conf = DISK_CONF_NONE + bdc.Count = 0 + + desc = strings.ToLower(desc) + if desc == "" { + return + } + + parts := strings.Split(desc, ":") + drvMap := make(map[string]string) + for _, drv := range DISK_DRIVERS.List() { + drvMap[strings.ToLower(drv)] = drv + } + for _, p := range parts { + if len(p) == 0 { + continue + } else if DISK_TYPES.Has(p) { + bdc.Type = p + } else if DISK_CONFS.Has(p) { + bdc.Conf = p + } else if drv, ok := drvMap[p]; ok { + bdc.Driver = drv + } else if utils.IsMatchInteger(p) { + bdc.Count, _ = strconv.ParseInt(p, 0, 0) + } else if len(p) > 2 && p[0] == '[' && p[len(p)-1] == ']' { + rg, err1 := ParseRange(p[1:(len(p) - 1)]) + if err1 != nil { + err = err1 + return + } + bdc.Range = rg + } else if len(p) > 2 && p[0] == '(' && p[len(p)-1] == ')' { + bdc.Splits = p[1 : len(p)-1] + } else if utils.HasPrefix(p, "strip") { + bdc.Strip = parseStrip(p[len("strip"):], "k") + } else if utils.HasPrefix(p, "adapter") { + ada, _ := strconv.ParseInt(p[len("adapter"):], 0, 64) + pada := int(ada) + bdc.Adapter = &pada + } else if p == "ra" { + bdc.RA = true + } else if p == "nora" { + bdc.RA = false + } else if p == "wt" { + bdc.WT = true + } else if p == "wb" { + bdc.WT = false + } else if p == "direct" { + bdc.Direct = true + } else if p == "cached" { + bdc.Direct = false + } else if p == "cachedbadbbu" { + bdc.Cachedbadbbu = true + } else if p == "nocachedbadbbu" { + bdc.Cachedbadbbu = false + } else { + err = fmt.Errorf("ParseDiskConfig unkown option %q", p) + return + } + } + + return +} + +func isDiskConfigStorageMatch( + config *api.BaremetalDiskConfig, + confDriver *string, + confAdapter *int, + storage *api.BaremetalStorage, + selected []*api.BaremetalStorage, +) bool { + isRotate := storage.Rotate + adapter := storage.Adapter + index := storage.Index + driver := storage.Driver + + typeIsHybrid := config.Type == DISK_TYPE_HYBRID + typeIsRotate := config.Type == DISK_TYPE_ROTATE && isRotate + typeIsSSD := config.Type == DISK_TYPE_SSD && !isRotate + rangeIsNoneAndCountZero := len(config.Range) == 0 && config.Count == 0 + rangeIsNotNoneAndIndexInRange := len(config.Range) != 0 && sets.NewInt64(config.Range...).Has(index) + rangeIsNoneAndSmallThanCount := len(config.Range) == 0 && int64(len(selected)) < config.Count + adapterIsEqual := (confAdapter == nil || *confAdapter == adapter) && + (confDriver == nil || *confDriver == driver) + + log.V(10).Debugf("typeIsHybrid: %v, typeIsRotate: %v, typeIsSSD: %v, rangeIsNoneAndCountZero: %v, rangeIsNotNoneAndIndexInRange: %v, rangeIsNoneAndSmallThanCount: %v, adapterIsEqual: %v", typeIsHybrid, typeIsRotate, typeIsSSD, rangeIsNoneAndCountZero, rangeIsNotNoneAndIndexInRange, rangeIsNoneAndSmallThanCount, adapterIsEqual) + + if (typeIsHybrid || typeIsRotate || typeIsSSD) && + (rangeIsNoneAndCountZero || rangeIsNotNoneAndIndexInRange || rangeIsNoneAndSmallThanCount) && + adapterIsEqual { + if confDriver == nil { + confDriver = &driver + } + if confAdapter == nil { + confAdapter = &adapter + } + return true + } + return false +} + +func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*api.BaremetalStorage) (selected, rest []*api.BaremetalStorage) { + var confDriver *string = nil + var confAdapter *int = nil + + if diskConfig.Adapter != nil { + confAdapter = diskConfig.Adapter + } + if diskConfig.Driver != "" { + confDriver = &diskConfig.Driver + } + + selected = make([]*api.BaremetalStorage, 0) + rest = make([]*api.BaremetalStorage, 0) + idx := 0 + + for _, storage := range storages { + if storage.Index == 0 { + storage.Index = int64(idx) + } + + if isDiskConfigStorageMatch(diskConfig, confDriver, confAdapter, storage, selected) { + selected = append(selected, storage) + } else { + rest = append(rest, storage) + } + idx++ + } + return +} + +func GetMinDiskRequirement(diskConfig string) int { + minDisk := 1 + if diskConfig == DISK_CONF_RAID1 { + minDisk = 2 + } + if diskConfig == DISK_CONF_RAID5 { + minDisk = 3 + } else if diskConfig == DISK_CONF_RAID10 { + minDisk = 4 + } + return minDisk +} + +func RequireEvenDisks(diskConfig string) bool { + if sets.NewString( + DISK_CONF_RAID10, + DISK_CONF_RAID1, + ).Has(diskConfig) { + return true + + } + return false +} + +type Layout struct { + Disks []*api.BaremetalStorage `json:"disks"` + Conf *api.BaremetalDiskConfig `json:"conf"` + Size int64 `json:"size"` +} + +func (l Layout) String() string { + bytes, _ := json.MarshalIndent(l, "", " ") + return string(bytes) +} + +func RetrieveStorageDrivers(storages []*api.BaremetalStorage) sets.String { + ret := sets.NewString() + for _, s := range storages { + ret = ret.Union(sets.NewString(s.Driver)) + } + return ret +} + +func MeetConfig( + conf *api.BaremetalDiskConfig, + storages []*api.BaremetalStorage, +) error { + storageDrvs := RetrieveStorageDrivers(storages) + if len(storageDrvs.List()) > 1 { + return fmt.Errorf("%v more than 1 storages drivers", storageDrvs) + } + driver := storageDrvs.List()[0] + if conf.Conf != DISK_CONF_NONE && !DISK_DRIVERS_RAID.Has(driver) { + return fmt.Errorf("BaremetalStorage driver %s not support RAID", driver) + } + + minDisk := GetMinDiskRequirement(conf.Conf) + if len(storages) < minDisk { + return fmt.Errorf("%q requires at least %d disks", conf.Conf, minDisk) + } + + if RequireEvenDisks(conf.Conf) && (len(storages)%2) != 0 { + return fmt.Errorf("%q requires event number of disks", conf.Conf) + } + + if len(conf.Splits) > 0 && + sets.NewString( + DISK_CONF_NONE, + DISK_DRIVER_MPT2SAS).Has(conf.Conf) { + return fmt.Errorf("Cannot divide a normal disk into splits") + } + + if driver == DISK_DRIVER_MPT2SAS { + if conf.Conf == DISK_CONF_RAID5 { + return fmt.Errorf("%q not support RAID5", DISK_DRIVER_MPT2SAS) + } + if conf.Conf == DISK_CONF_RAID0 && len(storages) < 2 { + return fmt.Errorf("%q %q requires at least 2 disks", DISK_DRIVER_MPT2SAS, DISK_CONF_RAID0) + } + if conf.Conf == DISK_CONF_RAID10 && len(storages) > 10 { + return fmt.Errorf("%q %q only support no more than 10 disks", DISK_DRIVER_MPT2SAS, DISK_CONF_RAID10) + } + } + + if driver == DISK_DRIVER_MEGARAID && conf.Strip != 0 { + minStripSize := storages[0].MinStripSize + maxStripSize := storages[0].MaxStripSize + if maxStripSize != 0 && minStripSize != 0 { + size := conf.Strip + if size > maxStripSize || size < minStripSize { + return fmt.Errorf("%q input strip size out of range(%d, %d)", DISK_DRIVER_MEGARAID, minStripSize, maxStripSize) + } + } + } + + return nil +} + +func GetStoragesMinSize(ss []*api.BaremetalStorage) int64 { + minSize := int64(-1) + for _, s := range ss { + if minSize < 0 || minSize > s.Size { + minSize = s.Size + } + } + return minSize +} + +func CalculateSize(conf string, storages []*api.BaremetalStorage) int64 { + if conf == "" { + conf = DEFAULT_DISK_CONF + } + var size int64 + + if conf == DISK_CONF_RAID5 { + size = GetStoragesMinSize(storages) * int64(len(storages)-1) + } else if sets.NewString(DISK_CONF_RAID10, DISK_CONF_RAID1).Has(conf) { + size = GetStoragesMinSize(storages) * int64((len(storages) / 2)) + } else { + for _, s := range storages { + size += s.Size + } + } + return size +} + +func GetSplitSizes(size int64, splitConf string) []int64 { + ssizes := strings.Split(splitConf, ",") + isizes := make([]int64, len(ssizes)) + leftoverIdx := -1 + subtotal := int64(0) + for index := range ssizes { + if strings.HasSuffix(ssizes[index], "%") { + ssizeFloat64, err := strconv.ParseFloat(ssizes[index][:len(ssizes[index])-1], 64) + if err != nil { + log.Errorf("GetSplitSizes ParseFloat err: %v", err) + continue + } + isizes[index] = int64(ssizeFloat64 / float64(100) * float64(size)) + subtotal += isizes[index] + } else if ssizes[index] != "" { + isizes[index], _ = utils.GetSizeMB(ssizes[index], "M") + subtotal += isizes[index] + } else { + if leftoverIdx >= 0 { + log.Errorln(ErrMoreThanOneSizeUnspecificSplit) + return []int64{} + } + leftoverIdx = index + } + } + if leftoverIdx >= 0 { + isizes[leftoverIdx] = size - subtotal + if isizes[leftoverIdx] <= 0 { + log.Errorln(ErrNoMoreSpaceForUnspecificSplit) + return []int64{} + } + } else { + if subtotal > size { + log.Errorln(ErrSubtotalOfSplitExceedsDiskSize) + return []int64{} + } + } + return isizes +} + +func ExpandNoneConf(layouts []Layout) (ret []Layout) { + for _, layout := range layouts { + if layout.Conf.Conf == DISK_CONF_NONE && len(layout.Disks) >= 1 { + conf := layout.Conf + conf.Count = 1 + for _, disk := range layout.Disks { + ret = append(ret, Layout{Disks: []*api.BaremetalStorage{disk}, Conf: conf, Size: disk.Size}) + } + } else { + ret = append(ret, layout) + } + } + return ret +} + +func CalculateLayout( + confs []*api.BaremetalDiskConfig, + storages []*api.BaremetalStorage, +) (layouts []Layout, err error) { + + if len(confs) == 0 { + err = fmt.Errorf("[]*api.BaremetalDiskConfig must be provided.") + return + } + + layouts = make([]Layout, 0) + var ( + plainDisks = make([]*api.BaremetalStorage, 0) + pcieDisks = make([]*api.BaremetalStorage, 0) + raidDisks = make([]*api.BaremetalStorage, 0) + ) + + for _, storage := range storages { + if storage.Driver != "" && storage.Driver == DISK_DRIVER_LINUX { + plainDisks = append(plainDisks, storage) + } else if storage.Driver != "" && storage.Driver == DISK_DRIVER_PCIE { + pcieDisks = append(pcieDisks, storage) + } else { + raidDisks = append(raidDisks, storage) + } + } + + if len(plainDisks) > 0 { + layouts = append(layouts, Layout{ + Disks: plainDisks, + Conf: &api.BaremetalDiskConfig{Conf: DISK_CONF_NONE}, + Size: 0, + }) + } + + storages = raidDisks + //totalStorageRequest := 0 + //for _, conf := range confs { + //totalStorageRequest += conf.Count + //} + //if totalStorageRequest > len(storages) { + //err = fmt.Errorf("requested number of disks is more than storages") + //return layouts, err + //} + + for _, conf := range confs { + if len(storages) == 0 { + continue + } + selected, storages1 := RetrieveStorages(conf, storages) + //selected, _ := RetrieveStorages(conf, storages) + storages = storages1 + if len(selected) == 0 { + err = fmt.Errorf("Not found matched storages by config: %#v", conf) + return + } + resultErr := MeetConfig(conf, selected) + if resultErr != nil { + err = fmt.Errorf("selected storages %#v not meet baremetal dick config: %#v, err: %v", selected, conf, resultErr) + return + } + sz := CalculateSize(conf.Conf, selected) + + if len(conf.Splits) == 0 { + layouts = append(layouts, Layout{ + Disks: selected, + Conf: conf, + Size: sz, + }) + } else { + layouts = append(layouts, Layout{ + Disks: selected, + Conf: conf, + Size: sz, + }) + /* splitSizes := GetSplitSizes(sz, conf.Splits) + if len(splitSizes) <= 0 { + break + } + + for _, ssz := range splitSizes { + subConfig := conf + layouts = append(layouts, Layout{ + Disks: selected, + Conf: subConfig, + Size: ssz, + }) + }*/ + } + } + + if len(pcieDisks) > 0 { + layouts = append(layouts, Layout{ + Disks: pcieDisks, + Conf: &api.BaremetalDiskConfig{Conf: DISK_CONF_NONE}, + Size: 0, + }) + } + layouts = ExpandNoneConf(layouts) + return +} + +func expandLayoutSplits(layouts []Layout) []Layout { + ret := make([]Layout, 0) + for _, l := range layouts { + splitSizes := GetSplitSizes(l.Size, l.Conf.Splits) + if len(splitSizes) <= 0 { + ret = append(ret, l) + } else { + for _, ssz := range splitSizes { + subLayout := l + subLayout.Size = ssz + ret = append(ret, subLayout) + } + } + } + return ret +} + +func CheckDisksAllocable(layouts []Layout, disks []*api.Disk) bool { + layouts = expandLayoutSplits(layouts) + storeIndex := 0 + storeFreeSize := int64(-1) + diskIndex := 0 + layoutLen := len(layouts) + for _, disk := range disks { + if storeIndex >= layoutLen { + break + } + if storeFreeSize < 0 { + storeFreeSize = layouts[storeIndex].Size - 2 // start, end space + } + if disk.Size > 0 { + if storeFreeSize >= disk.Size { + storeFreeSize -= disk.Size + diskIndex++ + if storeFreeSize == 0 { + storeIndex++ + storeFreeSize = -1 + } + } else { + storeIndex++ + storeFreeSize = -1 + } + } else { + diskIndex++ + storeIndex++ + storeFreeSize = -1 + } + } + if diskIndex < len(disks) { + return false + } + return true +} + +func NewBaremetalDiskConfigs(dss ...string) ([]*api.BaremetalDiskConfig, error) { + ret := make([]*api.BaremetalDiskConfig, 0) + for _, ds := range dss { + r, err := ParseDiskConfig(ds) + if err != nil { + return nil, err + } + ret = append(ret, &r) + } + return ret, nil +} + +type SpecSizeCount map[string]int +type DiskSpec map[string]SpecSizeCount + +func GetDiskSpec(storages []*api.BaremetalStorage) DiskSpec { + diskSpec := make(map[string]SpecSizeCount) + + for _, s := range storages { + var dtype string + if s.Rotate { + dtype = HDD_DISK_SPEC_TYPE + } else { + dtype = SSD_DISK_SPEC_TYPE + } + + sizeStr := fmt.Sprintf("%d", s.Size) + sc, ok := diskSpec[dtype] + if !ok { + sc = make(map[string]int) + diskSpec[dtype] = sc + } + if _, ok := sc[sizeStr]; !ok { + sc[sizeStr] = 0 + } + diskSpec[dtype][sizeStr] += 1 + } + return diskSpec +} diff --git a/pkg/scheduler/util/baremetal/diskconfig_test.go b/pkg/scheduler/util/baremetal/diskconfig_test.go new file mode 100644 index 0000000000..123dee5e58 --- /dev/null +++ b/pkg/scheduler/util/baremetal/diskconfig_test.go @@ -0,0 +1,1086 @@ +package baremetal + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/yunionio/log" + "github.com/yunionio/onecloud/pkg/scheduler/api" +) + +func TestParseDiskConfig(t *testing.T) { + type args struct { + desc string + } + + var tAda int = 2 + var zAda int = 0 + var tStrip64k int64 = 64 + var splits40 string = "40%, " + var splits100_32 string = "100g,32g," + + tests := []struct { + name string + args args + wantBdc api.BaremetalDiskConfig + wantErr bool + }{ + { + name: "rotate:[1-2,4-5]:MegaRaid", + args: args{"rotate:[1-2,4-5]:MegaRaid"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_ROTATE, + Conf: DISK_CONF_NONE, + Driver: DISK_DRIVER_MEGARAID, + Count: 0, + Range: []int64{1, 2, 4, 5}, + }, + wantErr: false, + }, + { + name: "rotate:[1-2,4,6]:raid10:marvelraid", + args: args{"rotate:[1-2,4,6]:raid10:marvelraid"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_ROTATE, + Conf: DISK_CONF_RAID10, + Driver: DISK_DRIVER_MARVELRAID, + Count: 0, + Range: []int64{1, 2, 4, 6}, + }, + wantErr: false, + }, + { + name: "rotate:[4,6]:raid10", + args: args{"rotate:[4,6]:raid10"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_ROTATE, + Conf: DISK_CONF_RAID10, + Count: 0, + Range: []int64{4, 6}, + }, + wantErr: false, + }, + { + name: "rotate:[4]:raid10:(40%, )", + args: args{"rotate:[4]:raid10:(40%, )"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_ROTATE, + Conf: DISK_CONF_RAID10, + Count: 0, + Splits: splits40, + Range: []int64{4}, + }, + wantErr: false, + }, + { + name: "[12-13]:raid1:(100g,32g,):adapter0:strip64k", + args: args{"[12-13]:raid1:(100g,32g,):adapter0:strip64k"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_RAID1, + Count: 0, + Splits: splits100_32, + Adapter: &zAda, + Strip: tStrip64k, + Range: []int64{12, 13}, + }, + wantErr: false, + }, + { + name: "6:raid5:adapter2", + args: args{"6:raid5:adapter2"}, + wantBdc: api.BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_RAID5, + Count: 6, + Adapter: &tAda, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotBdc, err := ParseDiskConfig(tt.args.desc) + if (err != nil) != tt.wantErr { + t.Errorf("ParseDiskConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotBdc, tt.wantBdc) { + t.Errorf("ParseDiskConfig() = %v, want %v", gotBdc, tt.wantBdc) + } + }) + } +} + +func TestParseRange(t *testing.T) { + type args struct { + rangeStr string + } + tests := []struct { + name string + args args + wantRet []int64 + wantErr bool + }{ + { + name: "range number and , test", + args: args{"11-7,, 9-9, 10, 15, 1-2,4-5,,"}, + wantRet: []int64{1, 2, 4, 5, 7, 8, 9, 10, 11, 15}, + wantErr: false, + }, + { + name: "range,range", + args: args{"1-2,4-5"}, + wantRet: []int64{1, 2, 4, 5}, + wantErr: false, + }, + { + name: "numbers", + args: args{"1, 1"}, + wantRet: []int64{1}, + wantErr: false, + }, + { + name: "numbersErr", + args: args{"1-, 1"}, + wantRet: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRet, err := ParseRange(tt.args.rangeStr) + if (err != nil) != tt.wantErr { + t.Errorf("ParseRange() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRet, tt.wantRet) { + t.Errorf("ParseRange() = %v, want %v", gotRet, tt.wantRet) + } + }) + } +} + +var ( + testStorages = []*api.BaremetalStorage{ + // 0-11 disk on adapter2 + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 2861056, + Adapter: 2, + }, + + // 12-13 disk on adapter0 + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 953344, + Adapter: 0, + }, + { + Driver: DISK_DRIVER_MEGARAID, + Rotate: true, + Size: 953344, + Adapter: 0, + }, + } + bitmainStorage = []*api.BaremetalStorage{ + { + Driver: DISK_DRIVER_MARVELRAID, + Rotate: true, + Size: 3814912, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "K7GTE9JL HGST HUS726040ALE610 APGNT907", + Slot: 0, + Status: "online", + }, + { + Driver: DISK_DRIVER_MARVELRAID, + Rotate: true, + Size: 3814912, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "K7GTE9JL HGST HUS726040ALE610 APGNT907", + Slot: 1, + Status: "online", + }, + { + Driver: DISK_DRIVER_MARVELRAID, + Rotate: true, + Size: 3814912, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "K7GTE9JL HGST HUS726040ALE610 APGNT907", + Slot: 2, + Status: "online", + }, + { + Driver: DISK_DRIVER_MARVELRAID, + Rotate: true, + Size: 3814912, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "K7GTE9JL HGST HUS726040ALE610 APGNT907", + Slot: 3, + Status: "online", + }, + { + Driver: DISK_DRIVER_PCIE, + Rotate: true, + Size: 2200000, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "NVME test", + Slot: 4, + Status: "online", + }, + { + Driver: DISK_DRIVER_PCIE, + Rotate: true, + Size: 2200000, + Adapter: 0, + Enclosure: 252, + MaxStripSize: 1024, + MinStripSize: 64, + Model: "NVME test", + Slot: 5, + Status: "online", + }, + } +) + +func TestCalculateLayout(t *testing.T) { + confs, err := NewBaremetalDiskConfigs( + "[12-13]:raid1:(100g,32g,):adapter0", + "6:raid5:adapter2", + "6:raid5:adapter2", + ) + if err != nil { + t.Fatalf("NewDiskConfigs err: %v", err) + } + + expectedLayoutJson := ` +[ + { + "disks": [ + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 0, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 953344, + "index": 12 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 0, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 953344, + "index": 13 + } + ], + "conf": { + "type": "hybrid", + "conf": "raid1", + "count": 0, + "range": [ + 12, + 13 + ], + "splits": "100g,32g,", + "adapter": 0, + "cachedbadbbu": false, + "strip": 0, + "ra": false, + "wt": false, + "direct": false + }, + "size": 953344 + }, + { + "disks": [ + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 0 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 1 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 2 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 3 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 4 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 5 + } + ], + "conf": { + "type": "hybrid", + "conf": "raid5", + "count": 6, + "range": null, + "splits": "", + "adapter": 2, + "cachedbadbbu": false, + "strip": 0, + "ra": false, + "wt": false, + "direct": false + }, + "size": 14305280 + }, + { + "disks": [ + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 6 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 7 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 8 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 9 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 10 + }, + { + "slot": 0, + "status": "", + "rotate": true, + "adapter": 2, + "driver": "MegaRaid", + "model": "", + "enclousure": 0, + "size": 2861056, + "index": 11 + } + ], + "conf": { + "type": "hybrid", + "conf": "raid5", + "count": 6, + "range": null, + "splits": "", + "adapter": 2, + "cachedbadbbu": false, + "strip": 0, + "ra": false, + "wt": false, + "direct": false + }, + "size": 14305280 + } +] +` + layout, err := CalculateLayout(confs, testStorages) + if err != nil { + t.Fatalf("CalculateLayout err: %v", err) + } + + var expectedLayout []Layout + err = json.Unmarshal([]byte(expectedLayoutJson), &expectedLayout) + if err != nil { + t.Fatalf("Unmarshal expectedLayoutJson err: %v", err) + } + if !reflect.DeepEqual(layout, expectedLayout) { + t.Errorf("CalculateLayout() = %v, want %v", layout, expectedLayout) + } +} + +func TestCheckDisksAllocable(t *testing.T) { + confs, err := NewBaremetalDiskConfigs( + "[12-13]:raid1:(100g,32g,):adapter0", + "6:raid5:adapter2", + "6:raid5:adapter2", + ) + bitmainConfs, err := NewBaremetalDiskConfigs("raid10:(60g,)") + if err != nil { + t.Fatalf("NewDiskConfigs err: %v", err) + } + + layout, err := CalculateLayout(confs, testStorages) + defaultLayout, err := CalculateLayout([]*api.BaremetalDiskConfig{&api.BaremetalDefaultDiskConfig}, testStorages[12:]) + if err != nil { + t.Fatalf("CalculateLayout err: %v", err) + } + bitmainLayout, err := CalculateLayout(bitmainConfs, bitmainStorage) + if err != nil { + t.Fatalf("Calculate bitmain layout err: %v", err) + } + + log.Debugf("defaultLayout: %s", defaultLayout) + log.Debugf("layout: %s", layout) + log.Debugf("Bitmain layout: %s", bitmainLayout) + + tdiskDefault := []*api.Disk{ + {Size: -1}, + {Size: -1}, + } + + tdisk1 := []*api.Disk{ + {Size: 960000}, + {Size: -1}, + {Size: -1}, + {Size: -1}, + } + + tdisk2 := []*api.Disk{ + {Size: -1}, + {Size: -1}, + {Size: -1}, + {Size: -1}, + } + + tdisk3 := []*api.Disk{ + {Size: 102398}, + {Size: -1}, + {Size: -1}, + {Size: -1}, + } + + btdisk1 := []*api.Disk{ + {Size: 61438}, + {Size: -1}, + } + + btdisk2 := []*api.Disk{ + {Size: 61440}, + {Size: -1}, + } + + btdisk3 := []*api.Disk{ + {Size: -1}, + {Size: -1}, + } + + btPcieDisk := []*api.Disk{ + {Size: 44440}, + {Size: -1}, + {Size: -1}, + {Size: -1}, + {Size: -1}, + } + + type args struct { + layouts []Layout + disks []*api.Disk + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "default none type config should allocable", + args: args{ + layouts: defaultLayout, + disks: tdiskDefault, + }, + want: true, + }, + { + name: "should not allocable", + args: args{ + layouts: layout, + disks: tdisk1, + }, + want: false, + }, + { + name: "should allocable", + args: args{ + layouts: layout, + disks: tdisk2, + }, + want: true, + }, + { + name: "should allocable2", + args: args{ + layouts: layout, + disks: tdisk3, + }, + want: true, + }, + { + name: "Bitmain allocable 61438 should true", + args: args{ + layouts: bitmainLayout, + disks: btdisk1, + }, + want: true, + }, + { + name: "Bitmain allocable 61440 should false", + args: args{ + layouts: bitmainLayout, + disks: btdisk2, + }, + want: false, + }, + { + name: "Bitmain allocable autoextend should true", + args: args{ + layouts: bitmainLayout, + disks: btdisk3, + }, + want: true, + }, + { + name: "Bitmain allocable PCIE disk should true", + args: args{ + layouts: bitmainLayout, + disks: btPcieDisk, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckDisksAllocable(tt.args.layouts, tt.args.disks); got != tt.want { + t.Errorf("CheckDisksAllocable() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetDiskSpec(t *testing.T) { + type args struct { + storages []*api.BaremetalStorage + } + tests := []struct { + name string + args args + want DiskSpec + }{ + { + name: "Only HDD", + args: args{testStorages}, + want: map[string]SpecSizeCount{ + "HDD": map[string]int{ + "2861056": 12, + "953344": 2, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetDiskSpec(tt.args.storages); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetDiskSpec() = %v, want %v", got, tt.want) + } else { + log.Debugf("GetDiskSpec() = %v", got) + } + }) + } +} + +var testStorages2 string = ` +[ + { + "adapter": 0, + "driver": "MarvelRaid", + "model": "SSDSCKJB120G7R", + "rotate": true, + "size": 114473, + "slot": 0, + "sn": "PHDW80440155150A" + }, + { + "adapter": 0, + "driver": "MarvelRaid", + "model": "SSDSCKJB120G7R", + "rotate": true, + "size": 114473, + "slot": 1, + "sn": "PHDW80440180150A" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "BTYM73040578960CGNSSDSC2KG960G7R SCV1DL56", + "rotate": false, + "size": 915200, + "slot": 0, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "BTYM73040566960CGNSSDSC2KG960G7R SCV1DL56", + "rotate": false, + "size": 915200, + "slot": 1, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "BTYM734403W7960CGNSSDSC2KG960G7R SCV1DL56", + "rotate": false, + "size": 915200, + "slot": 2, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "BTYM734600B1960CGNSSDSC2KG960G7R SCV1DL56", + "rotate": false, + "size": 915200, + "slot": 3, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A192FL7E", + "rotate": true, + "size": 1716352, + "slot": 4, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A1A4FL7E", + "rotate": true, + "size": 1716352, + "slot": 5, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A190FL7E", + "rotate": true, + "size": 1716352, + "slot": 6, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A15WFL7E", + "rotate": true, + "size": 1716352, + "slot": 7, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A18CFL7E", + "rotate": true, + "size": 1716352, + "slot": 8, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A1ABFL7E", + "rotate": true, + "size": 1716352, + "slot": 9, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A1ACFL7E", + "rotate": true, + "size": 1716352, + "slot": 10, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012840A23VFL7E", + "rotate": true, + "size": 1716352, + "slot": 11, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012840A21SFL7E", + "rotate": true, + "size": 1716352, + "slot": 12, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A191FL7E", + "rotate": true, + "size": 1716352, + "slot": 13, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A15UFL7E", + "rotate": true, + "size": 1716352, + "slot": 14, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012840A23TFL7E", + "rotate": true, + "size": 1716352, + "slot": 15, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A13GFL7E", + "rotate": true, + "size": 1716352, + "slot": 16, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A13JFL7E", + "rotate": true, + "size": 1716352, + "slot": 17, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A19EFL7E", + "rotate": true, + "size": 1716352, + "slot": 18, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A19UFL7E", + "rotate": true, + "size": 1716352, + "slot": 19, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A1BSFL7E", + "rotate": true, + "size": 1716352, + "slot": 20, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A1B5FL7E", + "rotate": true, + "size": 1716352, + "slot": 21, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A16FFL7E", + "rotate": true, + "size": 1716352, + "slot": 22, + "status": "offline" + }, + { + "adapter": 0, + "driver": "MegaRaid", + "enclosure": 64, + "max_strip_size": 1024, + "min_strip_size": 64, + "model": "TOSHIBA AL14SEB18EQY EB012850A165FL7E", + "rotate": true, + "size": 1716352, + "slot": 23, + "status": "offline" + } +] +` + +func TestStorageLoad(t *testing.T) { + ss := make([]*api.BaremetalStorage, 0) + confs, err := NewBaremetalDiskConfigs( + "2:raid1:MarvelRaid", + "4:raid10:MegaRaid", + "raid5", + ) + if err != nil { + t.Fatalf("NewDiskConfigs err: %v", err) + } + json.Unmarshal([]byte(testStorages2), &ss) + layout, err := CalculateLayout(confs, ss) + if err != nil { + t.Fatalf("CalculateLayout err: %v", err) + } + log.Debugf("layout: %s", layout) +} diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go new file mode 100644 index 0000000000..9a1593ba10 --- /dev/null +++ b/pkg/util/aliyun/aliyun.go @@ -0,0 +1,119 @@ +package aliyun + +import ( + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "github.com/yunionio/jsonutils" + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" +) + +const ( + CLOUD_PROVIDER_ALIYUN = "Aliyun" + CLOUD_PROVIDER_ALIYUN_CN = "阿里云" + + ALIYUN_DEFAULT_REGION = "cn-hangzhou" +) + +type SAliyunClient struct { + providerId string + accessKey string + secret string + regions []SRegion + iregions []cloudprovider.ICloudRegion +} + +func NewAliyunClient(providerId string, accessKey string, secret string) (*SAliyunClient, error) { + client := SAliyunClient{providerId: providerId, accessKey: accessKey, secret: secret} + err := client.fetchRegions() + if err != nil { + return nil, err + } + return &client, nil +} + +func jsonRequest(client *sdk.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + req := requests.NewCommonRequest() + req.Domain = "ecs.aliyuncs.com" + req.Version = "2014-05-26" + req.ApiName = apiName + if params != nil { + for k, v := range params { + req.QueryParams[k] = v + } + } + + resp, err := client.ProcessCommonRequest(req) + if err != nil { + log.Errorf("request error %s", err) + return nil, err + } + body, err := jsonutils.Parse(resp.GetHttpContentBytes()) + if err != nil { + log.Errorf("parse json fail %s", err) + return nil, err + } + return body, nil +} + +func (self *SAliyunClient) getDefaultClient() (*sdk.Client, error) { + return sdk.NewClientWithAccessKey(ALIYUN_DEFAULT_REGION, self.accessKey, self.secret) +} + +func (self *SAliyunClient) jsonRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient() + if err != nil { + return nil, err + } + return jsonRequest(cli, apiName, params) +} + +func (self *SAliyunClient) fetchRegions() error { + body, err := self.jsonRequest("DescribeRegions", nil) + if err != nil { + return err + } + + self.regions = make([]SRegion, 0) + err = body.Unmarshal(&self.regions, "Regions", "Region") + if err != nil { + log.Errorf("unmarshal json error %s", err) + return err + } + self.iregions = make([]cloudprovider.ICloudRegion, len(self.regions)) + for i := 0; i < len(self.regions); i += 1 { + self.regions[i].client = self + self.iregions[i] = &self.regions[i] + } + return nil +} + +func (self *SAliyunClient) GetRegions() []SRegion { + return self.regions +} + +func (self *SAliyunClient) GetIRegions() []cloudprovider.ICloudRegion { + return self.iregions +} + +func (self *SAliyunClient) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { + for i := 0; i < len(self.iregions); i += 1 { + if self.iregions[i].GetGlobalId() == id { + return self.iregions[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SAliyunClient) GetRegion(regionId string) *SRegion { + if len(regionId) == 0 { + regionId = ALIYUN_DEFAULT_REGION + } + for i := 0; i < len(self.regions); i += 1 { + if self.regions[i].RegionId == regionId { + return &self.regions[i] + } + } + return nil +} diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go new file mode 100644 index 0000000000..88f9b22905 --- /dev/null +++ b/pkg/util/aliyun/disk.go @@ -0,0 +1,162 @@ +package aliyun + +import ( + "fmt" + "time" + + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type SMountInstances struct { + MountInstance []string +} + +type STags struct { + Tag []string +} + +type SDisk struct { + storage *SStorage + + AttachedTime time.Time + AutoSnapshotPolicyId string + Category string + CreationTime time.Time + DeleteAutoSnapshot bool + DeleteWithInstance bool + Description string + DetachedTime time.Time + Device string + DiskChargeType string + DiskId string + DiskName string + EnableAutoSnapshot bool + EnableAutomatedSnapshotPolicy bool + Encrypted bool + ExpiredTime time.Time + ImageId string + InstanceId string + MountInstances SMountInstances + OperationLocks SOperationLocks + Portable bool + ProductCode string + RegionId string + ResourceGroupId string + Size int + SourceSnapshotId string + Status string + Tags STags + Type string + ZoneId string +} + +func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, offset int, limit int) ([]SDisk, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + + if len(instanceId) > 0 { + params["InstanceId"] = instanceId + } + if len(zoneId) > 0 { + params["ZoneId"] = zoneId + } + if len(category) > 0 { + params["Category"] = category + } + + body, err := self.ecsRequest("DescribeDisks", params) + if err != nil { + log.Errorf("GetDisks fail %s", err) + return nil, 0, err + } + + disks := make([]SDisk, 0) + err = body.Unmarshal(&disks, "Disks", "Disk") + if err != nil { + log.Errorf("Unmarshal disk details fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return disks, int(total), nil +} + +func (self *SDisk) GetId() string { + return self.DiskId +} + +func (self *SDisk) GetName() string { + return self.DiskId +} + +func (self *SDisk) GetGlobalId() string { + return self.DiskId +} + +func (self *SDisk) GetIStorge() cloudprovider.ICloudStorage { + return self.storage +} + +func (self *SDisk) GetStatus() string { + // In_use Available Attaching Detaching Creating ReIniting All + switch self.Status { + case "Creating", "ReIniting": + return models.DISK_ALLOCATING + default: + return models.DISK_READY + } +} + +func (self *SDisk) GetDiskFormat() string { + return "vhd" +} + +func (self *SDisk) GetDiskSizeMB() int { + return self.Size * 1024 +} + +func (self *SDisk) GetIsAutoDelete() bool { + return self.DeleteWithInstance +} + +func (self *SDisk) GetTemplateId() string { + return self.ImageId +} + +func (self *SDisk) GetDiskType() string { + switch self.Type { + case "system": + return models.DISK_TYPE_SYS + case "data": + return models.DISK_TYPE_DATA + default: + return models.DISK_TYPE_DATA + } +} + +func (self *SDisk) GetFsFormat() string { + return "" +} + +func (self *SDisk) GetIsNonPersistent() bool { + return false +} + +func (self *SDisk) GetDriver() string { + return "scsi" +} + +func (self *SDisk) GetCacheMode() string { + return "none" +} + +func (self *SDisk) GetMountpoint() string { + return "" +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go new file mode 100644 index 0000000000..dc77f15f6f --- /dev/null +++ b/pkg/util/aliyun/host.go @@ -0,0 +1,235 @@ +package aliyun + +import ( + "fmt" + "time" + + "github.com/yunionio/jsonutils" + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type SHost struct { + zone *SZone +} + +func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { + return self.zone.GetIWires() +} + +func (self *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + return self.zone.GetIStorages() +} + +func (self *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { + vms := make([]SInstance, 0) + for { + parts, total, err := self.zone.region.GetInstances(self.zone.ZoneId, nil, len(vms), 50) + if err != nil { + return nil, err + } + vms = append(vms, parts...) + if len(vms) >= total { + break + } + } + ivms := make([]cloudprovider.ICloudVM, len(vms)) + for i := 0; i < len(vms); i += 1 { + vms[i].host = self + ivms[i] = &vms[i] + } + return ivms, nil +} + +func (self *SHost) VMGlobalId2Id(gid string) string { + return gid +} + +func (self *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { + id := self.VMGlobalId2Id(gid) + parts, _, err := self.zone.region.GetInstances(self.zone.ZoneId, []string{id}, 0, 1) + if err != nil { + return nil, err + } + if len(parts) == 0 { + return nil, cloudprovider.ErrNotFound + } + if len(parts) > 1 { + return nil, cloudprovider.ErrDuplicateId + } + parts[0].host = self + return &parts[0], nil +} + +func (self *SHost) GetId() string { + return self.zone.GetId() +} + +func (self *SHost) GetName() string { + return self.GetId() +} + +func (self *SHost) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.GetId()) +} + +func (self *SHost) GetStatus() string { + return models.HOST_STATUS_RUNNING +} + +func (self *SHost) GetHostStatus() string { + return models.HOST_ONLINE +} + +func (self *SHost) GetEnabled() bool { + return true +} + +func (self *SHost) GetAccessIp() string { + return "" +} + +func (self *SHost) GetAccessMac() string { + return "" +} + +func (self *SHost) GetSysInfo() jsonutils.JSONObject { + info := jsonutils.NewDict() + info.Add(jsonutils.NewString(CLOUD_PROVIDER_ALIYUN), "manufacture") + return info +} + +func (self *SHost) GetSN() string { + return "" +} + +func (self *SHost) GetCpuCount() int8 { + return 0 +} + +func (self *SHost) GetNodeCount() int8 { + return 0 +} + +func (self *SHost) GetCpuDesc() string { + return "" +} + +func (self *SHost) GetCpuMhz() int { + return 0 +} + +func (self *SHost) GetMemSizeMB() int { + return 0 +} + +func (self *SHost) GetStorageSizeMB() int { + return 0 +} + +func (self *SHost) GetStorageType() string { + return models.DISK_TYPE_HYBRID +} + +func (self *SHost) GetHostType() string { + return models.HOST_TYPE_ALIYUN +} + +func (self *SHost) GetManagerId() string { + return self.zone.region.client.providerId +} + +func (self *SHost) getInstanceById(instanceId string) (*SInstance, error) { + inst, err := self.zone.region.GetInstance(instanceId) + if err != nil { + return nil, err + } + inst.host = self + return inst, nil +} + +func (self *SHost) CreateVM(name string, imgId string, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int) (cloudprovider.ICloudVM, error) { + vmId, err := self._createVM(name, imgId, cpu, memMB, vswitchId, ipAddr, desc, passwd, storageType, diskSizes) + if err != nil { + return nil, err + } + vm, err := self.getInstanceById(vmId) + if err != nil { + return nil, err + } + err = vm.waitStatus(InstanceStatusStopped, time.Second*10, time.Second*1800) + return vm, err +} + +func (self *SHost) _createVM(name string, imgId string, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int) (string, error) { + net := self.zone.getNetworkById(vswitchId) + if net == nil { + return "", fmt.Errorf("invalid switch ID %s", vswitchId) + } + if net.wire == nil { + log.Errorf("vsiwtch's wire is empty") + return "", fmt.Errorf("vsiwtch's wire is empty") + } + if net.wire.vpc == nil { + log.Errorf("vsiwtch's wire' vpc is empty") + return "", fmt.Errorf("vsiwtch's wire's vpc is empty") + } + + secgroups, err := net.wire.vpc.GetSecurityGroups() + if err != nil { + return "", fmt.Errorf("get security group error %s", err) + } + + if len(secgroups) == 0 { + return "", fmt.Errorf("no secgroup for vpc!!") + } + + secgroupId := secgroups[0].SecurityGroupId + + img, err := self.zone.region.GetImage(imgId) + if err != nil { + log.Errorf("getiamge fail %s", err) + return "", err + } + if img.Status != ImageStatusAvailable { + log.Errorf("image %s status %s", imgId, img.Status) + return "", fmt.Errorf("image not ready") + } + + _, err = self.zone.getStorageByCategory(storageType) + if err != nil { + return "", fmt.Errorf("Storage %s not avaiable: %s", storageType, err) + } + + disks := make([]SDisk, len(diskSizes)+1) + disks[0].Size = img.Size + disks[0].Category = storageType + + for i, sz := range diskSizes { + disks[i+1].Size = sz + disks[i+1].Category = storageType + } + + instanceTypes, err := self.zone.region.GetMatchInstanceTypes(cpu, memMB, 0, self.zone.ZoneId) + if err != nil { + return "", err + } + if len(instanceTypes) == 0 { + return "", fmt.Errorf("instance type %dC%dMB not avaiable", cpu, memMB) + } + + for _, instType := range instanceTypes { + instanceTypeId := instType.InstanceTypeId + log.Debugf("Try instancetype : %s", instanceTypeId) + vmId, err := self.zone.region.CreateInstance(name, imgId, instanceTypeId, secgroupId, self.zone.ZoneId, desc, passwd, disks, vswitchId, ipAddr, "") + if err != nil { + log.Errorf("Failed for %s: %s", instanceTypeId, err) + } else { + return vmId, nil + } + } + + return "", fmt.Errorf("Failed to create, specification not supported") +} diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go new file mode 100644 index 0000000000..f13e7fa378 --- /dev/null +++ b/pkg/util/aliyun/image.go @@ -0,0 +1,210 @@ +package aliyun + +import ( + "fmt" + "strings" + "time" + + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type ImageStatusType string + +const ( + ImageStatusCreating ImageStatusType = "Creating" + ImageStatusAvailable ImageStatusType = "Available" + ImageStatusUnAvailable ImageStatusType = "UnAvailable" + ImageStatusCreateFailed ImageStatusType = "CreateFailed" +) + +type ImageOwnerType string + +const ( + ImageOwnerSystem ImageOwnerType = "system" + ImageOwnerSelf ImageOwnerType = "self" + ImageOwnerOthers ImageOwnerType = "others" + ImageOwnerMarketplace ImageOwnerType = "marketplace" +) + +type ImageUsageType string + +const ( + ImageUsageInstance ImageUsageType = "instance" + ImageUsageNone ImageUsageType = "none" +) + +type SImage struct { + storageCache *SStoragecache + + Architecture string + CreationTime time.Time + Description string + ImageId string + ImageName string + OSName string + OSType string + ImageOwnerAlias ImageOwnerType + IsSupportCloudinit bool + IsSupportIoOptimized bool + Platform string + Size int + Status ImageStatusType + Usage string +} + +func (self *SImage) GetId() string { + return self.ImageId +} + +func (self *SImage) GetName() string { + return self.ImageName +} + +func (self *SImage) GetGlobalId() string { + return fmt.Sprintf("%s-%s") +} + +func (self *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return self.storageCache +} + +func (self *SImage) GetStatus() string { + switch self.Status { + case ImageStatusCreating: + return models.IMAGE_STATUS_QUEUED + case ImageStatusAvailable: + return models.IMAGE_STATUS_ACTIVE + case ImageStatusUnAvailable: + return models.IMAGE_STATUS_DELETED + case ImageStatusCreateFailed: + return models.IMAGE_STATUS_KILLED + default: + return models.IMAGE_STATUS_KILLED + } +} + +// {"ImageId":"m-j6c1qlpa7oebbg1n2k60","RegionId":"cn-hongkong","RequestId":"F8B2F6A1-F6AA-4C92-A54C-C4A309CF811F","TaskId":"t-j6c1qlpa7oebbg1rcl9t"} + +type ImageImportTask struct { + ImageId string + RegionId string + // RequestId string + TaskId string +} + +func (self *SRegion) ImportImage(name string, bucket string, key string) (*ImageImportTask, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["ImageName"] = name + params["Architecture"] = "x86_64" + params["OSType"] = "linux" + params["Platform"] = "Others Linux" + params["DiskDeviceMapping.1.OSSBucket"] = bucket + params["DiskDeviceMapping.1.OSSObject"] = key + + body, err := self.ecsRequest("ImportImage", params) + if err != nil { + log.Errorf("ImportImage fail %s", err) + return nil, err + } + + log.Infof("%s", body) + result := ImageImportTask{} + err = body.Unmarshal(&result) + if err != nil { + log.Errorf("unmarshal result error %s", err) + return nil, err + } + + return &result, nil +} + +func (self *SRegion) GetImage(imageId string) (*SImage, error) { + images, _, err := self.GetImages("", ImageOwnerSelf, []string{imageId}, "", 0, 1) + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, fmt.Errorf("image %s not found", imageId) + } + return &images[0], nil +} + +func (self *SRegion) GetImageByName(name string) (*SImage, error) { + images, _, err := self.GetImages("", ImageOwnerSelf, nil, name, 0, 1) + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &images[0], nil +} + +func (self *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) { + image, err := self.GetImage(imageId) + if err != nil { + return "", err + } + return image.Status, nil +} + +func (self *SRegion) GetImages(status ImageStatusType, owner ImageOwnerType, imageId []string, name string, offset int, limit int) ([]SImage, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + + if len(status) > 0 { + params["Status"] = string(status) + } else { + params["Status"] = "Creating,Available,UnAvailable,CreateFailed" + } + if imageId != nil && len(imageId) > 0 { + params["ImageId"] = strings.Join(imageId, ",") + } + if len(owner) > 0 { + params["ImageOwnerAlias"] = string(owner) + } + + if len(name) > 0 { + params["ImageName"] = name + } + + log.Debugf("%s", params) + + body, err := self.ecsRequest("DescribeImages", params) + if err != nil { + log.Errorf("DescribeImages fail %s", err) + return nil, 0, err + } + + images := make([]SImage, 0) + err = body.Unmarshal(&images, "Images", "Image") + if err != nil { + log.Errorf("unmarshal images fail %s", err) + return nil, 0, nil + } + total, _ := body.Int("TotalCount") + return images, int(total), nil +} + +func (self *SRegion) DeleteImage(imageId string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["ImageId"] = imageId + params["Force"] = "true" + + _, err := self.ecsRequest("DeleteImage", params) + if err != nil { + log.Errorf("DeleteImage fail %s", err) + return err + } + return nil +} diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go new file mode 100644 index 0000000000..dbb5d61dde --- /dev/null +++ b/pkg/util/aliyun/instance.go @@ -0,0 +1,479 @@ +package aliyun + +import ( + "fmt" + "github.com/yunionio/jsonutils" + "github.com/yunionio/log" + "github.com/yunionio/pkg/util/osprofile" + "github.com/yunionio/pkg/util/seclib" + "github.com/yunionio/pkg/utils" + "time" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +const ( + // Running：运行中 + //Starting：启动中 + //Stopping：停止中 + //Stopped：已停止 + + InstanceStatusStopped = "Stopped" + InstanceStatusRunning = "Running" + InstanceStatusStopping = "Stopping" + InstanceStatusStarting = "Starting" +) + +type SDedicatedHostAttribute struct { + DedicatedHostId string + DedicatedHostName string +} + +type SEipAddress struct { + AllocationId string + InternetChargeType string + IpAddress string +} + +func (self *SEipAddress) GetIP() string { + return self.IpAddress +} + +func (self *SEipAddress) GetAllocationId() string { + return self.AllocationId +} + +func (self *SEipAddress) GetChargeType() string { + return self.GetChargeType() +} + +type SIpAddress struct { + IpAddress []string +} + +type SNetworkInterfaces struct { + NetworkInterface []SNetworkInterface +} + +type SNetworkInterface struct { + MacAddress string + NetworkInterfaceId string + PrimaryIpAddress string +} + +type SOperationLocks struct { + LockReason []string +} + +type SSecurityGroupIds struct { + SecurityGroupId []string +} + +// {"NatIpAddress":"","PrivateIpAddress":{"IpAddress":["192.168.220.214"]},"VSwitchId":"vsw-2ze9cqwza4upoyujq1thd","VpcId":"vpc-2zer4jy8ix3i8f0coc5uw"} + +type SVpcAttributes struct { + NatIpAddress string + PrivateIpAddress SIpAddress + VSwitchId string + VpcId string +} + +type SInstance struct { + host *SHost + + idisks []cloudprovider.ICloudDisk + + AutoReleaseTime string + ClusterId string + Cpu int8 + CreationTime time.Time + DedicatedHostAttribute SDedicatedHostAttribute + Description string + DeviceAvailable bool + EipAddress SEipAddress + ExpiredTime time.Time + GPUAmount int + GPUSpec string + HostName string + ImageId string + InnerIpAddress SIpAddress + InstanceChargeType InstanceChargeType + InstanceId string + InstanceName string + InstanceNetworkType string + InstanceType string + InstanceTypeFamily string + InternetChargeType string + InternetMaxBandwidthIn int + InternetMaxBandwidthOut int + IoOptimized bool + Memory int + NetworkInterfaces SNetworkInterfaces + OSName string + OSType string + OperationLocks SOperationLocks + PublicIpAddress SIpAddress + Recyclable bool + RegionId string + ResourceGroupId string + SaleCycle string + SecurityGroupIds SSecurityGroupIds + SerialNumber string + SpotPriceLimit string + SpotStrategy string + StartTime time.Time + Status string + StoppedMode string + VlanId string + VpcAttributes SVpcAttributes + ZoneId string +} + +// {"AutoReleaseTime":"","ClusterId":"","Cpu":1,"CreationTime":"2018-05-23T07:58Z","DedicatedHostAttribute":{"DedicatedHostId":"","DedicatedHostName":""},"Description":"","DeviceAvailable":true,"EipAddress":{"AllocationId":"","InternetChargeType":"","IpAddress":""},"ExpiredTime":"2018-05-30T16:00Z","GPUAmount":0,"GPUSpec":"","HostName":"iZ2ze57isp1ali72tzkjowZ","ImageId":"centos_7_04_64_20G_alibase_201701015.vhd","InnerIpAddress":{"IpAddress":[]},"InstanceChargeType":"PrePaid","InstanceId":"i-2ze57isp1ali72tzkjow","InstanceName":"gaoxianqi-test-7days","InstanceNetworkType":"vpc","InstanceType":"ecs.t5-lc2m1.nano","InstanceTypeFamily":"ecs.t5","InternetChargeType":"PayByBandwidth","InternetMaxBandwidthIn":-1,"InternetMaxBandwidthOut":0,"IoOptimized":true,"Memory":512,"NetworkInterfaces":{"NetworkInterface":[{"MacAddress":"00:16:3e:10:f0:c9","NetworkInterfaceId":"eni-2zecqsagtpztl6x5hu2r","PrimaryIpAddress":"192.168.220.214"}]},"OSName":"CentOS 7.4 64位","OSType":"linux","OperationLocks":{"LockReason":[]},"PublicIpAddress":{"IpAddress":[]},"Recyclable":false,"RegionId":"cn-beijing","ResourceGroupId":"","SaleCycle":"Week","SecurityGroupIds":{"SecurityGroupId":["sg-2zecqsagtpztl6x9zynl"]},"SerialNumber":"df05d9b4-df3d-4400-88d1-5f843f0dd088","SpotPriceLimit":0.000000,"SpotStrategy":"NoSpot","StartTime":"2018-05-23T07:58Z","Status":"Running","StoppedMode":"Not-applicable","VlanId":"","VpcAttributes":{"NatIpAddress":"","PrivateIpAddress":{"IpAddress":["192.168.220.214"]},"VSwitchId":"vsw-2ze9cqwza4upoyujq1thd","VpcId":"vpc-2zer4jy8ix3i8f0coc5uw"},"ZoneId":"cn-beijing-f"} + +func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit int) ([]SInstance, int, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + + if len(zoneId) > 0 { + params["ZoneId"] = zoneId + } + + if ids != nil && len(ids) > 0 { + params["InstanceIds"] = jsonutils.Marshal(ids).String() + } + + body, err := self.ecsRequest("DescribeInstances", params) + if err != nil { + log.Errorf("GetInstances fail %s", err) + return nil, 0, err + } + + instances := make([]SInstance, 0) + err = body.Unmarshal(&instances, "Instances", "Instance") + if err != nil { + log.Errorf("Unmarshal security group details fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return instances, int(total), nil +} + +func (self *SInstance) GetCreateTime() time.Time { + return self.CreationTime +} + +func (self *SInstance) GetIHost() cloudprovider.ICloudHost { + return self.host +} + +func (self *SInstance) GetId() string { + return self.InstanceId +} + +func (self *SInstance) GetName() string { + return self.HostName +} + +func (self *SInstance) GetGlobalId() string { + return self.InstanceId +} + +func (self *SInstance) fetchDisks() error { + disks, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", 0, 50) + if err != nil { + log.Errorf("fetchDisks fail %s", err) + return err + } + if total > len(disks) { + disks, _, err = self.host.zone.region.GetDisks(self.InstanceId, "", "", 0, total) + } + self.idisks = make([]cloudprovider.ICloudDisk, len(disks)) + for i := 0; i < len(disks); i += 1 { + store, err := self.host.zone.getStorageByCategory(disks[i].Category) + if err != nil { + return err + } + disks[i].storage = store + self.idisks[i] = &disks[i] + } + return nil +} + +func (self *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + if self.idisks == nil { + err := self.fetchDisks() + if err != nil { + return nil, err + } + } + return self.idisks, nil +} + +func (self *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) { + nics := make([]cloudprovider.ICloudNic, 0) + for _, ip := range self.VpcAttributes.PrivateIpAddress.IpAddress { + nic := SInstanceNic{instance: self, ipAddr: ip} + nics = append(nics, &nic) + } + return nics, nil +} + +func (self *SInstance) GetEIP() cloudprovider.ICloudEIP { + return &self.EipAddress +} + +func (self *SInstance) GetVcpuCount() int8 { + return self.Cpu +} + +func (self *SInstance) GetVmemSizeMB() int { + return self.Memory +} + +func (self *SInstance) GetBootOrder() string { + return "dcn" +} + +func (self *SInstance) GetVga() string { + return "std" +} + +func (self *SInstance) GetVdi() string { + return "vnc" +} + +func (self *SInstance) GetOSType() string { + return osprofile.NormalizeOSType(self.OSType) +} + +func (self *SInstance) GetOSName() string { + return self.OSName +} + +func (self *SInstance) GetBios() string { + return "BIOS" +} + +func (self *SInstance) GetMachine() string { + return "pc" +} + +func (self *SInstance) GetStatus() string { + // Running：运行中 + //Starting：启动中 + //Stopping：停止中 + //Stopped：已停止 + switch self.Status { + case InstanceStatusRunning: + return models.VM_RUNNING + case InstanceStatusStarting: + return models.VM_STARTING + case InstanceStatusStopping: + return models.VM_STOPPING + case InstanceStatusStopped: + return models.VM_READY + default: + return models.VM_UNKNOWN + } +} + +func (self *SInstance) GetRemoteStatus() string { + // Running：运行中 + //Starting：启动中 + //Stopping：停止中 + //Stopped：已停止 + switch self.Status { + case InstanceStatusRunning: + return cloudprovider.CloudVMStatusRunning + case InstanceStatusStarting: + return cloudprovider.CloudVMStatusStopped + case InstanceStatusStopping: + return cloudprovider.CloudVMStatusRunning + case InstanceStatusStopped: + return cloudprovider.CloudVMStatusStopped + default: + return cloudprovider.CloudVMStatusOther + } +} + +func (self *SInstance) GetHypervisor() string { + return models.HYPERVISOR_ALIYUN +} + +func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { + instances, _, err := self.GetInstances("", []string{instanceId}, 0, 1) + if err != nil { + return nil, err + } + if len(instances) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &instances[0], nil +} + +func (self *SRegion) CreateInstance(name string, imageId string, instanceType string, securityGroupId string, + zoneId string, desc string, passwd string, disks []SDisk, vSwitchId string, ipAddr string, + keypair string) (string, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["ImageId"] = imageId + params["InstanceType"] = instanceType + params["SecurityGroupId"] = securityGroupId + params["ZoneId"] = zoneId + params["InstanceName"] = name + params["Description"] = desc + params["InternetChargeType"] = "PayByTraffic" + params["InternetMaxBandwidthIn"] = "200" + params["InternetMaxBandwidthOut"] = "100" + params["HostName"] = name + params["Password"] = passwd + params["IoOptimized"] = "optimized" + for i, d := range disks { + if i == 0 { + params["SystemDisk.Category"] = d.Category + params["SystemDisk.Size"] = fmt.Sprintf("%d", d.Size) + params["SystemDisk.DiskName"] = d.GetName() + params["SystemDisk.Description"] = d.Description + } else { + params[fmt.Sprintf("DataDisk.%d.Size", i)] = fmt.Sprintf("%d", d.Size) + params[fmt.Sprintf("DataDisk.%d.Category", i)] = d.Category + params[fmt.Sprintf("DataDisk.%d.DiskName", i)] = d.GetName() + params[fmt.Sprintf("DataDisk.%d.Description", i)] = d.Description + params[fmt.Sprintf("DataDisk.%d.Encrypted", i)] = "false" + } + } + params["VSwitchId"] = vSwitchId + params["PrivateIpAddress"] = ipAddr + params["InstanceChargeType"] = "PostPaid" + params["SpotStrategy"] = "NoSpot" + if len(keypair) > 0 { + params["KeyPairName"] = keypair + } + params["ClientToken"] = utils.GenRequestId(12) + + body, err := self.ecsRequest("CreateInstance", params) + if err != nil { + log.Errorf("CreateInstance fail %s", err) + return "", err + } + instanceId, _ := body.GetString("InstanceId") + return instanceId, nil +} + +func (self *SRegion) doStartVM(instanceId string) error { + return self.instanceOperation(instanceId, "StartInstance", nil) +} + +func (self *SRegion) doStopVM(instanceId string, isForce bool) error { + params := make(map[string]string) + if isForce { + params["ForceStop"] = "true" + } else { + params["ForceStop"] = "false" + } + params["StoppedMode"] = "KeepCharging" + return self.instanceOperation(instanceId, "StopInstance", params) +} + +func (self *SRegion) doDeleteVM(instanceId string) error { + return self.instanceOperation(instanceId, "DeleteInstance", nil) +} + +func (self *SRegion) waitInstanceStatus(instanceId string, target string, interval time.Duration, timeout time.Duration) error { + startTime := time.Now() + for time.Now().Sub(startTime) < timeout { + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + return err + } + if status == target { + return nil + } + time.Sleep(interval) + } + return cloudprovider.ErrTimeout +} + +func (self *SInstance) waitStatus(target string, interval time.Duration, timeout time.Duration) error { + return self.host.zone.region.waitInstanceStatus(self.InstanceId, target, interval, timeout) +} + +func (self *SRegion) StartVM(instanceId string) error { + status, err := self.GetInstanceStatus(instanceId) + if status != InstanceStatusStopped { + return cloudprovider.ErrInvalidStatus + } + err = self.doStartVM(instanceId) + if err != nil { + return err + } + return self.waitInstanceStatus(instanceId, InstanceStatusRunning, time.Second*5, time.Second*180) // 3 minutes to timeout +} + +func (self *SRegion) StopVM(instanceId string, isForce bool) error { + status, err := self.GetInstanceStatus(instanceId) + if status != InstanceStatusRunning { + return cloudprovider.ErrInvalidStatus + } + err = self.doStopVM(instanceId, isForce) + if err != nil { + return err + } + return self.waitInstanceStatus(instanceId, InstanceStatusStopped, time.Second*10, time.Second*300) // 5 minutes to timeout +} + +func (self *SRegion) DeleteVM(instanceId string) error { + status, err := self.GetInstanceStatus(instanceId) + if status == InstanceStatusRunning { + err := self.StopVM(instanceId, true) + if err != nil { + return err + } + } else if status != InstanceStatusStopped { + return cloudprovider.ErrInvalidStatus + } + err = self.doDeleteVM(instanceId) + if err != nil { + return err + } + err = self.waitInstanceStatus(instanceId, InstanceStatusRunning, time.Second*10, time.Second*300) // 5 minutes to timeout + if err == cloudprovider.ErrNotFound { + return nil + } else if err == nil { + return cloudprovider.ErrTimeout + } else { + return err + } +} + +func (self *SInstance) StartVM() error { + return self.host.zone.region.StartVM(self.InstanceId) +} + +func (self *SInstance) StopVM(isForce bool) error { + return self.host.zone.region.StopVM(self.InstanceId, isForce) +} + +func (self *SInstance) DeleteVM() error { + return self.host.zone.region.DeleteVM(self.InstanceId) +} + +func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { + url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) + if err != nil { + return nil, err + } + passwd := seclib.RandomPassword(6) + err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) + if err != nil { + return nil, err + } + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(url), "url") + ret.Add(jsonutils.NewString(passwd), "password") + ret.Add(jsonutils.NewString("aliyun"), "protocol") + ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") + return ret, nil +} diff --git a/pkg/util/aliyun/instancenic.go b/pkg/util/aliyun/instancenic.go new file mode 100644 index 0000000000..7284d4319b --- /dev/null +++ b/pkg/util/aliyun/instancenic.go @@ -0,0 +1,40 @@ +package aliyun + +import ( + "github.com/yunionio/onecloud/pkg/cloudprovider" + "yunion.io/yunioncloud/pkg/util/netutils" +) + +type SInstanceNic struct { + instance *SInstance + ipAddr string +} + +func (self *SInstanceNic) GetIP() string { + return self.ipAddr +} + +func (self *SInstanceNic) GetMAC() string { + ip, _ := netutils.NewIPV4Addr(self.ipAddr) + return ip.ToMac("00:16:") +} + +func (self *SInstanceNic) GetDriver() string { + return "virtio" +} + +func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork { + vswitchId := self.instance.VpcAttributes.VSwitchId + wires, err := self.instance.host.GetIWires() + if err != nil { + return nil + } + for i := 0; i < len(wires); i += 1 { + wire := wires[i].(*SWire) + net := wire.getNetworkById(vswitchId) + if net != nil { + return net + } + } + return nil +} diff --git a/pkg/util/aliyun/instancetype.go b/pkg/util/aliyun/instancetype.go new file mode 100644 index 0000000000..c2104b8dad --- /dev/null +++ b/pkg/util/aliyun/instancetype.go @@ -0,0 +1,49 @@ +package aliyun + +import ( + // "time" + "github.com/yunionio/log" +) + +// {"CpuCoreCount":1,"EniQuantity":1,"GPUAmount":0,"GPUSpec":"","InstanceTypeFamily":"ecs.t1","InstanceTypeId":"ecs.t1.xsmall","LocalStorageCategory":"","MemorySize":0.500000} +// InstanceBandwidthRx":26214400,"InstanceBandwidthTx":26214400,"InstancePpsRx":4500000,"InstancePpsTx":4500000 + +type SInstanceType struct { + CpuCoreCount int + MemorySize float32 + EniQuantity int // 实例规格支持网卡数量 + GPUAmount int + GPUSpec string + InstanceTypeFamily string + InstanceTypeId string + LocalStorageCategory string + LocalStorageAmount int + LocalStorageCapacity int64 + InstanceBandwidthRx int + InstanceBandwidthTx int + InstancePpsRx int + InstancePpsTx int +} + +func (self *SRegion) GetInstanceTypes() ([]SInstanceType, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + + body, err := self.ecsRequest("DescribeInstanceTypes", params) + if err != nil { + log.Errorf("GetInstanceTypes fail %s", err) + return nil, err + } + + instanceTypes := make([]SInstanceType, 0) + err = body.Unmarshal(&instanceTypes, "InstanceTypes", "InstanceType") + if err != nil { + log.Errorf("Unmarshal instance type details fail %s", err) + return nil, err + } + return instanceTypes, nil +} + +func (self *SInstanceType) memoryMB() int { + return int(self.MemorySize * 1024) +} diff --git a/pkg/util/aliyun/keypair.go b/pkg/util/aliyun/keypair.go new file mode 100644 index 0000000000..bb3d18f5a5 --- /dev/null +++ b/pkg/util/aliyun/keypair.go @@ -0,0 +1,65 @@ +package aliyun + +import ( + "fmt" + + "github.com/yunionio/log" +) + +type SKeypair struct { + KeyPairFingerPrint string + KeyPairName string +} + +func (self *SRegion) GetKeypairs(finger string, name string, offset int, limit int) ([]SKeypair, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + if len(finger) > 0 { + params["KeyPairFingerPrint"] = finger + } + if len(name) > 0 { + params["KeyPairName"] = name + } + + body, err := self.ecsRequest("DescribeKeyPairs", params) + if err != nil { + log.Errorf("GetKeypairs fail %s", err) + return nil, 0, err + } + + keypairs := make([]SKeypair, 0) + err = body.Unmarshal(&keypairs, "KeyPairs", "KeyPair") + if err != nil { + log.Errorf("Unmarshal keypair fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return keypairs, int(total), nil +} + +func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PublicKeyBody"] = pubKey + params["KeyPairName"] = name + + body, err := self.ecsRequest("ImportKeyPair", params) + if err != nil { + log.Errorf("ImportKeypair fail %s", err) + return nil, err + } + + log.Debugf("%s", body) + keypair := SKeypair{} + err = body.Unmarshal(&keypair) + if err != nil { + log.Errorf("Unmarshall keypair fail %s", err) + return nil, err + } + return &keypair, nil +} diff --git a/pkg/util/aliyun/provider/provider.go b/pkg/util/aliyun/provider/provider.go new file mode 100644 index 0000000000..2078cf839f --- /dev/null +++ b/pkg/util/aliyun/provider/provider.go @@ -0,0 +1,68 @@ +package provider + +import ( + "github.com/yunionio/jsonutils" + // "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/util/aliyun" +) + +type SAliyunProviderFactory struct { + providerTable map[string]*SAliyunProvider +} + +func (self *SAliyunProviderFactory) GetId() string { + return aliyun.CLOUD_PROVIDER_ALIYUN +} + +func (self *SAliyunProviderFactory) GetProvider(providerId, url, account, secret string) (cloudprovider.ICloudProvider, error) { + provider, ok := self.providerTable[providerId] + if ok { + return provider, nil + } + client, err := aliyun.NewAliyunClient(providerId, account, secret) + if err != nil { + return nil, err + } + self.providerTable[providerId] = &SAliyunProvider{client: client} + return self.providerTable[providerId], nil +} + +func init() { + factory := SAliyunProviderFactory{ + providerTable: make(map[string]*SAliyunProvider), + } + cloudprovider.RegisterFactory(&factory) +} + +type SAliyunProvider struct { + client *aliyun.SAliyunClient +} + +func (self *SAliyunProvider) IsPublicCloud() bool { + return true +} + +func (self *SAliyunProvider) GetId() string { + return aliyun.CLOUD_PROVIDER_ALIYUN +} + +func (self *SAliyunProvider) GetName() string { + return aliyun.CLOUD_PROVIDER_ALIYUN_CN +} + +func (self *SAliyunProvider) GetSysInfo() (jsonutils.JSONObject, error) { + regions := self.client.GetRegions() + info := jsonutils.NewDict() + info.Add(jsonutils.NewInt(int64(len(regions))), "region_count") + return info, nil +} + +func (self *SAliyunProvider) GetIRegions() []cloudprovider.ICloudRegion { + return self.client.GetIRegions() +} + +func (self *SAliyunProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { + return self.client.GetIRegionById(id) +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go new file mode 100644 index 0000000000..3870c40274 --- /dev/null +++ b/pkg/util/aliyun/region.go @@ -0,0 +1,439 @@ +package aliyun + +import ( + "fmt" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/yunionio/jsonutils" + "github.com/yunionio/log" + "github.com/yunionio/pkg/utils" + "strings" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type SRegion struct { + client *SAliyunClient + ecsClient *sdk.Client + ossClient *oss.Client + + RegionId string + LocalName string + + izones []cloudprovider.ICloudZone + + ivpcs []cloudprovider.ICloudVpc + + storageCache *SStoragecache + + instanceTypes []SInstanceType +} + +func (self *SRegion) GetClient() *SAliyunClient { + return self.client +} + +func (self *SRegion) getEcsClient() (*sdk.Client, error) { + if self.ecsClient == nil { + cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret) + if err != nil { + return nil, err + } + self.ecsClient = cli + } + return self.ecsClient, nil +} + +// oss endpoint +// https://help.aliyun.com/document_detail/31837.html?spm=a2c4g.11186623.2.6.6E8ZkO +func (self *SRegion) GetOSSExternalDomain() string { + return fmt.Sprintf("oss-%s.aliyuncs.com", self.RegionId) +} + +func (self *SRegion) GetOSSInternalDomain() string { + return fmt.Sprintf("oss-%s-internal.aliyuncs.com", self.RegionId) +} + +func (self *SRegion) GetOssClient() (*oss.Client, error) { + if self.ossClient == nil { + // https://help.aliyun.com/document_detail/31837.html?spm=a2c4g.11186623.2.6.XqEgD1 + ep := self.GetOSSExternalDomain() + cli, err := oss.New(ep, self.client.accessKey, self.client.secret) + if err != nil { + return nil, err + } + self.ossClient = cli + } + return self.ossClient, nil +} + +func (self *SRegion) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getEcsClient() + if err != nil { + return nil, err + } + return jsonRequest(cli, apiName, params) +} + +///////////////////////////////////////////////////////////////////////////// +func (self *SRegion) GetId() string { + return self.RegionId +} + +func (self *SRegion) GetName() string { + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_ALIYUN_CN, self.LocalName) +} + +func (self *SRegion) GetGlobalId() string { + return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_ALIYUN, self.RegionId) +} + +func (self *SRegion) GetLatitude() float32 { + return 0.0 +} + +func (self *SRegion) GetLongitude() float32 { + return 0.0 +} + +func (self *SRegion) GetStatus() string { + return models.CLOUD_REGION_STATUS_INSERVER +} + +func (self *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { + if self.izones == nil { + err := self.fetchInfrastructure() + if err != nil { + return nil, err + } + } + return self.izones, nil +} + +func (self *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) { + if self.izones == nil { + err := self.fetchInfrastructure() + if err != nil { + return nil, err + } + } + for i := 0; i < len(self.izones); i += 1 { + if self.izones[i].GetGlobalId() == id { + return self.izones[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) getStoragecache() *SStoragecache { + if self.storageCache == nil { + self.storageCache = &SStoragecache{region: self} + } + return self.storageCache +} + +func (self *SRegion) _fetchZones(chargeType InstanceChargeType, spotStrategy SpotStrategyType) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + if len(chargeType) > 0 { + params["InstanceChargeType"] = string(chargeType) + } + if len(spotStrategy) > 0 { + params["SpotStrategy"] = string(spotStrategy) + } + body, err := self.ecsRequest("DescribeZones", params) + if err != nil { + return err + } + + zones := make([]SZone, 0) + err = body.Unmarshal(&zones, "Zones", "Zone") + if err != nil { + return err + } + + self.izones = make([]cloudprovider.ICloudZone, len(zones)) + + for i := 0; i < len(zones); i += 1 { + zones[i].region = self + self.izones[i] = &zones[i] + } + + return nil +} + +func (self *SRegion) getZoneById(id string) (*SZone, error) { + if self.izones == nil { + err := self.fetchInfrastructure() + if err != nil { + return nil, err + } + } + for i := 0; i < len(self.izones); i += 1 { + zone := self.izones[i].(*SZone) + if zone.ZoneId == id { + return zone, nil + } + } + return nil, fmt.Errorf("no such zone %s", id) +} + +func (self *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) { + if self.ivpcs == nil { + err := self.fetchInfrastructure() + if err != nil { + return nil, err + } + } + return self.ivpcs, nil +} + +func (self *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { + if self.ivpcs == nil { + err := self.fetchInfrastructure() + if err != nil { + return nil, err + } + } + for i := 0; i < len(self.ivpcs); i += 1 { + if self.ivpcs[i].GetGlobalId() == id { + return self.ivpcs[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) fetchIVpcs() error { + vpcs := make([]SVpc, 0) + for { + part, total, err := self.GetVpcs(len(vpcs), 50) + if err != nil { + return err + } + vpcs = append(vpcs, part...) + if len(vpcs) >= total { + break + } + } + self.ivpcs = make([]cloudprovider.ICloudVpc, len(vpcs)) + for i := 0; i < len(vpcs); i += 1 { + vpcs[i].region = self + self.ivpcs[i] = &vpcs[i] + } + return nil +} + +func (self *SRegion) fetchInfrastructure() error { + err := self._fetchZones(PostPaidInstanceChargeType, NoSpotStrategy) + if err != nil { + return err + } + err = self.fetchIVpcs() + if err != nil { + return err + } + for i := 0; i < len(self.ivpcs); i += 1 { + for j := 0; j < len(self.izones); j += 1 { + zone := self.izones[j].(*SZone) + vpc := self.ivpcs[i].(*SVpc) + wire := SWire{zone: zone, vpc: vpc} + zone.addWire(&wire) + vpc.addWire(&wire) + } + } + return nil +} + +func (self *SRegion) GetVpcs(offset int, limit int) ([]SVpc, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + + body, err := self.ecsRequest("DescribeVpcs", params) + if err != nil { + log.Errorf("GetVpcs fail %s", err) + return nil, 0, err + } + + vpcs := make([]SVpc, 0) + err = body.Unmarshal(&vpcs, "Vpcs", "Vpc") + if err != nil { + log.Errorf("Unmarshal vpc fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return vpcs, int(total), nil +} + +func (self *SRegion) GetVRouters(offset int, limit int) ([]SVRouter, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + + body, err := self.ecsRequest("DescribeVRouters", params) + if err != nil { + log.Errorf("GetVRouters fail %s", err) + return nil, 0, err + } + + vrouters := make([]SVRouter, 0) + err = body.Unmarshal(&vrouters, "VRouters", "VRouter") + if err != nil { + log.Errorf("Unmarshal vrouter fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return vrouters, int(total), nil +} + +func (self *SRegion) GetRouteTables(ids []string, offset int, limit int) ([]SRouteTable, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + if ids != nil && len(ids) > 0 { + params["RouteTableId"] = strings.Join(ids, ",") + } + + body, err := self.ecsRequest("DescribeRouteTables", params) + if err != nil { + log.Errorf("GetRouteTables fail %s", err) + return nil, 0, err + } + + routetables := make([]SRouteTable, 0) + err = body.Unmarshal(&routetables, "RouteTables", "RouteTable") + if err != nil { + log.Errorf("Unmarshal routetables fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return routetables, int(total), nil +} + +func (self *SRegion) GetVSwitches(ids []string, offset int, limit int) ([]SVSwitch, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + if ids != nil && len(ids) > 0 { + params["RouteTableId"] = strings.Join(ids, ",") + } + + body, err := self.ecsRequest("DescribeVSwitches", params) + if err != nil { + log.Errorf("GetVSwitches fail %s", err) + return nil, 0, err + } + + switches := make([]SVSwitch, 0) + err = body.Unmarshal(&switches, "VSwitches", "VSwitch") + if err != nil { + log.Errorf("Unmarshal vswitches fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return switches, int(total), nil +} + +func (self *SRegion) GetMatchInstanceTypes(cpu int, memMB int, gpu int, zoneId string) ([]SInstanceType, error) { + if self.instanceTypes == nil { + types, err := self.GetInstanceTypes() + if err != nil { + log.Errorf("GetInstanceTypes %s", err) + return nil, err + } + self.instanceTypes = types + } + var available []string + if len(zoneId) > 0 { + zone, err := self.getZoneById(zoneId) + if err != nil { + return nil, err + } + available = zone.AvailableInstanceTypes.InstanceTypes + } + ret := make([]SInstanceType, 0) + for _, t := range self.instanceTypes { + if t.CpuCoreCount == cpu && memMB == t.memoryMB() && gpu == t.GPUAmount { + if available == nil || utils.IsInStringArray(t.InstanceTypeId, available) { + ret = append(ret, t) + } + } + } + return ret, nil +} + +func (self *SRegion) CreateInstanceSimple(name string, imgId string, cpu int, memGB int, storageType string, dataDiskSizesGB []int, vswitchId string, passwd string) (*SInstance, error) { + self.fetchInfrastructure() + for i := 0; i < len(self.izones); i += 1 { + z := self.izones[i].(*SZone) + log.Debugf("Search in zone %s", z.LocalName) + net := z.getNetworkById(vswitchId) + if net != nil { + inst, err := z.getHost().CreateVM(name, imgId, cpu, memGB*1024, vswitchId, "", "", passwd, storageType, dataDiskSizesGB) + if err != nil { + return nil, err + } + return inst.(*SInstance), nil + } + } + return nil, fmt.Errorf("cannot find vswitch %s", vswitchId) +} + +func (self *SRegion) instanceOperation(instanceId string, opname string, extra map[string]string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["InstanceId"] = instanceId + if extra != nil && len(extra) > 0 { + for k, v := range extra { + params[k] = v + } + } + _, err := self.ecsRequest(opname, params) + return err +} + +func (self *SRegion) GetInstanceStatus(instanceId string) (string, error) { + instance, err := self.GetInstance(instanceId) + if err != nil { + return "", err + } + return instance.Status, nil +} + +func (self *SRegion) GetInstanceVNCUrl(instanceId string) (string, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["InstanceId"] = instanceId + body, err := self.ecsRequest("DescribeInstanceVncUrl", params) + if err != nil { + return "", err + } + return body.GetString("VncUrl") +} + +func (self *SRegion) ModifyInstanceVNCUrlPassword(instanceId string, passwd string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["InstanceId"] = instanceId + params["VncPassword"] = passwd // must be 6 digital + alphabet + _, err := self.ecsRequest("ModifyInstanceVncPasswd", params) + return err +} diff --git a/pkg/util/aliyun/routetable.go b/pkg/util/aliyun/routetable.go new file mode 100644 index 0000000000..65e8f6101e --- /dev/null +++ b/pkg/util/aliyun/routetable.go @@ -0,0 +1,33 @@ +package aliyun + +import ( + "time" +) + +// {"CreationTime":"2017-03-19T13:37:40Z","RouteEntrys":{"RouteEntry":[{"DestinationCidrBlock":"172.31.32.0/20","InstanceId":"","NextHopType":"local","NextHops":{"NextHop":[]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","Status":"Available","Type":"System"},{"DestinationCidrBlock":"100.64.0.0/10","InstanceId":"","NextHopType":"service","NextHops":{"NextHop":[]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","Status":"Available","Type":"System"}]},"RouteTableId":"vtb-j6c60lectdi80rk5xz43g","RouteTableType":"System","VRouterId":"vrt-j6c00qrol733dg36iq4qj"} + +type SNextHops struct { + NextHop []string +} + +type SRouteEntry struct { + DestinationCidrBlock string + InstanceId string + NextHopType string + NextHops SNextHops + RouteTableId string + Status string + Type string +} + +type SRouteEntrys struct { + RouteEntry []SRouteEntry +} + +type SRouteTable struct { + CreationTime time.Time + RouteEntrys SRouteEntrys + RouteTableId string + RouteTableType string + VRouterId string +} diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go new file mode 100644 index 0000000000..63c51ea4f9 --- /dev/null +++ b/pkg/util/aliyun/securitygroup.go @@ -0,0 +1,101 @@ +package aliyun + +import ( + "fmt" + "time" + + "github.com/yunionio/log" +) + +// {"CreationTime":"2017-03-19T13:37:48Z","Description":"System created security group.","SecurityGroupId":"sg-j6cannq0xxj2r9z0yxwl","SecurityGroupName":"sg-j6cannq0xxj2r9z0yxwl","Tags":{"Tag":[]},"VpcId":"vpc-j6c86z3sh8ufhgsxwme0q"} +// {"Description":"System created security group.","InnerAccessPolicy":"Accept","Permissions":{"Permission":[{"CreateTime":"2017-03-19T13:37:54Z","Description":"","DestCidrIp":"","DestGroupId":"","DestGroupName":"","DestGroupOwnerAccount":"","Direction":"ingress","IpProtocol":"ALL","NicType":"intranet","Policy":"Accept","PortRange":"-1/-1","Priority":110,"SourceCidrIp":"0.0.0.0/0","SourceGroupId":"","SourceGroupName":"","SourceGroupOwnerAccount":""},{"CreateTime":"2017-03-19T13:37:55Z","Description":"","DestCidrIp":"0.0.0.0/0","DestGroupId":"","DestGroupName":"","DestGroupOwnerAccount":"","Direction":"egress","IpProtocol":"ALL","NicType":"intranet","Policy":"Accept","PortRange":"-1/-1","Priority":110,"SourceCidrIp":"","SourceGroupId":"","SourceGroupName":"","SourceGroupOwnerAccount":""}]},"RegionId":"cn-hongkong","RequestId":"FBFE0950-5F2D-40DE-8C3C-E5A62AE7F7DA","SecurityGroupId":"sg-j6cannq0xxj2r9z0yxwl","SecurityGroupName":"sg-j6cannq0xxj2r9z0yxwl","VpcId":"vpc-j6c86z3sh8ufhgsxwme0q"} + +type SecurityGroupPermissionNicType string + +const ( + IntranetNicType SecurityGroupPermissionNicType = "intranet" + InternetNicType SecurityGroupPermissionNicType = "internet" +) + +type SPermission struct { + CreateTime time.Time + Description string + DestCidrIp string + DestGroupId string + DestGroupName string + DestGroupOwnerAccount string + Direction string + IpProtocol string + NicType SecurityGroupPermissionNicType + Policy string + PortRange string + Priority int + SourceCidrIp string + SourceGroupId string + SourceGroupName string + SourceGroupOwnerAccount string +} + +type SPermissions struct { + Permission []SPermission +} + +type SSecurityGroup struct { + CreationTime time.Time + Description string + SecurityGroupId string + SecurityGroupName string + VpcId string + InnerAccessPolicy string + Permissions SPermissions + RegionId string +} + +func (self *SRegion) GetSecurityGroups(vpcId string, offset int, limit int) ([]SSecurityGroup, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + + body, err := self.ecsRequest("DescribeSecurityGroups", params) + if err != nil { + log.Errorf("GetSecurityGroups fail %s", err) + return nil, 0, err + } + + secgrps := make([]SSecurityGroup, 0) + err = body.Unmarshal(&secgrps, "SecurityGroups", "SecurityGroup") + if err != nil { + log.Errorf("Unmarshal security groups fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return secgrps, int(total), nil +} + +func (self *SRegion) GetSecurityGroupDetails(secGroupId string) (*SSecurityGroup, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGroupId + + body, err := self.ecsRequest("DescribeSecurityGroupAttribute", params) + if err != nil { + log.Errorf("DescribeSecurityGroupAttribute fail %s", err) + return nil, err + } + + log.Debugf("%s", body) + secgrp := SSecurityGroup{} + err = body.Unmarshal(&secgrp) + if err != nil { + log.Errorf("Unmarshal security group details fail %s", err) + return nil, err + } + return &secgrp, nil +} diff --git a/pkg/util/aliyun/shell/disk.go b/pkg/util/aliyun/shell/disk.go new file mode 100644 index 0000000000..1642d0d66f --- /dev/null +++ b/pkg/util/aliyun/shell/disk.go @@ -0,0 +1,23 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type DiskListOptions struct { + Instance string `help:"Instance ID"` + Zone string `help:"Zone ID"` + Category string `help:"Disk category"` + Offset int `help:"List offset"` + Limit int `help:"List limit"` + } + R(&DiskListOptions{}, "disk-list", "List disks", func(cli *aliyun.SRegion, args *DiskListOptions) error { + disks, total, e := cli.GetDisks(args.Instance, args.Zone, args.Category, args.Offset, args.Limit) + if e != nil { + return e + } + printList(disks, total, args.Offset, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/image.go b/pkg/util/aliyun/shell/image.go new file mode 100644 index 0000000000..cf5f021907 --- /dev/null +++ b/pkg/util/aliyun/shell/image.go @@ -0,0 +1,31 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type ImageListOptions struct { + Status string `help:"image status type" choices:"Creating|Available|UnAvailable|CreateFailed"` + Owner string `help:"Owner type" choices:"system|self|others|marketplace"` + Id []string `help:"Image ID"` + Name string `help:"image name"` + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&ImageListOptions{}, "image-list", "List images", func(cli *aliyun.SRegion, args *ImageListOptions) error { + images, total, e := cli.GetImages(aliyun.ImageStatusType(args.Status), aliyun.ImageOwnerType(args.Owner), args.Id, args.Name, args.Offset, args.Limit) + if e != nil { + return e + } + printList(images, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type ImageDeleteOptions struct { + ID string `help:"ID or Name to delete"` + } + R(&ImageDeleteOptions{}, "image-delete", "Delete image", func(cli *aliyun.SRegion, args *ImageDeleteOptions) error { + return cli.DeleteImage(args.ID) + }) +} diff --git a/pkg/util/aliyun/shell/instance.go b/pkg/util/aliyun/shell/instance.go new file mode 100644 index 0000000000..d87675522f --- /dev/null +++ b/pkg/util/aliyun/shell/instance.go @@ -0,0 +1,81 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" + "fmt" +) + +func init() { + type InstanceListOptions struct { + Id []string `help:"IDs of instances to show"` + Zone string `help:"Zone ID"` + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&InstanceListOptions{}, "instance-list", "List intances", func(cli *aliyun.SRegion, args *InstanceListOptions) error { + instances, total, e := cli.GetInstances(args.Zone, args.Id, args.Offset, args.Limit) + if e != nil { + return e + } + printList(instances, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type InstanceCrateOptions struct { + NAME string `help:"name of instance"` + IMAGE string `help:"image ID"` + CPU int `help:"CPU count"` + MEMORYGB int `help:"MemoryGB"` + Disk []int `help:"Data disk sizes int GB"` + STORAGE string `help:"Storage type"` + VSWITCH string `help:"Vswitch ID"` + PASSWD string `help:"password"` + } + R(&InstanceCrateOptions{}, "instance-create", "Create a instance", func(cli *aliyun.SRegion, args *InstanceCrateOptions) error { + instance, e := cli.CreateInstanceSimple(args.NAME, args.IMAGE, args.CPU, args.MEMORYGB, args.STORAGE, args.Disk, args.VSWITCH, args.PASSWD) + if e != nil { + return e + } + printObject(instance) + return nil + }) + + type InstanceOperationOptions struct { + ID string `help:"instance ID"` + } + R(&InstanceOperationOptions{}, "instance-start", "Start a instance", func(cli *aliyun.SRegion, args *InstanceOperationOptions) error { + err := cli.StartVM(args.ID) + if err != nil { + return err + } + return nil + }) + + R(&InstanceOperationOptions{}, "instance-vnc", "Get a instance VNC url", func(cli *aliyun.SRegion, args *InstanceOperationOptions) error { + url, err := cli.GetInstanceVNCUrl(args.ID) + if err != nil { + return err + } + fmt.Println(url) + return nil + }) + + type InstanceStopOptions struct { + ID string `help:"instance ID"` + Force bool `help:"Force stop instance"` + } + R(&InstanceStopOptions{}, "instance-stop", "Stop a instance", func(cli *aliyun.SRegion, args *InstanceStopOptions) error { + err := cli.StopVM(args.ID, args.Force) + if err != nil { + return err + } + return nil + }) + R(&InstanceOperationOptions{}, "instance-delete", "Delete a instance", func(cli *aliyun.SRegion, args *InstanceOperationOptions) error { + err := cli.DeleteVM(args.ID) + if err != nil { + return err + } + return nil + }) +} diff --git a/pkg/util/aliyun/shell/instancetype.go b/pkg/util/aliyun/shell/instancetype.go new file mode 100644 index 0000000000..b66722fca6 --- /dev/null +++ b/pkg/util/aliyun/shell/instancetype.go @@ -0,0 +1,33 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type InstanceTypeListOptions struct { + } + R(&InstanceTypeListOptions{}, "instance-type-list", "List intance types", func(cli *aliyun.SRegion, args *InstanceTypeListOptions) error { + instanceTypes, e := cli.GetInstanceTypes() + if e != nil { + return e + } + printList(instanceTypes, 0, 0, 0, []string{}) + return nil + }) + + type InstanceMatchOptions struct { + CPU int `help:"CPU count"` + MEM int `help:"Memory in MB"` + GPU int `help:"GPU size"` + Zone string `help:"Test in zone"` + } + R(&InstanceMatchOptions{}, "instance-type-select", "Select matching instance types", func(cli *aliyun.SRegion, args *InstanceMatchOptions) error { + instanceTypes, e := cli.GetMatchInstanceTypes(args.CPU, args.MEM, args.GPU, args.Zone) + if e != nil { + return e + } + printList(instanceTypes, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/keypair.go b/pkg/util/aliyun/shell/keypair.go new file mode 100644 index 0000000000..d6d0ee22ec --- /dev/null +++ b/pkg/util/aliyun/shell/keypair.go @@ -0,0 +1,33 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type KeyPairListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&KeyPairListOptions{}, "keypair-list", "List keypairs", func(cli *aliyun.SRegion, args *KeyPairListOptions) error { + keypairs, total, e := cli.GetKeypairs("", "", args.Offset, args.Limit) + if e != nil { + return e + } + printList(keypairs, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type KeyPairImportOptions struct { + NAME string `help:"Name of new keypair"` + PUBKEY string `help:"Public key string"` + } + R(&KeyPairImportOptions{}, "keypair-import", "Import a keypair", func(cli *aliyun.SRegion, args *KeyPairImportOptions) error { + keypair, err := cli.ImportKeypair(args.NAME, args.PUBKEY) + if err != nil { + return err + } + printObject(keypair) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/oss.go b/pkg/util/aliyun/shell/oss.go new file mode 100644 index 0000000000..c3ad891efa --- /dev/null +++ b/pkg/util/aliyun/shell/oss.go @@ -0,0 +1,91 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type OssListOptions struct { + } + R(&OssListOptions{}, "oss-list", "List OSS buckets", func(cli *aliyun.SRegion, args *OssListOptions) error { + oss, err := cli.GetOssClient() + if err != nil { + return err + } + result, err := oss.ListBuckets() + if err != nil { + return err + } + printList(result.Buckets, len(result.Buckets), 0, 50, nil) + return nil + }) + + type OssListBucketOptions struct { + BUCKET string `help:"bucket name"` + } + + R(&OssListBucketOptions{}, "oss-list-bucket", "List content of a OSS bucket", func(cli *aliyun.SRegion, args *OssListBucketOptions) error { + oss, err := cli.GetOssClient() + if err != nil { + return err + } + bucket, err := oss.Bucket(args.BUCKET) + if err != nil { + return err + } + result, err := bucket.ListObjects() + if err != nil { + return err + } + printList(result.Objects, len(result.Objects), 0, len(result.Objects), nil) + return nil + }) + + R(&OssListBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *aliyun.SRegion, args *OssListBucketOptions) error { + oss, err := cli.GetOssClient() + if err != nil { + return err + } + err = oss.CreateBucket(args.BUCKET) + if err != nil { + return err + } + return nil + }) + + type OssUploadOptions struct { + BUCKET string `help:"bucket name"` + KEY string `help:"Object key"` + FILE string `help:"Local file path"` + } + R(&OssUploadOptions{}, "oss-upload", "Upload a file to a OSS bucket", func(cli *aliyun.SRegion, args *OssUploadOptions) error { + oss, err := cli.GetOssClient() + if err != nil { + return err + } + bucket, err := oss.Bucket(args.BUCKET) + if err != nil { + return err + } + err = bucket.UploadFile(args.KEY, args.FILE, 1024*1024) + return err + }) + + type OssDeleteOptions struct { + BUCKET string `help:"bucket name"` + KEY string `help:"Object key"` + } + + R(&OssDeleteOptions{}, "oss-delete", "Delete a file from a OSS bucket", func(cli *aliyun.SRegion, args *OssDeleteOptions) error { + oss, err := cli.GetOssClient() + if err != nil { + return err + } + bucket, err := oss.Bucket(args.BUCKET) + if err != nil { + return err + } + err = bucket.DeleteObject(args.KEY) + return err + }) +} diff --git a/pkg/util/aliyun/shell/region.go b/pkg/util/aliyun/shell/region.go new file mode 100644 index 0000000000..89b4a43eba --- /dev/null +++ b/pkg/util/aliyun/shell/region.go @@ -0,0 +1,15 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type RegionListOptions struct { + } + R(&RegionListOptions{}, "region-list", "List regions", func(cli *aliyun.SRegion, args *RegionListOptions) error { + regions := cli.GetClient().GetRegions() + printList(regions, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/routetable.go b/pkg/util/aliyun/shell/routetable.go new file mode 100644 index 0000000000..a23ee43902 --- /dev/null +++ b/pkg/util/aliyun/shell/routetable.go @@ -0,0 +1,36 @@ +package shell + +import ( + "fmt" + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type RouteTableListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&RouteTableListOptions{}, "routetable-list", "List routetables", func(cli *aliyun.SRegion, args *RouteTableListOptions) error { + routetables, total, e := cli.GetRouteTables(nil, args.Offset, args.Limit) + if e != nil { + return e + } + printList(routetables, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type RouteTableShowOptions struct { + ID string `help:"ID or name of routetable"` + } + R(&RouteTableShowOptions{}, "routetable-show", "Show routetable", func(cli *aliyun.SRegion, args *RouteTableShowOptions) error { + routetables, _, e := cli.GetRouteTables([]string{args.ID}, 0, 1) + if e != nil { + return e + } + if len(routetables) == 0 { + return fmt.Errorf("No such ID %s", args.ID) + } + printObject(routetables[0]) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/secgroup.go b/pkg/util/aliyun/shell/secgroup.go new file mode 100644 index 0000000000..f741395ca3 --- /dev/null +++ b/pkg/util/aliyun/shell/secgroup.go @@ -0,0 +1,33 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type SecurityGroupListOptions struct { + VpcId string `help:"VPC ID"` + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *aliyun.SRegion, args *SecurityGroupListOptions) error { + secgrps, total, e := cli.GetSecurityGroups(args.VpcId, args.Offset, args.Limit) + if e != nil { + return e + } + printList(secgrps, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type SecurityGroupShowOptions struct { + ID string `help:"ID or name of security group"` + } + R(&SecurityGroupShowOptions{}, "security-group-show", "Show details of a security group", func(cli *aliyun.SRegion, args *SecurityGroupShowOptions) error { + secgrp, err := cli.GetSecurityGroupDetails(args.ID) + if err != nil { + return err + } + printObject(secgrp) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/shell.go b/pkg/util/aliyun/shell/shell.go new file mode 100644 index 0000000000..ee42a74e4c --- /dev/null +++ b/pkg/util/aliyun/shell/shell.go @@ -0,0 +1,14 @@ +package shell + +type CMD struct { + Options interface{} + Command string + Desc string + Callback interface{} +} + +var CommandTable = make([]CMD, 0) + +func R(options interface{}, command string, desc string, callback interface{}) { + CommandTable = append(CommandTable, CMD{options, command, desc, callback}) +} diff --git a/pkg/util/aliyun/shell/task.go b/pkg/util/aliyun/shell/task.go new file mode 100644 index 0000000000..f235f86601 --- /dev/null +++ b/pkg/util/aliyun/shell/task.go @@ -0,0 +1,20 @@ +package shell + +import "yunion.io/yunioncloud/pkg/util/aliyun" + +func init() { + type TaskListOptions struct { + TYPE string `help:"Task types, either ImportImage or ExportImage" choices:"ImportImage|ExportImage"` + Task []string `help:"Task ID"` + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&TaskListOptions{}, "task-list", "List tasks", func(cli *aliyun.SRegion, args *TaskListOptions) error { + tasks, total, err := cli.GetTasks(aliyun.TaskActionType(args.TYPE), args.Task, args.Offset, args.Limit) + if err != nil { + return err + } + printList(tasks, total, args.Offset, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/utils.go b/pkg/util/aliyun/shell/utils.go new file mode 100644 index 0000000000..4f4a856a0c --- /dev/null +++ b/pkg/util/aliyun/shell/utils.go @@ -0,0 +1,35 @@ +package shell + +import ( + "fmt" + "reflect" + "github.com/yunionio/jsonutils" + "yunion.io/yunioncloud/pkg/mcclient/modules" + "yunion.io/yunioncloud/pkg/util/printjson" +) + +func printList(data interface{}, total, offset, limit int, columns []string) { + dataValue := reflect.ValueOf(data) + if dataValue.Kind() != reflect.Slice { + fmt.Println("Invalid list data") + return + } + jsonList := make([]jsonutils.JSONObject, dataValue.Len()) + for i := 0; i < dataValue.Len(); i += 1 { + jsonList[i] = jsonutils.Marshal(dataValue.Index(i).Interface()) + } + if total == 0 { + total = dataValue.Len() + } + list := &modules.ListResult{ + Data: jsonList, + Total: total, + Limit: limit, + Offset: offset, + } + printjson.PrintList(list, columns) +} + +func printObject(obj interface{}) { + printjson.PrintObject(jsonutils.Marshal(obj)) +} diff --git a/pkg/util/aliyun/shell/vpc.go b/pkg/util/aliyun/shell/vpc.go new file mode 100644 index 0000000000..38c2762773 --- /dev/null +++ b/pkg/util/aliyun/shell/vpc.go @@ -0,0 +1,20 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type VpcListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&VpcListOptions{}, "vpc-list", "List vpcs", func(cli *aliyun.SRegion, args *VpcListOptions) error { + vpcs, total, e := cli.GetVpcs(args.Offset, args.Limit) + if e != nil { + return e + } + printList(vpcs, total, args.Offset, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/vrouter.go b/pkg/util/aliyun/shell/vrouter.go new file mode 100644 index 0000000000..d86757bd94 --- /dev/null +++ b/pkg/util/aliyun/shell/vrouter.go @@ -0,0 +1,20 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type VRouterListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&VRouterListOptions{}, "vrouter-list", "List vrouters", func(cli *aliyun.SRegion, args *VRouterListOptions) error { + vrouters, total, e := cli.GetVRouters(args.Offset, args.Limit) + if e != nil { + return e + } + printList(vrouters, total, args.Offset, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/vswitch.go b/pkg/util/aliyun/shell/vswitch.go new file mode 100644 index 0000000000..d16c9fca66 --- /dev/null +++ b/pkg/util/aliyun/shell/vswitch.go @@ -0,0 +1,20 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type VSwitchListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + R(&VSwitchListOptions{}, "vswitch-list", "List vswitches", func(cli *aliyun.SRegion, args *VSwitchListOptions) error { + vswitches, total, e := cli.GetVSwitches(nil, args.Offset, args.Limit) + if e != nil { + return e + } + printList(vswitches, total, args.Offset, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/aliyun/shell/zone.go b/pkg/util/aliyun/shell/zone.go new file mode 100644 index 0000000000..24d670a01b --- /dev/null +++ b/pkg/util/aliyun/shell/zone.go @@ -0,0 +1,25 @@ +package shell + +import ( + "yunion.io/yunioncloud/pkg/util/aliyun" +) + +func init() { + type ZoneListOptions struct { + Details bool `help:"show Details"` + // ChargeType string `help:"charge type" choices:"PrePaid|PostPaid" default:"PrePaid"` + // SpotStrategy string `help:"Spot strategy, NoSpot|SpotWithPriceLimit|SpotAsPriceGo" choices:"NoSpot|SpotWithPriceLimit|SpotAsPriceGo" default:"NoSpot"` + } + R(&ZoneListOptions{}, "zone-list", "List zones", func(cli *aliyun.SRegion, args *ZoneListOptions) error { + zones, e := cli.GetIZones() + if e != nil { + return e + } + cols := []string{"zone_id", "local_name", "available_resource_creation", "available_disk_categories"} + if args.Details { + cols = []string{} + } + printList(zones, 0, 0, 0, cols) + return nil + }) +} diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go new file mode 100644 index 0000000000..2c09fa1331 --- /dev/null +++ b/pkg/util/aliyun/storage.go @@ -0,0 +1,88 @@ +package aliyun + +import ( + "fmt" + "strings" + + "github.com/yunionio/jsonutils" + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type SStorage struct { + zone *SZone + storageType string +} + +func (self *SStorage) GetId() string { + return fmt.Sprintf("%s-%s", self.zone.GetId(), self.storageType) +} + +func (self *SStorage) GetName() string { + return self.GetId() +} + +func (self *SStorage) GetGlobalId() string { + return fmt.Sprintf("%-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) +} + +func (self *SStorage) GetIZone() cloudprovider.ICloudZone { + return self.zone +} + +func (self *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + disks := make([]SDisk, 0) + for { + parts, total, err := self.zone.region.GetDisks("", self.zone.GetId(), self.storageType, len(disks), 50) + if err != nil { + log.Errorf("GetDisks fail %s", err) + return nil, err + } + disks = append(disks, parts...) + if len(disks) >= total { + break + } + } + idisks := make([]cloudprovider.ICloudDisk, len(disks)) + for i := 0; i < len(disks); i += 1 { + disks[i].storage = self + idisks[i] = &disks[i] + } + return idisks, nil +} + +func (self *SStorage) GetStorageType() string { + //return models.STORAGE_PUBLIC_CLOUD + return self.storageType +} + +func (self *SStorage) GetMediumType() string { + if strings.HasSuffix(self.storageType, "_ssd") { + return models.DISK_TYPE_SSD + } else { + return models.DISK_TYPE_ROTATE + } +} + +func (self *SStorage) GetCapacityMB() int { + return 0 // unlimited +} + +func (self *SStorage) GetStorageConf() jsonutils.JSONObject { + conf := jsonutils.NewDict() + return conf +} + +func (self *SStorage) GetStatus() string { + return models.STORAGE_ENABLED +} + +func (self *SStorage) GetEnabled() bool { + return true +} + +func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return self.zone.region.getStoragecache() +} diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go new file mode 100644 index 0000000000..467fe49e04 --- /dev/null +++ b/pkg/util/aliyun/storagecache.go @@ -0,0 +1,145 @@ +package aliyun + +import ( + "fmt" + "strings" + "time" + + "github.com/yunionio/log" + "github.com/yunionio/mcclient" + "github.com/yunionio/mcclient/auth" + "github.com/yunionio/mcclient/modules" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/options" +) + +type SStoragecache struct { + region *SRegion + + iimages []cloudprovider.ICloudImage +} + +func (self *SStoragecache) GetId() string { + return self.region.GetId() +} + +func (self *SStoragecache) GetName() string { + return self.GetId() +} + +func (self *SStoragecache) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) +} + +func (self *SStoragecache) fetchImages() error { + images := make([]SImage, 0) + for { + parts, total, err := self.region.GetImages(ImageStatusType(""), ImageOwnerSelf, nil, "", len(images), 50) + if err != nil { + return err + } + images = append(images, parts...) + if len(images) >= total { + break + } + } + self.iimages = make([]cloudprovider.ICloudImage, len(images)) + for i := 0; i < len(images); i += 1 { + images[i].storageCache = self + self.iimages[i] = &images[i] + } + return nil +} + +func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { + if self.iimages == nil { + err := self.fetchImages() + if err != nil { + return nil, err + } + } + return self.iimages, nil +} + +func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageId string, extId string, isForce bool) (string, error) { + if len(extId) > 0 { + status, _ := self.region.GetImageStatus(extId) + if status == ImageStatusAvailable && !isForce { + return extId, nil + } + } + return self.uploadImage(userCred, imageId, isForce) +} + +func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageId string, isForce bool) (string, error) { + // first upload image to oss + s := auth.GetAdminSession(options.Options.Region, "") + + meta, reader, err := modules.Images.Download(s, imageId) + if err != nil { + return "", err + } + log.Infof("meta data %s", meta) + oss, err := self.region.GetOssClient() + if err != nil { + log.Errorf("GetOssClient err %s", err) + return "", err + } + bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.client.providerId)) + exist, err := oss.IsBucketExist(bucketName) + if err != nil { + log.Errorf("IsBucketExist err %s", err) + return "", err + } + if !exist { + err = oss.CreateBucket(bucketName) + if err != nil { + log.Errorf("Create bucket error %s", err) + return "", err + } + } + bucket, err := oss.Bucket(bucketName) + if err != nil { + log.Errorf("Bucket error %s %s", bucketName, err) + return "", err + } + err = bucket.PutObject(imageId, reader) + if err != nil { + log.Errorf("PutObject error %s %s", imageId, err) + return "", err + } + + imageName := imageId + nameIdx := 1 + + // check image name, avoid name conflict + for { + _, err = self.region.GetImageByName(imageName) + if err != nil { + if err == cloudprovider.ErrNotFound { + break + } else { + return "", err + } + } + imageName = fmt.Sprintf("%s-%d", imageId, nameIdx) + nameIdx += 1 + } + + task, err := self.region.ImportImage(imageName, bucketName, imageId) + + if err != nil { + log.Errorf("ImportImage error %s %s %s", imageId, bucketName, err) + return "", err + } + + // timeout: 1hour = 3600 seconds + err = self.region.WaitTaskStatus(ImportImageTask, task.TaskId, "Finished", 15*time.Second, 3600*time.Second) + if err != nil { + log.Errorf("waitTaskStatus %s", err) + return task.ImageId, err + } + + return task.ImageId, nil +} diff --git a/pkg/util/aliyun/task.go b/pkg/util/aliyun/task.go new file mode 100644 index 0000000000..6ffdbeaca5 --- /dev/null +++ b/pkg/util/aliyun/task.go @@ -0,0 +1,81 @@ +package aliyun + +import ( + "fmt" + "strings" + "time" + + "github.com/yunionio/log" +) + +type TaskActionType string + +const ( + ImportImageTask = TaskActionType("ImportImage") + ExportImageTask = TaskActionType("ExportImage") +) + +type STask struct { + TaskId string + TaskStatus string + TaskAction string + SupportCancel bool + FinishedTime time.Time + CreationTime time.Time +} + +func (self *SRegion) WaitTaskStatus(action TaskActionType, taskId string, targetStatus string, interval time.Duration, timeout time.Duration) error { + start := time.Now() + for time.Now().Sub(start) < timeout { + status, err := self.GetTaskStatus(action, taskId) + if err != nil { + return err + } + if status == targetStatus { + break + } else { + time.Sleep(interval) + } + } + return nil +} + +func (self *SRegion) GetTaskStatus(action TaskActionType, taskId string) (string, error) { + tasks, _, err := self.GetTasks(action, []string{taskId}, 0, 1) + if err != nil { + return "", err + } + return tasks[0].TaskStatus, nil +} + +func (self *SRegion) GetTasks(action TaskActionType, taskId []string, offset int, limit int) ([]STask, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["PageSize"] = fmt.Sprintf("%d", limit) + params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1) + + params["TaskAction"] = string(action) + if taskId != nil && len(taskId) > 0 { + params["TaskIds"] = strings.Join(taskId, ",") + } + + body, err := self.ecsRequest("DescribeTasks", params) + if err != nil { + log.Errorf("GetTasks fail %s", err) + return nil, 0, err + } + + log.Infof("%s", body) + tasks := make([]STask, 0) + err = body.Unmarshal(&tasks, "TaskSet", "Task") + if err != nil { + log.Errorf("Unmarshal task fail %s", err) + return nil, 0, err + } + total, _ := body.Int("TotalCount") + return tasks, int(total), nil +} diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go new file mode 100644 index 0000000000..00bfba043a --- /dev/null +++ b/pkg/util/aliyun/vpc.go @@ -0,0 +1,131 @@ +package aliyun + +import ( + "time" + + "github.com/yunionio/onecloud/pkg/cloudprovider" +) + +// "CidrBlock":"172.31.0.0/16","CreationTime":"2017-03-19T13:37:40Z","Description":"System created default VPC.","IsDefault":true,"RegionId":"cn-hongkong","Status":"Available","UserCidrs":{"UserCidr":[]},"VRouterId":"vrt-j6c00qrol733dg36iq4qj","VSwitchIds":{"VSwitchId":["vsw-j6c3gig5ub4fmi2veyrus"]},"VpcId":"vpc-j6c86z3sh8ufhgsxwme0q","VpcName":"" + +type SUserCIDRs struct { + UserCidr []string +} + +type SVSwitchIds struct { + VSwitchId []string +} + +type SVpc struct { + region *SRegion + + iwires []cloudprovider.ICloudWire + + secgroups []SSecurityGroup + + CidrBlock string + CreationTime time.Time + Description string + IsDefault bool + RegionId string + Status string + UserCidrs SUserCIDRs + VRouterId string + VSwitchIds SVSwitchIds + VpcId string + VpcName string +} + +func (self *SVpc) GetId() string { + return self.VpcId +} + +func (self *SVpc) GetName() string { + return self.VpcId +} + +func (self *SVpc) GetGlobalId() string { + return self.VpcId +} + +func (self *SVpc) GetIsDefault() bool { + return self.IsDefault +} + +func (self *SVpc) GetCidrBlock() string { + return self.CidrBlock +} + +func (self *SVpc) GetStatus() string { + return self.Status +} + +func (self *SVpc) GetRegion() cloudprovider.ICloudRegion { + return self.region +} + +func (self *SVpc) addWire(wire *SWire) { + if self.iwires == nil { + self.iwires = make([]cloudprovider.ICloudWire, 0) + } + self.iwires = append(self.iwires, wire) +} + +func (self *SVpc) getWireByZoneId(zoneId string) *SWire { + for i := 0; i <= len(self.iwires); i += 1 { + wire := self.iwires[i].(*SWire) + if wire.zone.ZoneId == zoneId { + return wire + } + } + return nil +} + +func (self *SVpc) fetchVSwitches() error { + switches, _, err := self.region.GetVSwitches(self.VSwitchIds.VSwitchId, 0, len(self.VSwitchIds.VSwitchId)) + if err != nil { + return err + } + for i := 0; i < len(switches); i += 1 { + wire := self.getWireByZoneId(switches[i].ZoneId) + switches[i].wire = wire + wire.addNetwork(&switches[i]) + } + return nil +} + +func (self *SVpc) GetIWires() ([]cloudprovider.ICloudWire, error) { + if self.iwires == nil { + err := self.fetchVSwitches() + if err != nil { + return nil, err + } + } + return self.iwires, nil +} + +func (self *SVpc) fetchSecurityGroups() error { + secgroups := make([]SSecurityGroup, 0) + for { + parts, total, err := self.region.GetSecurityGroups(self.VpcId, len(secgroups), 50) + if err != nil { + return err + } + secgroups = append(secgroups, parts...) + if len(secgroups) >= total { + break + } + } + self.secgroups = secgroups + return nil +} + +func (self *SVpc) GetSecurityGroups() ([]SSecurityGroup, error) { + if self.secgroups == nil { + err := self.fetchSecurityGroups() + if err != nil { + return nil, err + } + } + return self.secgroups, nil +} diff --git a/pkg/util/aliyun/vrouter.go b/pkg/util/aliyun/vrouter.go new file mode 100644 index 0000000000..e607edfd6a --- /dev/null +++ b/pkg/util/aliyun/vrouter.go @@ -0,0 +1,21 @@ +package aliyun + +import ( + "time" +) + +// "CreationTime":"2017-03-19T13:37:40Z","Description":"","RegionId":"cn-hongkong","RouteTableIds":{"RouteTableId":["vtb-j6c60lectdi80rk5xz43g"]},"VRouterId":"vrt-j6c00qrol733dg36iq4qj","VRouterName":"","VpcId":"vpc-j6c86z3sh8ufhgsxwme0q" + +type SRouteTableIds struct { + RouteTableId []string +} + +type SVRouter struct { + CreationTime time.Time + Description string + RegionId string + RouteTableIds SRouteTableIds + VRouterId string + VRouterName string + VpcId string +} diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go new file mode 100644 index 0000000000..75d3ce41ce --- /dev/null +++ b/pkg/util/aliyun/vswitch.go @@ -0,0 +1,84 @@ +package aliyun + +import ( + "time" + + "github.com/yunionio/pkg/util/netutils" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +// {"AvailableIpAddressCount":4091,"CidrBlock":"172.31.32.0/20","CreationTime":"2017-03-19T13:37:44Z","Description":"System created default virtual switch.","IsDefault":true,"Status":"Available","VSwitchId":"vsw-j6c3gig5ub4fmi2veyrus","VSwitchName":"","VpcId":"vpc-j6c86z3sh8ufhgsxwme0q","ZoneId":"cn-hongkong-b"} + +type SVSwitch struct { + wire *SWire + + AvailableIpAddressCount int + CidrBlock string + CreationTime time.Time + Description string + IsDefault bool + Status string + VSwitchId string + VSwitchName string + VpcId string + ZoneId string +} + +func (self *SVSwitch) GetId() string { + return self.VSwitchId +} + +func (self *SVSwitch) GetName() string { + return self.VSwitchId +} + +func (self *SVSwitch) GetGlobalId() string { + return self.VSwitchId +} + +func (self *SVSwitch) GetStatus() string { + return self.Status +} + +func (self *SVSwitch) GetIWire() cloudprovider.ICloudWire { + return self.wire +} + +func (self *SVSwitch) GetIpStart() string { + pref, _ := netutils.NewIPV4Prefix(self.CidrBlock) + startIp := pref.Address.NetAddr(pref.MaskLen) // 0 + startIp = startIp.StepUp() // 1 + return startIp.String() +} + +func (self *SVSwitch) GetIpEnd() string { + pref, _ := netutils.NewIPV4Prefix(self.CidrBlock) + endIp := pref.Address.BroadcastAddr(pref.MaskLen) // 255 + endIp = endIp.StepDown() // 254 + endIp = endIp.StepDown() // 253 + endIp = endIp.StepDown() // 252 + return endIp.String() +} + +func (self *SVSwitch) GetIpMask() int8 { + pref, _ := netutils.NewIPV4Prefix(self.CidrBlock) + return pref.MaskLen +} + +func (self *SVSwitch) GetGateway() string { + pref, _ := netutils.NewIPV4Prefix(self.CidrBlock) + endIp := pref.Address.BroadcastAddr(pref.MaskLen) // 255 + endIp = endIp.StepDown() // 254 + return endIp.String() +} + +func (self *SVSwitch) GetServerType() string { + return models.SERVER_TYPE_GUEST +} + +func (self *SVSwitch) GetIsPublic() bool { + // return self.IsDefault + return true +} diff --git a/pkg/util/aliyun/wire.go b/pkg/util/aliyun/wire.go new file mode 100644 index 0000000000..09a3f96b77 --- /dev/null +++ b/pkg/util/aliyun/wire.go @@ -0,0 +1,82 @@ +package aliyun + +import ( + "fmt" + + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" +) + +type SWire struct { + zone *SZone + vpc *SVpc + + inetworks []cloudprovider.ICloudNetwork +} + +func (self *SWire) GetId() string { + return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.zone.GetId()) +} + +func (self *SWire) GetName() string { + return self.GetId() +} + +func (self *SWire) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.vpc.GetGlobalId(), self.zone.GetGlobalId()) +} + +func (self *SWire) GetIVpc() cloudprovider.ICloudVpc { + return self.vpc +} + +func (self *SWire) GetIZone() cloudprovider.ICloudZone { + return self.zone +} + +func (self *SWire) addNetwork(vswitch *SVSwitch) { + if self.inetworks == nil { + self.inetworks = make([]cloudprovider.ICloudNetwork, 0) + } + find := false + for i := 0; i < len(self.inetworks); i += 1 { + if self.inetworks[i].GetId() == vswitch.VSwitchId { + find = true + break + } + } + if !find { + self.inetworks = append(self.inetworks, vswitch) + } +} + +func (self *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { + if self.inetworks == nil { + err := self.vpc.fetchVSwitches() + if err != nil { + return nil, err + } + } + return self.inetworks, nil +} + +func (self *SWire) getNetworkById(vswitchId string) *SVSwitch { + networks, err := self.GetINetworks() + if err != nil { + return nil + } + log.Debugf("search for networks %d", len(networks)) + for i := 0; i < len(networks); i += 1 { + log.Debugf("search %s", networks[i].GetName()) + network := networks[i].(*SVSwitch) + if network.VSwitchId == vswitchId { + return network + } + } + return nil +} + +func (self *SWire) GetBandwidth() int { + return 10000 +} diff --git a/pkg/util/aliyun/zone.go b/pkg/util/aliyun/zone.go new file mode 100644 index 0000000000..d0365f44e4 --- /dev/null +++ b/pkg/util/aliyun/zone.go @@ -0,0 +1,223 @@ +package aliyun + +import ( + "fmt" + + "github.com/yunionio/log" + + "github.com/yunionio/onecloud/pkg/cloudprovider" + "github.com/yunionio/onecloud/pkg/compute/models" +) + +type InstanceChargeType string + +const ( + PrePaidInstanceChargeType InstanceChargeType = "PrePaid" + PostPaidInstanceChargeType InstanceChargeType = "PostPaid" + DefaultInstanceChargeType = PostPaidInstanceChargeType +) + +type SpotStrategyType string + +const ( + NoSpotStrategy SpotStrategyType = "NoSpot" + SpotWithPriceLimitStrategy SpotStrategyType = "SpotWithPriceLimit" + SpotAsPriceGoStrategy SpotStrategyType = "SpotAsPriceGo" + DefaultSpotStrategy = NoSpotStrategy +) + +type SDedicatedHostGenerations struct { + DedicatedHostGeneration []string +} + +type SVolumeCategories struct { + VolumeCategories []string +} + +type SSupportedDataDiskCategories struct { + SupportedDataDiskCategory []string +} + +type SSupportedInstanceGenerations struct { + SupportedInstanceGeneration []string +} + +type SSupportedInstanceTypeFamilies struct { + SupportedInstanceTypeFamily []string +} + +type SSupportedInstanceTypes struct { + SupportedInstanceType []string +} + +type SSupportedNetworkTypes struct { + SupportedNetworkCategory []string +} + +type SSupportedSystemDiskCategories struct { + SupportedSystemDiskCategory []string +} + +type SResourcesInfo struct { + DataDiskCategories SSupportedDataDiskCategories + InstanceGenerations SSupportedInstanceGenerations + InstanceTypeFamilies SSupportedInstanceTypeFamilies + InstanceTypes SSupportedInstanceTypes + IoOptimized bool + NetworkTypes SSupportedNetworkTypes + SystemDiskCategories SSupportedSystemDiskCategories +} + +type SResources struct { + ResourcesInfo []SResourcesInfo +} + +type SResourceCreation struct { + ResourceTypes []string +} + +type SInstanceTypes struct { + InstanceTypes []string +} + +type SDiskCategories struct { + DiskCategories []string +} + +type SDedicatedHostTypes struct { + DedicatedHostType []string +} + +type SZone struct { + region *SRegion + + iwires []cloudprovider.ICloudWire + + host *SHost + + istorages []cloudprovider.ICloudStorage + + ZoneId string + LocalName string + DedicatedHostGenerations SDedicatedHostGenerations + AvailableVolumeCategories SVolumeCategories + /* 可供创建的具体资源，AvailableResourcesType 组成的数组 */ + AvailableResources SResources + /* 允许创建的资源类型集合 */ + AvailableResourceCreation SResourceCreation + /* 允许创建的实例规格类型 */ + AvailableInstanceTypes SInstanceTypes + /* 支持的磁盘种类集合 */ + AvailableDiskCategories SDiskCategories + AvailableDedicatedHostTypes SDedicatedHostTypes +} + +func (self *SZone) GetId() string { + return self.ZoneId +} + +func (self *SZone) GetName() string { + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_ALIYUN_CN, self.LocalName) +} + +func (self *SZone) GetGlobalId() string { + return fmt.Sprintf("%s/%s", self.region.GetGlobalId(), self.ZoneId) +} + +func (self *SZone) GetStatus() string { + if len(self.AvailableResourceCreation.ResourceTypes) == 0 { + return models.ZONE_SOLDOUT + } else { + return models.ZONE_ENABLE + } +} + +func (self *SZone) GetIRegion() cloudprovider.ICloudRegion { + return self.region +} + +func (self *SZone) fetchStorages() error { + self.istorages = make([]cloudprovider.ICloudStorage, len(self.AvailableDiskCategories.DiskCategories)) + + for i, sc := range self.AvailableDiskCategories.DiskCategories { + storage := SStorage{zone: self, storageType: sc} + self.istorages[i] = &storage + } + return nil +} + +func (self *SZone) getStorageByCategory(category string) (*SStorage, error) { + storages, err := self.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(storages); i += 1 { + storage := storages[i].(*SStorage) + if storage.storageType == category { + return storage, nil + } + } + return nil, fmt.Errorf("No such storage %s", category) +} + +func (self *SZone) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + if self.istorages == nil { + self.fetchStorages() + } + return self.istorages, nil +} + +func (self *SZone) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + if self.istorages == nil { + self.fetchStorages() + } + for i := 0; i < len(self.istorages); i += 1 { + if self.istorages[i].GetGlobalId() == id { + return self.istorages[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SZone) getHost() *SHost { + if self.host == nil { + self.host = &SHost{zone: self} + } + return self.host +} + +func (self *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { + return []cloudprovider.ICloudHost{self.getHost()}, nil +} + +func (self *SZone) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + host := self.getHost() + if host.GetGlobalId() == id { + return host, nil + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SZone) addWire(wire *SWire) { + if self.iwires == nil { + self.iwires = make([]cloudprovider.ICloudWire, 0) + } + self.iwires = append(self.iwires, wire) +} + +func (self *SZone) GetIWires() ([]cloudprovider.ICloudWire, error) { + return self.iwires, nil +} + +func (self *SZone) getNetworkById(vswitchId string) *SVSwitch { + log.Debugf("Search in wires %d", len(self.iwires)) + for i := 0; i < len(self.iwires); i += 1 { + log.Debugf("Search in wire %s", self.iwires[i].GetName()) + wire := self.iwires[i].(*SWire) + net := wire.getNetworkById(vswitchId) + if net != nil { + return net + } + } + return nil +}