diff --git a/cmd/torrent/main.go b/cmd/torrent/main.go index a31539e76a..af4d462e43 100644 --- a/cmd/torrent/main.go +++ b/cmd/torrent/main.go @@ -1,21 +1,25 @@ package main import ( + "crypto/sha1" "fmt" + "io/ioutil" + "net/http" "os" "os/signal" - "path" "path/filepath" "syscall" "time" "github.com/anacrolix/torrent" "github.com/anacrolix/torrent/metainfo" + "github.com/anacrolix/torrent/storage" "yunion.io/x/log" "yunion.io/x/pkg/util/version" "yunion.io/x/structarg" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/nodeid" "yunion.io/x/onecloud/pkg/util/torrentutils" ) @@ -28,7 +32,10 @@ type Options struct { Tracker []string `help:"Tracker urls, e.g. http://10.168.222.252:6969/announce or udp://tracker.istole.it:6969"` - Debug bool `help:"turn on debug"` + Debug bool `help:"turn on debug" default:"false"` + Verbose bool `help:"verbose mode" default:"false"` + + CallbackURL string `help:"callback notification URL"` } func exitSignalHandlers(client *torrent.Client) { @@ -71,13 +78,15 @@ func main() { } var mi *metainfo.MetaInfo + var rootDir string - if len(options.Tracker) > 0 { + if len(options.Tracker) > 0 && !fileutils2.Exists(options.TORRENT) { // server mode mi, err = torrentutils.GenerateTorrent(root, options.Tracker, options.TORRENT) if err != nil { log.Fatalf("fail to save torrent file %s", err) } + rootDir = filepath.Dir(root) } else { // client mode, load mi from torrent file @@ -85,28 +94,49 @@ func main() { if err != nil { log.Fatalf("fail to open torrent file %s", err) } + rootDir = root } + info, err := mi.UnmarshalInfo() + if err != nil { + log.Errorf("fail to unmarshalinfo %s", err) + return + } + + hasher := sha1.New() + nodeId, err := nodeid.GetNodeId() if err != nil { log.Errorf("fail to generate node id: %s", err) return } - log.Infof("Set torrent server as node %s", nodeId) + hasher.Write(nodeId) + hasher.Write(info.Pieces) + + peerIdStr := fmt.Sprintf("%x", hasher.Sum(nil)) + + log.Infof("Set torrent server as node %s", peerIdStr[:20]) clientConfig := torrent.NewDefaultClientConfig() - clientConfig.PeerID = nodeId[:20] + clientConfig.PeerID = peerIdStr[:20] clientConfig.Debug = options.Debug clientConfig.Seed = true clientConfig.NoUpload = false - if len(options.Tracker) > 0 { - // server mode - clientConfig.DataDir = path.Dir(root) - } else { - // client mode - clientConfig.DataDir = root - } + + log.Infof("To sync torrent files for %s", info.Name) + tmpDir := filepath.Join(rootDir, fmt.Sprintf("%s%s", info.Name, ".tmp")) + + os.RemoveAll(tmpDir) + os.MkdirAll(tmpDir, 0700) + defer os.RemoveAll(tmpDir) + + clientConfig.DefaultStorage = storage.NewFileWithCustomPathMaker(tmpDir, + func(baseDir string, info *metainfo.Info, infoHash metainfo.Hash) string { + return filepath.Dir(baseDir) + }, + ) + clientConfig.DisableTrackers = false clientConfig.DisablePEX = true clientConfig.NoDHT = true @@ -122,32 +152,50 @@ func main() { go exitSignalHandlers(client) + start := time.Now() + t, err := client.AddTorrent(mi) if err != nil { log.Fatalf("%s", err) } - go func() { - <-t.GotInfo() + <-t.GotInfo() + t.DownloadAll() - files := t.Info().Files - log.Debugf("Got Info, start download %d files", len(files)) - for i := 0; i < len(files); i += 1 { - log.Debugf("%d: %s", i, files[i].Path) - } - - t.DownloadAll() - }() + stop := false go func() { <-client.Closed() log.Debugf("client closed, exit!") - os.Exit(0) + stop = true }() - for { + finish := false + + for !stop { if t.BytesCompleted() == t.Info().TotalLength() { + if !finish { + finish = true + fmt.Printf("Download complete, takes %d seconds\n", time.Now().Sub(start)/time.Second) + if len(options.CallbackURL) > 0 { + maxTried := 10 + for tried := 0; tried < maxTried; tried += 1 { + resp, err := http.Post(options.CallbackURL, "", nil) + if err == nil && resp.StatusCode < 300 { + break + } + if err != nil { + log.Errorf("callback fail %s", err) + } else { + defer resp.Body.Close() + respBody, _ := ioutil.ReadAll(resp.Body) + log.Errorf("callback response error %s", string(respBody)) + } + time.Sleep(time.Duration(tried+1) * 10 * time.Second) + } + } + } fmt.Printf("\rSeeding.............") } else { fmt.Printf("\rDownload: %.1f%%", float64(t.BytesCompleted())*100.0/float64(t.Info().TotalLength())) diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index a35fc82ea7..3b24dcb823 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -5,13 +5,16 @@ import ( "fmt" "math/rand" "net/http" + "os" "strings" "sync" + "syscall" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/trace" + "yunion.io/x/pkg/util/signalutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" @@ -37,8 +40,8 @@ type Application struct { cors *Cors middlewares []MiddlewareFunc - // record Http server for handle shotdown - server *http.Server + isExiting bool + idleConnsClosed chan struct{} } const ( @@ -329,32 +332,73 @@ func (app *Application) initServer(addr string) *http.Server { return s } +func (app *Application) registerCleanShutdown(s *http.Server, onStop func()) { + app.idleConnsClosed = make(chan struct{}) + + // dump goroutine stack + signalutils.RegisterSignal(func() { + utils.DumpAllGoroutineStack(log.Logger().Out) + }, syscall.SIGUSR1) + + quitSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM} + signalutils.RegisterSignal(func() { + if app.isExiting { + log.Infof("Quit signal received!!! clean up in progress, be patient...") + return + } + app.isExiting = true + log.Infof("Quit signal received!!! do cleanup...") + + if err := s.Shutdown(context.Background()); err != nil { + // Error from closing listeners, or context timeout: + log.Errorf("HTTP server Shutdown: %v", err) + } + if onStop != nil { + func() { + defer func() { + if r := recover(); r != nil { + log.Errorf("app exiting error: %s", r) + } + }() + onStop() + }() + } + close(app.idleConnsClosed) + }, quitSignals...) + + signalutils.StartTrap() +} + +func (app *Application) waitCleanShutdown() { + <-app.idleConnsClosed + log.Infof("Service stopped.") +} + func (app *Application) ListenAndServe(addr string) { - app.server = app.initServer(addr) - err := app.server.ListenAndServe() - if err != nil { - log.Errorf("ListAndServer: %s", err) - panic(err) - } -} - -func (app *Application) IsInServe() bool { - return app.server != nil -} - -func (app *Application) ShutDown(ctx context.Context) error { - if app.server != nil { - return app.server.Shutdown(ctx) - } - return fmt.Errorf("Not init http server ??") + app.ListenAndServeWithCleanup(addr, nil) } func (app *Application) ListenAndServeTLS(addr string, certFile, keyFile string) { + app.ListenAndServeTLSWithCleanup(addr, certFile, keyFile, nil) +} + +func (app *Application) ListenAndServeWithCleanup(addr string, onStop func()) { + app.ListenAndServeTLSWithCleanup(addr, "", "", onStop) +} + +func (app *Application) ListenAndServeTLSWithCleanup(addr string, certFile, keyFile string, onStop func()) { s := app.initServer(addr) - err := s.ListenAndServeTLS(certFile, keyFile) + app.registerCleanShutdown(s, onStop) + var err error + if len(certFile) == 0 && len(keyFile) == 0 { + err = s.ListenAndServe() + } else { + err = s.ListenAndServeTLS(certFile, keyFile) + } if err != nil && err != http.ErrServerClosed { log.Fatalf("ListAndServer fail: %s", err) } + app.waitCleanShutdown() } func isJsonContentType(r *http.Request) bool { diff --git a/pkg/baremetal/service/service.go b/pkg/baremetal/service/service.go index 13bac7f266..a0596ed205 100644 --- a/pkg/baremetal/service/service.go +++ b/pkg/baremetal/service/service.go @@ -1,7 +1,6 @@ package service import ( - "context" "os" "yunion.io/x/log" @@ -16,7 +15,6 @@ import ( type BaremetalService struct { service.SServiceBase - isExiting bool } func New() *BaremetalService { @@ -30,25 +28,9 @@ func (s *BaremetalService) StartService() { app := cloudcommon.InitApp(&o.Options.CommonOptions, false) handler.InitHandlers(app) - s.RegisterSIGUSR1() - s.RegisterQuitSignals(func() { - log.Infof("Baremetal agent quit !!!") - if s.isExiting { - return - } else { - s.isExiting = true - } - - if app.IsInServe() { - if err := app.ShutDown(context.Background()); err != nil { - log.Errorf("App shutdown err: %v", err) - } - } + cloudcommon.ServeForeverWithCleanup(app, &o.Options.CommonOptions, func() { tasks.OnStop() - os.Exit(0) }) - - cloudcommon.ServeForever(app, &o.Options.CommonOptions) } func (s *BaremetalService) startAgent() { diff --git a/pkg/cloudcommon/app.go b/pkg/cloudcommon/app.go index 5ed94b973a..6463dfbbea 100644 --- a/pkg/cloudcommon/app.go +++ b/pkg/cloudcommon/app.go @@ -24,6 +24,10 @@ func InitApp(options *CommonOptions, dbAccess bool) *appsrv.Application { } func ServeForever(app *appsrv.Application, options *CommonOptions) { + ServeForeverWithCleanup(app, options, nil) +} + +func ServeForeverWithCleanup(app *appsrv.Application, options *CommonOptions, onStop func()) { AppDBInit(app) addr := net.JoinHostPort(options.Address, strconv.Itoa(options.Port)) proto := "http" @@ -47,8 +51,8 @@ func ServeForever(app *appsrv.Application, options *CommonOptions) { if len(options.SslKeyfile) == 0 { log.Fatalf("Missing ssl-keyfile") } - app.ListenAndServeTLS(addr, certfile, options.SslKeyfile) + app.ListenAndServeTLSWithCleanup(addr, certfile, options.SslKeyfile, onStop) } else { - app.ListenAndServe(addr) + app.ListenAndServeWithCleanup(addr, onStop) } } diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index fd8270a38c..f9fa6cc936 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -148,7 +148,9 @@ func (self *SCronJobManager) Start() { } func (self *SCronJobManager) Stop() { - close(self.stop) + if self.stop != nil { + close(self.stop) + } } func (self *SCronJobManager) run() { diff --git a/pkg/cloudcommon/policy/defaults.go b/pkg/cloudcommon/policy/defaults.go index d6eaa528a9..85378b27b2 100644 --- a/pkg/cloudcommon/policy/defaults.go +++ b/pkg/cloudcommon/policy/defaults.go @@ -137,6 +137,13 @@ var ( Action: PolicyActionGet, Result: rbacutils.OwnerAllow, }, + { + Service: "image", + Resource: "images", + Action: PolicyActionPerform, + Extra: []string{"update-torrent-status"}, + Result: rbacutils.GuestAllow, + }, { Service: "log", Resource: "actions", diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go index a9df96880f..13c5aa4c7d 100644 --- a/pkg/cloudcommon/policy/policy.go +++ b/pkg/cloudcommon/policy/policy.go @@ -190,7 +190,7 @@ func queryKey(isAdmin bool, userCred mcclient.TokenCredential, service string, r } func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.TRbacResult { - if manager.cache != nil { + if manager.cache != nil && userCred != nil { key := queryKey(isAdmin, userCred, service, resource, action, extra...) val := manager.cache.Get(key) if val != nil { @@ -231,7 +231,12 @@ func (manager *SPolicyManager) allowWithoutCache(isAdmin bool, userCred mcclient log.Warningf("no policies fetched") return rbacutils.Deny } - userCredJson := userCred.ToJson() + var userCredJson jsonutils.JSONObject + if userCred != nil { + userCredJson = userCred.ToJson() + } else { + userCredJson = jsonutils.NewDict() + } currentPriv := rbacutils.Deny for _, p := range policies { result := p.Allow(userCredJson, service, resource, action, extra...) diff --git a/pkg/cloudcommon/service/services.go b/pkg/cloudcommon/service/services.go index b6d7cc66a1..e6d29208d6 100644 --- a/pkg/cloudcommon/service/services.go +++ b/pkg/cloudcommon/service/services.go @@ -1,26 +1,3 @@ package service -import ( - "os" - "syscall" - - "yunion.io/x/log" - "yunion.io/x/pkg/util/signalutils" - "yunion.io/x/pkg/utils" -) - type SServiceBase struct{} - -func (s *SServiceBase) RegisterQuitSignals(quitHandler signalutils.Trap) { - quitSignals := []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM} - signalutils.RegisterSignal(quitHandler, quitSignals...) - - signalutils.StartTrap() -} - -func (s *SServiceBase) RegisterSIGUSR1() { - // dump goroutine stack - signalutils.RegisterSignal(func() { - utils.DumpAllGoroutineStack(log.Logger().Out) - }, syscall.SIGUSR1) -} diff --git a/pkg/cloudir/service/service.go b/pkg/cloudir/service/service.go index 9a3c2799a7..ff07cc1334 100644 --- a/pkg/cloudir/service/service.go +++ b/pkg/cloudir/service/service.go @@ -23,11 +23,12 @@ func StartService() { log.Fatalf("init etcd fail: %s", err) return } - defer etcd.CloseDefaultEtcdClient() app := cloudcommon.InitApp(commonOpts, false) initHandlers(app) - cloudcommon.ServeForever(app, commonOpts) + cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + etcd.CloseDefaultEtcdClient() + }) } diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 3e160979fd..889cdca44e 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -33,6 +33,7 @@ type ICloudProviderFactory interface { IsPublicCloud() bool IsOnPremise() bool IsSupportPrepaidResources() bool + NeedSyncSkuFromCloud() bool } type ICloudProvider interface { diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go index fc85e3481e..c18523f6a6 100644 --- a/pkg/cloudprovider/fakeregion.go +++ b/pkg/cloudprovider/fakeregion.go @@ -115,3 +115,7 @@ func (region *SFakeOnPremiseRegion) CreateILoadBalancer(loadbalancer *SLoadbalan func (region *SFakeOnPremiseRegion) CreateILoadBalancerAcl(acl *SLoadbalancerAccessControlList) (ICloudLoadbalancerAcl, error) { return nil, ErrNotSupported } + +func (region *SFakeOnPremiseRegion) GetSkus(zoneId string) ([]ICloudSku, error) { + return nil, ErrNotSupported +} diff --git a/pkg/cloudprovider/instance.go b/pkg/cloudprovider/instance.go index 331ba0224f..61929f17b0 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/cloudprovider/instance.go @@ -11,6 +11,7 @@ import ( type SDiskInfo struct { StorageType string SizeGB int + Name string } type SManagedVMCreateConfig struct { diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 8b4d9b6544..eaa76d6a24 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -74,6 +74,8 @@ type ICloudRegion interface { CreateILoadBalancerAcl(acl *SLoadbalancerAccessControlList) (ICloudLoadbalancerAcl, error) CreateILoadBalancerCertificate(cert *SLoadbalancerCertificate) (ICloudLoadbalancerCertificate, error) + GetSkus(zoneId string) ([]ICloudSku, error) + GetProvider() string } @@ -211,7 +213,7 @@ type ICloudVM interface { GetInstanceType() string AssignSecurityGroup(secgroupId string) error - AssignSecurityGroups(secgroupIds []string) error + SetSecurityGroups(secgroupIds []string) error GetHypervisor() string @@ -522,3 +524,38 @@ type ICloudLoadbalancerAcl interface { Sync(acl *SLoadbalancerAccessControlList) error Delete() error } + +type ICloudSku interface { + ICloudResource + + GetInstanceTypeFamily() string + GetInstanceTypeCategory() string + + GetPrepaidStatus() string + GetPostpaidStatus() string + + GetCpuCoreCount() int + GetMemorySizeMB() int + + GetOsName() string + + GetSysDiskResizable() bool + GetSysDiskType() string + GetSysDiskMinSizeGB() int + GetSysDiskMaxSizeGB() int + + GetAttachedDiskType() string + GetAttachedDiskSizeGB() int + GetAttachedDiskCount() int + + GetDataDiskTypes() string + GetDataDiskMaxCount() int + + GetNicType() string + GetNicMaxCount() int + + GetGpuAttachable() bool + GetGpuSpec() string + GetGpuCount() int + GetGpuMaxCount() int +} diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 8372a907ef..cd7fa10e32 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -162,6 +162,8 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu if createErr != nil { return nil, createErr } + guest.SetExternalId(iVM.GetGlobalId()) + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) if err != nil { diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index 9854e065be..9de620a4d7 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -133,6 +133,8 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest return nil, createErr } + guest.SetExternalId(iVM.GetGlobalId()) + log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800) if err != nil { diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 72d1e4cec4..b8e85dbdac 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -154,11 +154,12 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue } iVM, createErr := ihost.CreateVM(&desc) - if createErr != nil { return nil, createErr } + guest.SetExternalId(iVM.GetGlobalId()) + log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) if err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800); err != nil { return nil, err diff --git a/pkg/compute/guestdrivers/huawei.go b/pkg/compute/guestdrivers/huawei.go index 65d532781c..76c040417f 100644 --- a/pkg/compute/guestdrivers/huawei.go +++ b/pkg/compute/guestdrivers/huawei.go @@ -123,6 +123,8 @@ func (self *SHuaweiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu if createErr != nil { return nil, createErr } + guest.SetExternalId(iVM.GetGlobalId()) + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800) if err != nil { diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index d096e21f4e..1e716a50a7 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -49,6 +49,7 @@ func (self *SManagedVirtualizedGuestDriver) GetJsonDescAtHost(ctx context.Contex disk := disks[i].GetDisk() storage := disk.GetStorage() if i == 0 { + config.SysDisk.Name = disk.Name config.SysDisk.StorageType = storage.StorageType config.SysDisk.SizeGB = disk.DiskSize / 1024 cache := storage.GetStoragecache() @@ -64,6 +65,7 @@ func (self *SManagedVirtualizedGuestDriver) GetJsonDescAtHost(ctx context.Contex dataDisk := cloudprovider.SDiskInfo{ SizeGB: disk.DiskSize / 1024, StorageType: storage.StorageType, + Name: disk.Name, } config.DataDisks = append(config.DataDisks, dataDisk) } @@ -467,7 +469,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncConfigOnHost(ctx context. } externalIds = append(externalIds, extID) } - return nil, iVM.AssignSecurityGroups(externalIds) + return nil, iVM.SetSecurityGroups(externalIds) } iDisks, err := iVM.GetIDisks() diff --git a/pkg/compute/guestdrivers/openstack.go b/pkg/compute/guestdrivers/openstack.go index fd3b75e4dc..b22ad00eb1 100644 --- a/pkg/compute/guestdrivers/openstack.go +++ b/pkg/compute/guestdrivers/openstack.go @@ -1,8 +1,21 @@ package guestdrivers import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/pkg/util/compare" ) type SOpenStackGuestDriver struct { @@ -29,3 +42,282 @@ func (self *SOpenStackGuestDriver) GetDefaultSysDiskBackend() string { func (self *SOpenStackGuestDriver) GetMinimalSysDiskSizeGb() int { return options.Options.DefaultDiskSizeMB / 1024 } + +func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string) *models.SStorage { + storages := host.GetAttachedStorages("") + for i := 0; i < len(storages); i++ { + if storages[i].StorageType == backend { + return &storages[i] + } + } + for _, stype := range []string{models.STORAGE_OPENSTACK_ISCSI} { + for i := 0; i < len(storages); i++ { + if storages[i].StorageType == stype { + return &storages[i] + } + } + } + return nil +} + +func (self *SOpenStackGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING}, nil +} + +func (self *SOpenStackGuestDriver) GetAttachDiskStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING}, nil +} + +func (self *SOpenStackGuestDriver) GetRebuildRootStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING, models.VM_REBUILD_ROOT_FAIL}, nil +} + +func (self *SOpenStackGuestDriver) GetChangeConfigStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING}, nil +} + +func (self *SOpenStackGuestDriver) GetDeployStatus() ([]string, error) { + return []string{models.VM_RUNNING}, nil +} + +func (self *SOpenStackGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + data, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, data) + if err != nil { + return nil, err + } + if data.Contains("net.0") && data.Contains("net.1") { + return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") + } + return data, nil +} + +func (self *SOpenStackGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + config, err := guest.GetDeployConfigOnHost(ctx, task.GetUserCred(), host, task.GetParams()) + if err != nil { + log.Errorf("GetDeployConfigOnHost error: %v", err) + return err + } + log.Debugf("RequestDeployGuestOnHost: %s", config) + + desc := cloudprovider.SManagedVMCreateConfig{} + if err := desc.GetConfig(config); err != nil { + return err + } + + action, err := config.GetString("action") + if err != nil { + return err + } + + ihost, err := host.GetIHost() + if err != nil { + return err + } + + if action == "create" { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + iVM, createErr := ihost.CreateVM(&desc) + if createErr != nil { + return nil, createErr + } + + // 避免部署失败后，不能删除openstack平台机器 + guest.SetExternalId(iVM.GetGlobalId()) + + log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800) + if err != nil { + return nil, err + } + log.Debugf("VMcreated %s, and status is running", iVM.GetGlobalId()) + + iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) + if err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + err = cloudprovider.RetryUntil(func() (bool, error) { + idisks, err := iVM.GetIDisks() + if err != nil { + log.Errorf("cannot find vm disks %s", err) + return false, err + } + if len(idisks) == len(desc.DataDisks)+1 { + return true, nil + } else { + return false, nil + } + }, 10) + if err != nil { + return nil, err + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", desc.Password, action) + return data, nil + }) + } else if action == "deploy" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") + } + + params := task.GetParams() + log.Debugf("Deploy VM params %s", params.String()) + + deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + err := iVM.DeployVM(ctx, desc.Name, desc.Password, desc.PublicKey, deleteKeypair, desc.Description) + if err != nil { + return nil, err + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", desc.Password, action) + return data, nil + }) + } else if action == "rebuild" { + + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB) + if err != nil { + return nil, err + } + + log.Debugf("VMrebuildRoot %s new diskID %s, wait status ready ...", iVM.GetGlobalId(), diskId) + + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return nil, err + } + log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId()) + + maxWaitSecs := 300 + waited := 0 + + for { + // hack, wait disk number consistent + idisks, err := iVM.GetIDisks() + if err != nil { + log.Errorf("fail to find VM idisks %s", err) + return nil, err + } + if len(idisks) < len(desc.DataDisks)+1 { + if waited > maxWaitSecs { + log.Errorf("inconsistent disk number, wait timeout, must be something wrong on remote") + return nil, cloudprovider.ErrTimeout + } + log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1) + time.Sleep(time.Second * 5) + waited += 5 + } else { + if idisks[0].GetGlobalId() != diskId { + log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId) + return nil, fmt.Errorf("inconsistent sys disk id after rebuild root") + } + + break + } + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", desc.Password, action) + + return data, nil + }) + + } else { + log.Errorf("RequestDeployGuestOnHost: Action %s not supported", action) + return fmt.Errorf("Action %s not supported", action) + } + + return nil +} + +func (self *SOpenStackGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + ihost, err := host.GetIHost() + if err != nil { + return nil, err + } + iVM, err := ihost.GetIVMById(guest.ExternalId) + if err != nil { + return nil, err + } + + if fwOnly, _ := task.GetParams().Bool("fw_only"); fwOnly { + iregion, err := host.GetIRegion() + if err != nil { + return nil, err + } + secgroups := guest.GetSecgroups() + externalIds := []string{} + for _, secgroup := range secgroups { + + lockman.LockRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-normal", guest.SecgrpId)) + defer lockman.ReleaseRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-normal", guest.SecgrpId)) + + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), secgroup.Id, "normal", host.GetRegion().Id, host.ManagerId) + if secgroupCache == nil { + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s", secgroup.Id) + } + extID, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, "", secgroup.Name, secgroup.Description, secgroup.GetSecRules("")) + if err != nil { + return nil, err + } + if err = secgroupCache.SetExternalId(extID); err != nil { + return nil, err + } + externalIds = append(externalIds, extID) + } + return nil, iVM.SetSecurityGroups(externalIds) + } + + iDisks, err := iVM.GetIDisks() + if err != nil { + return nil, err + } + disks := make([]models.SDisk, 0) + for _, guestdisk := range guest.GetDisks() { + disk := guestdisk.GetDisk() + disks = append(disks, *disk) + } + + added := make([]models.SDisk, 0) + commondb := make([]models.SDisk, 0) + commonext := make([]cloudprovider.ICloudDisk, 0) + removed := make([]cloudprovider.ICloudDisk, 0) + + if err := compare.CompareSets(disks, iDisks, &added, &commondb, &commonext, &removed); err != nil { + return nil, err + } + for _, disk := range removed { + if err := iVM.DetachDisk(ctx, disk.GetId()); err != nil { + return nil, err + } + } + for _, disk := range added { + if err := iVM.AttachDisk(ctx, disk.ExternalId); err != nil { + return nil, err + } + } + return nil, nil + }) + return nil +} + +func (self *SOpenStackGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SOpenStackGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool { + return false +} diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index 93f92215b3..e07af3536c 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -181,6 +181,7 @@ func (self *SQcloudGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu if createErr != nil { return nil, createErr } + guest.SetExternalId(iVM.GetGlobalId()) log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800) @@ -350,7 +351,7 @@ func (self *SQcloudGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue } externalIds = append(externalIds, extID) } - return nil, iVM.AssignSecurityGroups(externalIds) + return nil, iVM.SetSecurityGroups(externalIds) } iDisks, err := iVM.GetIDisks() diff --git a/pkg/compute/hostdrivers/openstack.go b/pkg/compute/hostdrivers/openstack.go index 973e4a6cfd..aa8cd59169 100644 --- a/pkg/compute/hostdrivers/openstack.go +++ b/pkg/compute/hostdrivers/openstack.go @@ -16,3 +16,11 @@ func init() { func (self *SOpenStackHostDriver) GetHostType() string { return models.HOST_TYPE_OPENSTACK } + +func (self *SOpenStackHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error { + return nil +} + +func (driver *SOpenStackHostDriver) GetStoragecacheQuota(host *models.SHost) int { + return 100 +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0ad48466bd..8952e8775e 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -161,6 +161,7 @@ var PUBLIC_CLOUD_HYPERVISORS = []string{ HYPERVISOR_AZURE, HYPERVISOR_QCLOUD, HYPERVISOR_HUAWEI, + HYPERVISOR_OPENSTACK, } // var HYPERVISORS = []string{HYPERVISOR_ALIYUN} @@ -2909,9 +2910,9 @@ func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, userCred mcclient registerVpcId := vpc.ExternalId externalVpcId := vpc.ExternalId switch self.Hypervisor { - case HYPERVISOR_ALIYUN, HYPERVISOR_AWS, HYPERVISOR_OPENSTACK, HYPERVISOR_HUAWEI: + case HYPERVISOR_ALIYUN, HYPERVISOR_AWS, HYPERVISOR_HUAWEI: break - case HYPERVISOR_QCLOUD: + case HYPERVISOR_QCLOUD, HYPERVISOR_OPENSTACK: registerVpcId = "normal" case HYPERVISOR_AZURE: registerVpcId, externalVpcId = "normal", "normal" diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index f6242064c0..5d8554bfe3 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -9,9 +9,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -644,6 +646,15 @@ func (manager *SServerSkuManager) GetSkuCountByRegion(regionId string) int { return q.Count() } +func (manager *SServerSkuManager) GetSkuCountByZone(zoneId string) []SServerSku { + skus := []SServerSku{} + q := manager.Query().Equals("zone_id", zoneId) + if err := db.FetchModelObjects(manager, q, &skus); err != nil { + log.Errorf("failed to get skus by zoneId %s error: %v", zoneId, err) + } + return skus +} + // 删除表中zone not found的记录 func (manager *SServerSkuManager) PendingDeleteInvalidSku() error { sq := ZoneManager.Query("id").Distinct().SubQuery() @@ -670,3 +681,99 @@ func (manager *SServerSkuManager) PendingDeleteInvalidSku() error { return nil } + +func (manager *SServerSkuManager) SyncCloudSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, skus []cloudprovider.ICloudSku) compare.SyncResult { + syncResult := compare.SyncResult{} + dbSkus := manager.GetSkuCountByZone(zone.Id) + + removed := []SServerSku{} + commondb := []SServerSku{} + commonext := []cloudprovider.ICloudSku{} + added := []cloudprovider.ICloudSku{} + + if err := compare.CompareSets(dbSkus, skus, &removed, &commondb, &commonext, &added); err != nil { + syncResult.Error(err) + return syncResult + } + for i := 0; i < len(removed); i++ { + if err := removed[i].ValidateDeleteCondition(ctx); err == nil { + removed[i].Delete(ctx, userCred) + } + syncResult.Delete() + } + + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudSku(ctx, userCred, commonext[i], zone, provider) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } + } + + for i := 0; i < len(added); i++ { + err := manager.newFromCloudSku(ctx, userCred, added[i], zone, provider) + if err != nil { + syncResult.AddError(err) + } else { + syncResult.Add() + } + } + return syncResult +} + +func (self *SServerSku) constructSku(extSku cloudprovider.ICloudSku) { + self.InstanceTypeFamily = extSku.GetInstanceTypeFamily() + self.InstanceTypeCategory = extSku.GetInstanceTypeCategory() + + self.PrepaidStatus = extSku.GetPrepaidStatus() + self.PostpaidStatus = extSku.GetPostpaidStatus() + + self.CpuCoreCount = extSku.GetCpuCoreCount() + self.MemorySizeMB = extSku.GetMemorySizeMB() + + self.OsName = extSku.GetOsName() + + self.SysDiskResizable = extSku.GetSysDiskResizable() + self.SysDiskType = extSku.GetSysDiskType() + self.SysDiskMinSizeGB = extSku.GetSysDiskMinSizeGB() + self.SysDiskMaxSizeGB = extSku.GetSysDiskMaxSizeGB() + + self.AttachedDiskType = extSku.GetAttachedDiskType() + self.AttachedDiskSizeGB = extSku.GetAttachedDiskSizeGB() + self.AttachedDiskCount = extSku.GetAttachedDiskCount() + + self.DataDiskTypes = extSku.GetDataDiskTypes() + self.DataDiskMaxCount = extSku.GetDataDiskMaxCount() + + self.NicType = extSku.GetNicType() + self.NicMaxCount = extSku.GetNicMaxCount() + + self.GpuAttachable = extSku.GetGpuAttachable() + self.GpuSpec = extSku.GetGpuSpec() + self.GpuCount = extSku.GetGpuCount() + self.GpuMaxCount = extSku.GetGpuMaxCount() + self.Name = extSku.GetName() +} + +func (self *SServerSku) syncWithCloudSku(ctx context.Context, userCred mcclient.TokenCredential, extSku cloudprovider.ICloudSku, zone *SZone, provider *SCloudprovider) error { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.constructSku(extSku) + return nil + }) + return err +} + +func (manager *SServerSkuManager) newFromCloudSku(ctx context.Context, userCred mcclient.TokenCredential, extSku cloudprovider.ICloudSku, zone *SZone, provider *SCloudprovider) error { + region := zone.GetRegion() + sku := &SServerSku{ + CloudregionId: region.Id, + ZoneId: zone.Id, + Provider: provider.Provider, + } + sku.constructSku(extSku) + sku.Name = extSku.GetName() + sku.ExternalId = extSku.GetGlobalId() + sku.SetModelManager(manager) + return manager.TableSpec().Insert(sku) +} diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index f2c7c63ab7..5834b9f958 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -93,6 +93,7 @@ var ( STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, + STORAGE_OPENSTACK_ISCSI, } STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index df269437c4..0dee1d6d93 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -710,7 +710,7 @@ func (self *SZone) getMaxDataDiskCount() int { } func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id", "cloudregion", "cloudregion_id"}) + regionStr := jsonutils.GetAnyString(data, []string{"region", "region_id", "cloudregion", "cloudregion_id"}) var regionId string if len(regionStr) > 0 { regionObj, err := CloudregionManager.FetchByIdOrName(nil, regionStr) diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index d151b5850b..5eaa683059 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -159,7 +159,9 @@ func syncPublicCloudProviderInfo(ctx context.Context, provider *models.SCloudpro localZones, remoteZones := syncRegionZones(ctx, provider, task, &localRegions[i], remoteRegions[i]) - syncRegionSkus(ctx, provider, task, &localRegions[i]) + if !driver.GetFactory().NeedSyncSkuFromCloud() { + syncRegionSkus(ctx, provider, task, &localRegions[i]) + } syncRegionVPCs(ctx, provider, task, &localRegions[i], remoteRegions[i], syncRange) @@ -177,6 +179,10 @@ func syncPublicCloudProviderInfo(ctx context.Context, provider *models.SCloudpro if len(newPairs) > 0 { storageCachePairs = append(storageCachePairs, newPairs...) } + + if driver.GetFactory().NeedSyncSkuFromCloud() { + syncRegionSkusFromCloud(ctx, provider, task, &localZones[i], remoteRegions[i], remoteZones[j]) + } } } syncRegionSnapshots(ctx, provider, task, &localRegions[i], remoteRegions[i], syncRange) @@ -329,6 +335,24 @@ func syncLoadbalancerBackends(ctx context.Context, provider *models.SCloudprovid } } +func syncRegionSkusFromCloud(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localZone *models.SZone, remoteRegion cloudprovider.ICloudRegion, remoteZone cloudprovider.ICloudZone) { + skus, err := remoteRegion.GetSkus(remoteZone.GetId()) + if err != nil { + msg := fmt.Sprintf("GetSkus for zone %s failed %v", localZone.Name, err) + log.Errorf(msg) + logSyncFailed(provider, task, msg) + return + } + + result := models.ServerSkuManager.SyncCloudSkusByRegion(ctx, task.GetUserCred(), provider, localZone, skus) + msg := result.Result() + log.Infof("SyncCloudSkusByRegion for zone %s result: %s", localZone.Name, msg) + if result.IsError() { + logSyncFailed(provider, task, msg) + return + } +} + func syncRegionSkus(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localRegion *models.SCloudregion) { if localRegion == nil { log.Debugf("local region is nil skipped.") diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index a14fd1b72a..5b2a0a610e 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -1,7 +1,6 @@ package hostman import ( - "context" "os" "yunion.io/x/log" @@ -23,8 +22,6 @@ import ( type SHostService struct { service.SServiceBase - - isExiting bool } func (host *SHostService) StartService() { @@ -37,28 +34,6 @@ func (host *SHostService) StartService() { log.Fatalf(err.Error()) } - host.RegisterSIGUSR1() - host.RegisterQuitSignals(func() { // register quit handler - if host.isExiting { - return - } else { - host.isExiting = true - } - - if app.IsInServe() { - if err := app.ShutDown(context.Background()); err != nil { - log.Errorln(err.Error()) - } - } - - hostinfo.Stop() - storageman.Stop() - hostmetrics.Stop() - guestman.Stop() - hostutils.GetWorkManager().Stop() - os.Exit(0) - }) - if err := storageman.Init(hostInstance); err != nil { log.Fatalf(err.Error()) } @@ -82,20 +57,13 @@ func (host *SHostService) StartService() { cronManager.AddJob2( "CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false) - go func() { - defer func() { - if r := recover(); r != nil { - if !host.isExiting { - log.Fatalf("%s", r) - } else { - log.Errorln(r) - } - } - - }() - cloudcommon.ServeForever(app, &options.HostOptions.CommonOptions) - }() - select {} // for quit handler + cloudcommon.ServeForeverWithCleanup(app, &options.HostOptions.CommonOptions, func() { + hostinfo.Stop() + storageman.Stop() + hostmetrics.Stop() + guestman.Stop() + hostutils.GetWorkManager().Stop() + }) } func (host *SHostService) initHandlers(app *appsrv.Application) { diff --git a/pkg/image/models/image_subs.go b/pkg/image/models/image_subs.go index c972b9740d..67ee947f6f 100644 --- a/pkg/image/models/image_subs.go +++ b/pkg/image/models/image_subs.go @@ -6,8 +6,6 @@ import ( "os" "path/filepath" - t "github.com/anacrolix/torrent" - "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -89,10 +87,12 @@ func (self *SImageSubformat) DoConvert(image *SImage) error { log.Errorf("fail to convert image torrent %s", err) return err } - err = self.seedTorrent() - if err != nil { - log.Errorf("fail to seed torrent %s", err) - return err + if options.Options.EnableTorrentService { + err = self.seedTorrent(image.Id) + if err != nil { + log.Errorf("fail to seed torrent %s", err) + return err + } } // log.Infof("Start seeding...") return nil @@ -206,10 +206,10 @@ func (self *SImageSubformat) getLocalTorrentLocation() string { return "" } -func (self *SImageSubformat) seedTorrent() error { +func (self *SImageSubformat) seedTorrent(imageId string) error { file := self.getLocalTorrentLocation() log.Debugf("add torrent %s to seed...", file) - return torrent.AddTorrent(file) + return torrent.SeedTorrent(file, imageId, self.Format) } func (self *SImageSubformat) StopTorrent() { @@ -237,14 +237,6 @@ func (self *SImageSubformat) RemoveFiles() error { return nil } -func (self *SImageSubformat) getTorrent() *t.Torrent { - torrentPath := self.getLocalTorrentLocation() - if len(torrentPath) > 0 { - return torrent.GetTorrent(torrentPath) - } - return nil -} - type SImageSubformatDetails struct { Format string @@ -258,7 +250,6 @@ type SImageSubformatDetails struct { TorrentStatus string TorrentSeeding bool - TorrentStats t.TorrentStats } func (self *SImageSubformat) GetDetails() SImageSubformatDetails { @@ -273,14 +264,11 @@ func (self *SImageSubformat) GetDetails() SImageSubformatDetails { details.TorrentChecksum = self.TorrentChecksum details.TorrentStatus = self.TorrentStatus - t := self.getTorrent() - - if t != nil { - if t.BytesMissing() == 0 { - details.TorrentSeeding = true - } - details.TorrentStats = t.Stats() + filePath := self.getLocalTorrentLocation() + if len(filePath) > 0 { + details.TorrentSeeding = torrent.GetTorrentSeeding(filePath) } + return details } @@ -342,3 +330,10 @@ func (self *SImageSubformat) checkStatus(useFast bool) { } } } + +func (self *SImageSubformat) SetStatusSeeding(seeding bool) { + filePath := self.getLocalTorrentLocation() + if len(filePath) > 0 { + torrent.SetTorrentSeeding(filePath, seeding) + } +} diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 15af820b6e..80f695d656 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -899,7 +899,7 @@ func (self *SImage) StopTorrents() { func (self *SImage) seedTorrents() { subimgs := ImageSubformatManager.GetAllSubImages(self.Id) for i := 0; i < len(subimgs); i += 1 { - subimgs[i].seedTorrent() + subimgs[i].seedTorrent(self.Id) } } @@ -1046,7 +1046,7 @@ func (self *SImage) DoCheckStatus(ctx context.Context, userCred mcclient.TokenCr if needConvert { log.Infof("Image %s is active and need convert", self.Name) self.StartImageConvertTask(ctx, userCred, "") - } else { + } else if options.Options.EnableTorrentService { self.seedTorrents() } } @@ -1081,3 +1081,20 @@ func (self *SImage) PerformMarkPublicProtected( } return nil, nil } + +func (self *SImage) AllowPerformUpdateTorrentStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return true +} + +func (self *SImage) PerformUpdateTorrentStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + formatStr, _ := query.GetString("format") + if len(formatStr) == 0 { + return nil, httperrors.NewInputParameterError("missing parameter format") + } + subimg := ImageSubformatManager.FetchSubImage(self.Id, formatStr) + if subimg == nil { + return nil, httperrors.NewResourceNotFoundError("format %s not found", formatStr) + } + subimg.SetStatusSeeding(true) + return nil, nil +} diff --git a/pkg/image/options/options.go b/pkg/image/options/options.go index edc4fcf17e..7f07c220b0 100644 --- a/pkg/image/options/options.go +++ b/pkg/image/options/options.go @@ -23,6 +23,8 @@ type SImageOptions struct { EnableTorrentService bool `help:"Enable torrent service" default:"false"` TargetImageFormats []string `help:"target image formats that the system will automatically convert to" default:"qcow2,vmdk,vhd"` + + TorrentClientPath string `help:"path to torrent executable" default:"/opt/yunion/bin/torrent"` } var ( diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go index d973cf0277..8e891f01dd 100644 --- a/pkg/image/service/service.go +++ b/pkg/image/service/service.go @@ -62,35 +62,37 @@ func StartService() { log.Infof("Auth complete!!") }) + trackers := torrent.GetTrackers() + if len(trackers) == 0 { + log.Errorf("no valid torrent-tracker") + return + } + cloudcommon.InitDB(dbOpts) - defer cloudcommon.CloseDB() app := cloudcommon.InitApp(commonOpts, true) initHandlers(app) - if opts.EnableTorrentService { - err := torrent.InitTorrentClient() - if err != nil { - log.Errorf("fail to initialize torrent client: %s", err) - return - } - torrent.InitTorrentHandler(app) - defer torrent.CloseTorrentClient() - } - if !db.CheckSync(opts.AutoSyncTable) { log.Fatalf("database schema not in sync!") } models.InitDB() - models.CheckImages() + go models.CheckImages() cron := cronman.GetCronJobManager(true) cron.AddJob1("CleanPendingDeleteImages", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.ImageManager.CleanPendingDeleteImages) cron.Start() - defer cron.Stop() - cloudcommon.ServeForever(app, commonOpts) + cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + cloudcommon.CloseDB() + + cron.Stop() + + if options.Options.EnableTorrentService { + torrent.StopTorrents() + } + }) } diff --git a/pkg/image/torrent/torrent.go b/pkg/image/torrent/torrent.go index d41a3ccf1a..8cecbdb3b0 100644 --- a/pkg/image/torrent/torrent.go +++ b/pkg/image/torrent/torrent.go @@ -1,28 +1,52 @@ package torrent import ( - "context" "fmt" - "net/http" - - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/metainfo" + "os" + "time" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/image/options" "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/util/nodeid" + "yunion.io/x/onecloud/pkg/util/sysutils" ) +type STorrentProcessState struct { + process *os.Process + seeding bool +} + var ( - torrentClient *torrent.Client - torrentTable = make(map[string]*torrent.Torrent) + torrentTable = make(map[string]*STorrentProcessState) + seedTaskWorkerMan *appsrv.SWorkerManager ) +const ( + TORRENT_TRACKER_SERVICE = "torrent-tracker" +) + +func init() { + seedTaskWorkerMan = appsrv.NewWorkerManager("seedTaskWorkerManager", 1, 1024, false) +} + +func (stat *STorrentProcessState) StopAndWait() error { + err := stat.process.Kill() + if err != nil { + log.Errorf("kill error %s", err) + return err + } + _, err = stat.process.Wait() + if err != nil { + log.Errorf("wait error %s", err) + return err + } + return nil +} + func GetTrackers() []string { - urls, err := auth.GetServiceURLs("torrent-tracker", options.Options.Region, "", "") + urls, err := auth.GetServiceURLs(TORRENT_TRACKER_SERVICE, options.Options.Region, "", "") if err != nil { log.Errorf("fail to get torrent-tracker") return nil @@ -30,93 +54,62 @@ func GetTrackers() []string { return urls } -func InitTorrentClient() error { - urls := GetTrackers() - if len(urls) == 0 { - log.Errorf("no valid torrent-tracker") - return fmt.Errorf("no valid torrent-tracker") - } - - nodeId, err := nodeid.GetNodeId() - if err != nil { - log.Errorf("fail to generate node id: %s", err) - return err - } - - log.Infof("Set torrent server as node %s", nodeId) - - clientConfig := torrent.NewDefaultClientConfig() - clientConfig.PeerID = nodeId[:20] - clientConfig.Debug = false - clientConfig.Seed = true - clientConfig.NoUpload = false - clientConfig.DataDir = options.Options.FilesystemStoreDatadir - clientConfig.DisableTrackers = false - clientConfig.DisablePEX = true - clientConfig.NoDHT = true - - client, err := torrent.NewClient(clientConfig) - if err != nil { - log.Errorf("error creating client: %s", err) - return err - } - torrentClient = client - - log.Infof("torrent client initialized") - +func SeedTorrent(torrentpath string, imageId, format string) error { + seedTaskWorkerMan.Run(func() { + log.Infof("Start seed %s ...", torrentpath) + err := seedTorrent(torrentpath, imageId, format) + if err == nil { + time.Sleep(10 * time.Second) + } + }, nil, nil) return nil } -func InitTorrentHandler(app *appsrv.Application) { - app.AddDefaultHandler("GET", "/torrent_stats", TorrentStatsHandler, "torrent_stats") -} - -func TorrentStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { - if torrentClient != nil { - torrentClient.WriteStatus(w) - } -} - -func CloseTorrentClient() { - if torrentClient != nil { - torrentClient.Close() - } -} - -func AddTorrent(filepath string) error { - if torrentClient == nil { - return nil - } - - mi, err := metainfo.LoadFromFile(filepath) +func seedTorrent(torrentpath string, imageId, format string) error { + url, err := auth.GetServiceURL("image", options.Options.Region, "", "public") if err != nil { - log.Errorf("fail to open torrent file %s", err) return err } - t, err := torrentClient.AddTorrent(mi) + args := []string{ + options.Options.TorrentClientPath, + options.Options.FilesystemStoreDatadir, + torrentpath, + "--callback-url", + fmt.Sprintf("%s/images/%s/update-torrent-status?format=%s", url, imageId, format), + } + proc, err := sysutils.Start(false, args...) if err != nil { - log.Errorf("AddTorrent fail %s", err) return err } - - torrentTable[filepath] = t - - <-t.GotInfo() - t.DownloadAll() - + torrentTable[torrentpath] = &STorrentProcessState{ + process: proc, + seeding: false, + } return nil } -func GetTorrent(filepath string) *torrent.Torrent { +func SetTorrentSeeding(filepath string, seeding bool) { + if _, ok := torrentTable[filepath]; ok { + torrentTable[filepath].seeding = seeding + } +} + +func GetTorrentSeeding(filepath string) bool { if t, ok := torrentTable[filepath]; ok { - return t + return t.seeding } - return nil + return false } func RemoveTorrent(filepath string) { if t, ok := torrentTable[filepath]; ok { - t.Drop() + t.StopAndWait() delete(torrentTable, filepath) } } + +func StopTorrents() { + for k := range torrentTable { + torrentTable[k].StopAndWait() + } +} diff --git a/pkg/mcclient/auth/middleware.go b/pkg/mcclient/auth/middleware.go index 5c892a33ce..e01649696b 100644 --- a/pkg/mcclient/auth/middleware.go +++ b/pkg/mcclient/auth/middleware.go @@ -12,6 +12,12 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" ) +var ( + GuestToken = mcclient.SSimpleToken{ + User: "guest", + } +) + const ( AUTH_TOKEN = appctx.AppContextKey("X_AUTH_TOKEN") ) @@ -23,24 +29,26 @@ func Authenticate(f appsrv.FilterHandler) appsrv.FilterHandler { func AuthenticateWithDelayDecision(f appsrv.FilterHandler, delayDecision bool) appsrv.FilterHandler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { tokenStr := r.Header.Get(mcclient.AUTH_TOKEN) + var token mcclient.TokenCredential if len(tokenStr) == 0 { log.Errorf("no auth_token found!") if !delayDecision { httperrors.UnauthorizedError(w, "Unauthorized") return } - } - token, err := Verify(tokenStr) - if err != nil { - log.Errorf("Verify token failed: %s", err) - if !delayDecision { - httperrors.UnauthorizedError(w, "InvalidToken") - return + token = &GuestToken + } else { + var err error + token, err = Verify(tokenStr) + if err != nil { + log.Errorf("Verify token failed: %s", err) + if !delayDecision { + httperrors.UnauthorizedError(w, "InvalidToken") + return + } } } - if token != nil { - ctx = context.WithValue(ctx, AUTH_TOKEN, token) - } + ctx = context.WithValue(ctx, AUTH_TOKEN, token) if taskId := r.Header.Get(mcclient.TASK_ID); taskId != "" { ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TASK_ID, taskId) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 6e2b529fdc..5e0f44448a 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -896,8 +896,8 @@ func (self *SInstance) AssignSecurityGroup(secgroupId string) error { return self.host.zone.region.AssignSecurityGroup(secgroupId, self.InstanceId) } -func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { - return self.host.zone.region.AssignSecurityGroups(secgroupIds, self.InstanceId) +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { + return self.host.zone.region.SetSecurityGroups(secgroupIds, self.InstanceId) } func (self *SInstance) GetBillingType() string { diff --git a/pkg/util/aliyun/provider/provider.go b/pkg/util/aliyun/provider/provider.go index 915b4e25db..7680fb7df6 100644 --- a/pkg/util/aliyun/provider/provider.go +++ b/pkg/util/aliyun/provider/provider.go @@ -38,6 +38,10 @@ func (self *SAliyunProviderFactory) IsSupportPrepaidResources() bool { return true } +func (self *SAliyunProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SAliyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { accessKeyID, _ := data.GetString("access_key_id") if len(accessKeyID) == 0 { diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index de46afcacb..ab3032d655 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -888,3 +888,7 @@ func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAc } return iAcl, region.AddAccessControlListEntry(aclId, acl.Entrys) } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 7385b5d487..458474ada9 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -501,10 +501,10 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur } func (self *SRegion) AssignSecurityGroup(secgroupId, instanceId string) error { - return self.AssignSecurityGroups([]string{secgroupId}, instanceId) + return self.SetSecurityGroups([]string{secgroupId}, instanceId) } -func (self *SRegion) AssignSecurityGroups(secgroupIds []string, instanceId string) error { +func (self *SRegion) SetSecurityGroups(secgroupIds []string, instanceId string) error { params := map[string]string{"InstanceId": instanceId} for _, secgroupId := range secgroupIds { params["SecurityGroupId"] = secgroupId diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go index 5a0e73a4fd..5b20e7f774 100644 --- a/pkg/util/aws/instance.go +++ b/pkg/util/aws/instance.go @@ -283,10 +283,10 @@ func (self *SInstance) GetMachine() string { } func (self *SInstance) AssignSecurityGroup(secgroupId string) error { - return self.AssignSecurityGroups([]string{secgroupId}) + return self.SetSecurityGroups([]string{secgroupId}) } -func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { ids := []*string{} for i := 0; i < len(secgroupIds); i++ { ids = append(ids, &secgroupIds[i]) diff --git a/pkg/util/aws/provider/provider.go b/pkg/util/aws/provider/provider.go index 8fda12f42b..408eeef9e6 100644 --- a/pkg/util/aws/provider/provider.go +++ b/pkg/util/aws/provider/provider.go @@ -37,6 +37,10 @@ func (self *SAwsProviderFactory) IsSupportPrepaidResources() bool { return true } +func (self *SAwsProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SAwsProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { accessKeyID, _ := data.GetString("access_key_id") if len(accessKeyID) == 0 { diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go index b84071b4ca..7cdab4638b 100644 --- a/pkg/util/aws/region.go +++ b/pkg/util/aws/region.go @@ -499,3 +499,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { return nil, cloudprovider.ErrNotImplemented } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go index 4f7add3332..bd3fb44378 100644 --- a/pkg/util/azure/classic_instance.go +++ b/pkg/util/azure/classic_instance.go @@ -487,7 +487,7 @@ type assignProperties struct { NetworkSecurityGroup SubResource `json:"networkSecurityGroup,omitempty"` } -func (self *SClassicInstance) AssignSecurityGroups(secgroupIds []string) error { +func (self *SClassicInstance) SetSecurityGroups(secgroupIds []string) error { return cloudprovider.ErrNotSupported } diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index 38e07c1f50..663cfd8e5c 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -1020,7 +1020,7 @@ func (self *SInstance) AssignSecurityGroup(secgroupId string) error { return self.host.zone.region.AssiginSecurityGroup(self.ID, secgroupId) } -func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { return cloudprovider.ErrNotSupported } diff --git a/pkg/util/azure/provider/provider.go b/pkg/util/azure/provider/provider.go index 534c896b66..936827e17c 100644 --- a/pkg/util/azure/provider/provider.go +++ b/pkg/util/azure/provider/provider.go @@ -39,6 +39,10 @@ func (self *SAzureProviderFactory) IsSupportPrepaidResources() bool { return true } +func (self *SAzureProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SAzureProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { directoryID, _ := data.GetString("directory_id") if len(directoryID) == 0 { diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 28df064f46..3eef898d28 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -551,3 +551,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { return nil, cloudprovider.ErrNotImplemented } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/esxi/provider/provider.go b/pkg/util/esxi/provider/provider.go index 68f4b030d5..aece6af9f1 100644 --- a/pkg/util/esxi/provider/provider.go +++ b/pkg/util/esxi/provider/provider.go @@ -43,6 +43,10 @@ func (self *SESXiProviderFactory) IsSupportPrepaidResources() bool { return false } +func (self *SESXiProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SESXiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { username, _ := data.GetString("username") if len(username) == 0 { diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index bbe9058d51..6ccdad303a 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -528,7 +528,7 @@ func (dc *SVirtualMachine) ChangeConfig2(ctx context.Context, instanceType strin return cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) AssignSecurityGroups(secgroupIds []string) error { +func (self *SVirtualMachine) SetSecurityGroups(secgroupIds []string) error { return cloudprovider.ErrNotImplemented } diff --git a/pkg/util/huawei/instance.go b/pkg/util/huawei/instance.go index 811a6f0b94..b0e30ab46f 100644 --- a/pkg/util/huawei/instance.go +++ b/pkg/util/huawei/instance.go @@ -9,6 +9,7 @@ import ( "strconv" "sort" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -398,10 +399,10 @@ func (self *SInstance) GetMachine() string { } func (self *SInstance) AssignSecurityGroup(secgroupId string) error { - return self.AssignSecurityGroups([]string{secgroupId}) + return self.SetSecurityGroups([]string{secgroupId}) } -func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { currentSecgroups, err := self.host.zone.region.GetInstanceSecrityGroupIds(self.GetId()) if err != nil { return err diff --git a/pkg/util/huawei/provider/provider.go b/pkg/util/huawei/provider/provider.go index 3761227d88..532f05b00a 100644 --- a/pkg/util/huawei/provider/provider.go +++ b/pkg/util/huawei/provider/provider.go @@ -37,6 +37,10 @@ func (self *SHuaweiProviderFactory) IsSupportPrepaidResources() bool { return true } +func (self *SHuaweiProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SHuaweiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { accessKeyID, _ := data.GetString("access_key_id") if len(accessKeyID) == 0 { diff --git a/pkg/util/huawei/region.go b/pkg/util/huawei/region.go index 4d686cd1f8..48967db46b 100644 --- a/pkg/util/huawei/region.go +++ b/pkg/util/huawei/region.go @@ -641,3 +641,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { return nil, cloudprovider.ErrNotImplemented } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/nodeid/nodeid.go b/pkg/util/nodeid/nodeid.go index 75cc6e72a0..8d456598d2 100644 --- a/pkg/util/nodeid/nodeid.go +++ b/pkg/util/nodeid/nodeid.go @@ -171,20 +171,20 @@ func getLinux() ([]string, error) { return ret, nil } -func GetNodeId() (string, error) { +func GetNodeId() ([]byte, error) { var f func() ([]string, error) if runtime.GOOS == "linux" { f = getLinux } else { - return "", fmt.Errorf("Unsupported OS") + return nil, fmt.Errorf("Unsupported OS") } ret, e := f() if e != nil || len(ret) < 2 { log.Debugf("service info %s", ret) - return "", fmt.Errorf("Fail to generate Node ID") + return nil, fmt.Errorf("Fail to generate Node ID") } sn := md5.Sum([]byte(ret[0] + ret[1])) - return fmt.Sprintf("%x", sn), nil + return sn[0:], nil } diff --git a/pkg/util/openstack/disk.go b/pkg/util/openstack/disk.go index 952833b0a4..78eafd20a9 100644 --- a/pkg/util/openstack/disk.go +++ b/pkg/util/openstack/disk.go @@ -2,6 +2,7 @@ package openstack import ( "context" + "fmt" "time" "yunion.io/x/jsonutils" @@ -104,7 +105,7 @@ func (disk *SDisk) GetMetadata() *jsonutils.JSONDict { } func (region *SRegion) GetDisks(category string) ([]SDisk, error) { - _, resp, err := region.CinderGet("/volumes/detail", "", nil) + _, resp, err := region.CinderList("/volumes/detail", "", nil) if err != nil { return nil, err } @@ -126,11 +127,11 @@ func (disk *SDisk) GetId() string { } func (disk *SDisk) Delete(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return disk.storage.zone.region.DeleteDisk(disk.ID) } func (disk *SDisk) Resize(ctx context.Context, sizeMb int64) error { - return cloudprovider.ErrNotImplemented + return disk.storage.zone.region.ResizeDisk(disk.ID, sizeMb) } func (disk *SDisk) GetName() string { @@ -179,8 +180,8 @@ func (disk *SDisk) Refresh() error { return jsonutils.Update(disk, new) } -func (disk *SDisk) ResizeDisk(newSize int64) error { - return disk.storage.zone.region.ResizeDisk(disk.ID, newSize) +func (disk *SDisk) ResizeDisk(sizeMb int64) error { + return disk.storage.zone.region.ResizeDisk(disk.ID, sizeMb) } func (disk *SDisk) GetDiskFormat() string { @@ -226,8 +227,22 @@ func (disk *SDisk) GetMountpoint() string { return "" } -func (disk *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (region *SRegion) CreateDisk(zoneName string, category string, name string, sizeGb int, desc string) (*SDisk, error) { + params := map[string]map[string]interface{}{ + "volume": { + "size": sizeGb, + "volume_type": category, + "name": name, + "description": desc, + "availability_zone": zoneName, + }, + } + _, resp, err := region.CinderCreate("/volumes", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + disk := &SDisk{} + return disk, resp.Unmarshal(disk, "volume") } func (region *SRegion) GetDisk(diskId string) (*SDisk, error) { @@ -239,20 +254,39 @@ func (region *SRegion) GetDisk(diskId string) (*SDisk, error) { return disk, resp.Unmarshal(disk, "volume") } -func (disk *SRegion) DeleteDisk(diskId string) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) DeleteDisk(diskId string) error { + _, err := region.CinderDelete("/volumes/"+diskId, "") + return err } -func (disk *SRegion) ResizeDisk(diskId string, sizeMb int64) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) ResizeDisk(diskId string, sizeMb int64) error { + params := map[string]map[string]interface{}{ + "os-extend": { + "new_size": sizeMb / 1024, + }, + } + _, _, err := region.CinderAction(fmt.Sprintf("/volumes/%s/action", diskId), "", jsonutils.Marshal(params)) + return err } -func (disk *SRegion) ResetDisk(diskId, snapshotId string) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) ResetDisk(diskId, snapshotId string) error { + //目前测试接口不能使用 + return cloudprovider.ErrNotSupported + // params := map[string]map[string]interface{}{ + // "revert": { + // "snapshot_id": snapshotId, + // }, + // } + // _, _, err := region.CinderAction(fmt.Sprintf("/volumes/%s/action", diskId), "3.40", jsonutils.Marshal(params)) + // return err } func (disk *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { - return nil, cloudprovider.ErrNotImplemented + snapshot, err := disk.storage.zone.region.CreateSnapshot(disk.ID, name, desc) + if err != nil { + return nil, err + } + return snapshot, cloudprovider.WaitStatus(snapshot, models.SNAPSHOT_READY, time.Second*5, time.Minute*5) } func (disk *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) { @@ -264,7 +298,7 @@ func (disk *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { } func (disk *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { - return "", disk.storage.zone.region.ResetDisk(disk.ID, snapshotId) + return disk.ID, disk.storage.zone.region.ResetDisk(disk.ID, snapshotId) } func (disk *SDisk) GetBillingType() string { @@ -280,9 +314,5 @@ func (disk *SDisk) GetAccessPath() string { } func (disk *SDisk) Rebuild(ctx context.Context) error { - return disk.storage.zone.region.RebuildDisk(disk.ID) -} - -func (region *SRegion) RebuildDisk(diskId string) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } diff --git a/pkg/util/openstack/flavor.go b/pkg/util/openstack/flavor.go new file mode 100644 index 0000000000..748e48eef1 --- /dev/null +++ b/pkg/util/openstack/flavor.go @@ -0,0 +1,244 @@ +package openstack + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SFlavor struct { + region *SRegion + ID string + Disk int + Ephemeral int + ExtraSpecs ExtraSpecs + OriginalName string + Name string + RAM int + Swap string + Vcpus int8 +} + +func (region *SRegion) GetFlavors() ([]SFlavor, error) { + _, resp, err := region.List("compute", "/flavors/detail", "", nil) + if err != nil { + return nil, err + } + flavors := []SFlavor{} + return flavors, resp.Unmarshal(&flavors, "flavors") +} + +func (region *SRegion) GetFlavor(flavorId string) (*SFlavor, error) { + _, resp, err := region.Get("compute", "/flavors/"+flavorId, "", nil) + if err != nil { + return nil, err + } + flavor := &SFlavor{region: region} + return flavor, resp.Unmarshal(flavor, "flavor") +} + +func (region *SRegion) syncFlavor(name string, cpu, memoryMb, diskGB int) (string, error) { + flavors, err := region.GetFlavors() + if err != nil { + return "", err + } + if len(name) > 0 { + for _, flavor := range flavors { + if flavor.GetName() == name { + return flavor.ID, nil + } + } + } + + if cpu == 0 && memoryMb == 0 { + return "", fmt.Errorf("failed to find instance type %s", name) + } + + for _, flavor := range flavors { + if flavor.GetCpuCoreCount() == cpu && flavor.GetMemorySizeMB() == memoryMb { + return flavor.ID, nil + } + } + + if len(name) == 0 { + suffix := "" + for i := 0; i < 10; i++ { + switch cpu { + case 1: + suffix = "tiny" + case 2, 3: + suffix = "small" + case 4, 6: + suffix = "medium" + case 8: + suffix = "large" + default: + suffix = "xlarge" + } + } + for i := 0; i < 10; i++ { + if _, err := region.GetFlavor(fmt.Sprintf("m%d.%s", i, suffix)); err != nil { + if err == cloudprovider.ErrNotFound { + name = fmt.Sprintf("m%d.%s", i, suffix) + break + } + } + } + if len(name) == 0 { + return "", fmt.Errorf("failed to find uniq flavor name for cpu %d memory %d", cpu, memoryMb) + } + } + + flavor, err := region.CreateFlavor(name, cpu, memoryMb, diskGB) + if err != nil { + return "", err + } + return flavor.ID, nil +} + +func (region *SRegion) CreateFlavor(name string, cpu int, memoryMb int, diskGB int) (*SFlavor, error) { + params := map[string]map[string]interface{}{ + "flavor": { + "name": name, + "ram": memoryMb, + "vcpus": cpu, + "disk": diskGB, + }, + } + _, resp, err := region.Post("compute", "/flavors", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + flavor := &SFlavor{} + return flavor, resp.Unmarshal(flavor, "flavor") +} + +func (region *SRegion) DeleteFlavor(flavorId string) error { + _, err := region.Delete("compute", "/flavors/"+flavorId, "") + return err +} + +func (flavor *SFlavor) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (flavor *SFlavor) IsEmulated() bool { + return false +} + +func (flavor *SFlavor) Refresh() error { + new, err := flavor.region.GetFlavor(flavor.ID) + if err != nil { + return err + } + return jsonutils.Update(flavor, new) +} + +func (flavor *SFlavor) GetName() string { + if len(flavor.OriginalName) > 0 { + return flavor.OriginalName + } + return flavor.Name +} + +func (flavor *SFlavor) GetStatus() string { + return "" +} + +func (flavor *SFlavor) GetId() string { + return flavor.ID +} + +func (flavor *SFlavor) GetGlobalId() string { + return flavor.ID +} + +func (flavor *SFlavor) GetInstanceTypeFamily() string { + return flavor.GetName() +} + +func (flavor *SFlavor) GetInstanceTypeCategory() string { + return flavor.GetName() +} + +func (flavor *SFlavor) GetPrepaidStatus() string { + return models.SkuStatusSoldout +} + +func (flavor *SFlavor) GetPostpaidStatus() string { + return models.SkuStatusAvailable +} + +func (flavor *SFlavor) GetCpuCoreCount() int { + return int(flavor.Vcpus) +} + +func (flavor *SFlavor) GetMemorySizeMB() int { + return flavor.RAM +} + +func (flavor *SFlavor) GetOsName() string { + return "Any" +} + +func (flavor *SFlavor) GetSysDiskResizable() bool { + return true +} + +func (flavor *SFlavor) GetSysDiskType() string { + return "iscsi" +} + +func (flavor *SFlavor) GetSysDiskMinSizeGB() int { + return 0 +} + +func (flavor *SFlavor) GetSysDiskMaxSizeGB() int { + return flavor.Disk +} + +func (flavor *SFlavor) GetAttachedDiskType() string { + return "iscsi" +} + +func (flavor *SFlavor) GetAttachedDiskSizeGB() int { + return 0 +} + +func (flavor *SFlavor) GetAttachedDiskCount() int { + return 6 +} + +func (flavor *SFlavor) GetDataDiskTypes() string { + return "iscsi" +} + +func (flavor *SFlavor) GetDataDiskMaxCount() int { + return 6 +} + +func (flavor *SFlavor) GetNicType() string { + return "vpc" +} + +func (flavor *SFlavor) GetNicMaxCount() int { + return 1 +} + +func (flavor *SFlavor) GetGpuAttachable() bool { + return false +} + +func (flavor *SFlavor) GetGpuSpec() string { + return "" +} + +func (flavor *SFlavor) GetGpuCount() int { + return 0 +} + +func (flavor *SFlavor) GetGpuMaxCount() int { + return 0 +} diff --git a/pkg/util/openstack/host.go b/pkg/util/openstack/host.go index e42c8c1650..df79baff87 100644 --- a/pkg/util/openstack/host.go +++ b/pkg/util/openstack/host.go @@ -1,9 +1,16 @@ package openstack import ( + "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/version" +) + +const ( + VOLUME_TYPES_API_VERSION = "2.67" ) type CpuInfo struct { @@ -116,7 +123,128 @@ func (host *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { } func (host *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { - return nil, cloudprovider.ErrNotImplemented + network, err := host.zone.region.GetNetwork(desc.ExternalNetworkId) + if err != nil { + return nil, err + } + + secgroups := []map[string]string{} + + for _, secgroupId := range desc.ExternalSecgroupIds { + secgroups = append(secgroups, map[string]string{"name": secgroupId}) + } + + image, err := host.zone.region.GetImage(desc.ExternalImageId) + if err != nil { + return nil, err + } + + storage, err := host.zone.getStorageByCategory(desc.SysDisk.StorageType) + if err != nil { + return nil, err + } + + sysDiskSizeGB := image.Size / 1024 / 1024 + if desc.SysDisk.SizeGB < sysDiskSizeGB { + desc.SysDisk.SizeGB = sysDiskSizeGB + } + + _, maxVersion, _ := host.zone.region.GetVersion("compute") + + BlockDeviceMappingV2 := []map[string]interface{}{ + { + "boot_index": 0, + "uuid": desc.ExternalImageId, + "source_type": "image", + "destination_type": "volume", + "volume_size": desc.SysDisk.SizeGB, + "delete_on_termination": true, + }, + } + + if version.GE(maxVersion, VOLUME_TYPES_API_VERSION) { + BlockDeviceMappingV2[0]["volume_type"] = storage.Name + } + + var _disk *SDisk + for _, disk := range desc.DataDisks { + storage, err = host.zone.getStorageByCategory(disk.StorageType) + if err != nil { + break + } + _disk, err = host.zone.region.CreateDisk(host.zone.ZoneName, storage.Name, "", disk.SizeGB, disk.Name) + if err != nil { + break + } + + mapping := map[string]interface{}{ + "source_type": "volume", + "destination_type": "volume", + "delete_on_termination": true, + "uuid": _disk.ID, + } + + BlockDeviceMappingV2 = append(BlockDeviceMappingV2, mapping) + } + if err != nil { + for _, blockMap := range BlockDeviceMappingV2 { + if blockMap["source_type"] == "volume" { + if uuid, ok := blockMap["uuid"].(string); ok { + host.zone.region.DeleteDisk(uuid) + } + } + } + return nil, err + } + + params := map[string]map[string]interface{}{ + "server": { + "name": desc.Name, + "adminPass": desc.Password, + //"description": desc.Description, + "accessIPv4": desc.IpAddr, + "availability_zone": fmt.Sprintf("%s:%s", host.zone.ZoneName, host.GetName()), + "networks": []map[string]string{ + { + "uuid": network.NetworkID, + "fixed_ip": desc.IpAddr, + }, + }, + "security_groups": secgroups, + "user_data": desc.UserData, + "imageRef": desc.ExternalImageId, + "block_device_mapping_v2": BlockDeviceMappingV2, + }, + } + + flavorId, err := host.zone.region.syncFlavor(desc.InstanceType, desc.Cpu, desc.MemoryMB, desc.SysDisk.SizeGB) + if err != nil { + return nil, err + } + params["server"]["flavorRef"] = flavorId + + if len(desc.PublicKey) > 0 { + keypairName, err := host.zone.region.syncKeypair(desc.Name, desc.PublicKey) + if err != nil { + return nil, err + } + params["server"]["key_name"] = keypairName + } + + _, resp, err := host.zone.region.Post("compute", "/servers", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + serverId, err := resp.GetString("server", "id") + if err != nil { + return nil, err + } + instance, err := host.zone.region.GetInstance(serverId) + if err != nil { + return nil, err + } + instance.host = host + return instance, nil } func (host *SHost) GetEnabled() bool { diff --git a/pkg/util/openstack/image.go b/pkg/util/openstack/image.go new file mode 100644 index 0000000000..ade44b6e59 --- /dev/null +++ b/pkg/util/openstack/image.go @@ -0,0 +1,230 @@ +package openstack + +import ( + "context" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/utils" +) + +const ( + QUEUED = "queued" // The Image service reserved an image ID for the image in the catalog but did not yet upload any image data. + SAVING = "saving" // The Image service is in the process of saving the raw data for the image into the backing store. + ACTIVE = "active" // The image is active and ready for consumption in the Image service. + KILLED = "killed" // An image data upload error occurred. + DELETED = "deleted" // The Image service retains information about the image but the image is no longer available for use. + PENDING_DELETE = "pending_delete" // Similar to the deleted status. An image in this state is not recoverable. + DEACTIVATED = "deactivated" // The image data is not available for use. + UPLOADING = "uploading" // Data has been staged as part of the interoperable image import process. It is not yet available for use. (Since Image API 2.6) + IMPORTING = "importing" // The image data is being processed as part of the interoperable image import process, but is not yet available for use. (Since Image API 2.6) +) + +type SImage struct { + storageCache *SStoragecache + + Status string + Name string + Tags []string + ContainerFormat string + CreatedAt time.Time + DiskFormat string + UpdatedAt time.Time + Visibility string + Self string + MinDisk int + Protected bool + ID string + File string + Checksum string + OsHashAlgo string + OsHashValue string + OsHidden bool + Owner string + Size int + MinRAM int + Schema string + VirtualSize int + visibility string +} + +func (region *SRegion) GetImages(name string, status string, imageIds []string) ([]SImage, error) { + params := url.Values{} + if utils.IsInStringArray(status, []string{QUEUED, SAVING, ACTIVE, KILLED, DELETED, PENDING_DELETE, DEACTIVATED, UPLOADING, IMPORTING}) { + params.Add("status", status) + } + if len(name) > 0 { + params.Add("name", name) + } + if len(imageIds) > 0 { + params.Add("id", "in:"+strings.Join(imageIds, ",")) + } + _, resp, err := region.List("image", "/v2/images?"+params.Encode(), "", nil) + if err != nil { + return nil, err + } + images := []SImage{} + return images, resp.Unmarshal(&images, "images") +} + +func (image *SImage) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (image *SImage) GetId() string { + return image.ID +} + +func (image *SImage) GetName() string { + return image.Name +} + +func (image *SImage) IsEmulated() bool { + return false +} + +func (image *SImage) GetGlobalId() string { + return image.ID +} + +func (image *SImage) Delete(ctx context.Context) error { + return image.storageCache.region.DeleteImage(image.ID) +} + +func (image *SImage) GetStatus() string { + switch image.Status { + case QUEUED, SAVING, UPLOADING, IMPORTING: + return models.CACHED_IMAGE_STATUS_CACHING + case ACTIVE: + return models.CACHED_IMAGE_STATUS_READY + case DELETED, DEACTIVATED, PENDING_DELETE, KILLED: + return models.CACHED_IMAGE_STATUS_CACHE_FAILED + default: + return models.CACHED_IMAGE_STATUS_CACHE_FAILED + } +} + +func (image *SImage) GetImageStatus() string { + switch image.Status { + case QUEUED, SAVING, UPLOADING, IMPORTING: + return cloudprovider.IMAGE_STATUS_SAVING + case ACTIVE: + return cloudprovider.IMAGE_STATUS_ACTIVE + case DELETED, DEACTIVATED, PENDING_DELETE: + return cloudprovider.IMAGE_STATUS_DELETED + case KILLED: + return cloudprovider.IMAGE_STATUS_KILLED + default: + return cloudprovider.IMAGE_STATUS_DELETED + } +} + +func (image *SImage) Refresh() error { + new, err := image.storageCache.region.GetImage(image.ID) + if err != nil { + return err + } + return jsonutils.Update(image, new) +} + +func (image *SImage) GetImageType() string { + switch image.Visibility { + case "public": + return cloudprovider.CachedImageTypeSystem + default: + return cloudprovider.CachedImageTypeCustomized + } +} + +func (image *SImage) GetSize() int64 { + return int64(image.Size) +} + +func (image *SImage) GetOsType() string { + return "Linux" +} + +func (image *SImage) GetOsDist() string { + return "Linux" +} + +func (image *SImage) GetOsVersion() string { + return "" +} + +func (image *SImage) GetOsArch() string { + return "x86_64" +} + +func (image *SImage) GetMinOsDiskSizeGb() int { + return 50 +} + +func (image *SImage) GetImageFormat() string { + return image.DiskFormat +} + +func (image *SImage) GetCreateTime() time.Time { + return image.CreatedAt +} + +func (region *SRegion) GetImage(imageId string) (*SImage, error) { + images, err := region.GetImages("", "", []string{imageId}) + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + if len(images) > 1 { + return nil, cloudprovider.ErrDuplicateId + } + return &images[0], nil +} + +func (image *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return image.storageCache +} + +func (region *SRegion) DeleteImage(imageId string) error { + _, err := region.Delete("image", "/v2/images/"+imageId, "") + return err +} + +func (region *SRegion) GetImageStatus(imageId string) (string, error) { + image, err := region.GetImage(imageId) + if err != nil { + return "", err + } + return image.Status, nil +} + +func (region *SRegion) GetImageByName(name string) (*SImage, error) { + images, err := region.GetImages(name, "", []string{}) + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &images[0], nil +} + +func (region *SRegion) CreateImage(imageName string) (*SImage, error) { + params := map[string]string{ + "container_format": "bare", + "disk_format": "vmdk", + "name": imageName, + } + + _, resp, err := region.Post("image", "/v2/images", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + image := &SImage{} + return image, resp.Unmarshal(image) +} diff --git a/pkg/util/openstack/instance.go b/pkg/util/openstack/instance.go index 523c7b8583..77ae1ae31b 100644 --- a/pkg/util/openstack/instance.go +++ b/pkg/util/openstack/instance.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/pkg/utils" ) const ( @@ -52,17 +53,6 @@ type ExtraSpecs struct { MemPageSize int `json:"hw:mem_page_size,omitempty"` } -type SFlavor struct { - ID string - Disk int - Ephemeral int - ExtraSpecs ExtraSpecs - OriginalName string - RAM int - Swap string - Vcpus int8 -} - type Resource struct { ID string Links []Link @@ -137,7 +127,7 @@ func (region *SRegion) GetSecurityGroupsByInstance(instanceId string) ([]Securit func (region *SRegion) GetInstances(zoneName string, hostName string) ([]SInstance, error) { _, maxVersion, _ := region.GetVersion("compute") - _, resp, err := region.Get("compute", "/servers/detail", maxVersion, nil) + _, resp, err := region.List("compute", "/servers/detail", maxVersion, nil) if err != nil { return nil, err } @@ -212,19 +202,18 @@ func (instance *SInstance) IsEmulated() bool { func (instance *SInstance) fetchFlavor() error { if len(instance.Flavor.ID) > 0 && instance.Flavor.Vcpus == 0 { - _, resp, err := instance.host.zone.region.Get("compute", "/flavors/"+instance.Flavor.ID, "", nil) + flavor, err := instance.host.zone.region.GetFlavor(instance.Flavor.ID) if err != nil { - log.Errorf("fetch instance %s flavor error: %v", instance.Name, err) return err } - return resp.Unmarshal(&instance.Flavor, "flavor") + instance.Flavor = *flavor } return nil } func (instance *SInstance) GetInstanceType() string { instance.fetchFlavor() - return instance.Flavor.OriginalName + return instance.Flavor.GetName() } func (instance *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { @@ -329,7 +318,16 @@ func (instance *SInstance) Refresh() error { } func (instance *SInstance) UpdateVM(ctx context.Context, name string) error { - return cloudprovider.ErrNotImplemented + if instance.Name != name { + params := map[string]map[string]string{ + "server": { + "name": name, + }, + } + _, _, err := instance.host.zone.region.Update("compute", "/servers/"+instance.ID, "", jsonutils.Marshal(params)) + return err + } + return nil } func (instance *SInstance) GetHypervisor() string { @@ -337,11 +335,17 @@ func (instance *SInstance) GetHypervisor() string { } func (instance *SInstance) StartVM(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + if err := instance.host.zone.region.StartVM(instance.ID); err != nil { + return err + } + return cloudprovider.WaitStatus(instance, models.VM_RUNNING, 10*time.Second, 8*time.Minute) } func (instance *SInstance) StopVM(ctx context.Context, isForce bool) error { - return cloudprovider.ErrNotImplemented + if err := instance.host.zone.region.StopVM(instance.ID, isForce); err != nil { + return err + } + return cloudprovider.WaitStatus(instance, models.VM_RUNNING, 10*time.Second, 8*time.Minute) } func (region *SRegion) GetInstanceVNCUrl(instanceId string) (string, error) { @@ -372,19 +376,45 @@ func (instance *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { } func (instance *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.DeployVM(instance.ID, name, password, publicKey, deleteKeypair, description) } func (instance *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - return "", cloudprovider.ErrNotImplemented + return "", instance.host.zone.region.ReplaceSystemDisk(instance.ID, imageId, passwd, publicKey, sysSizeGB) + } func (instance *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { - return cloudprovider.ErrNotImplemented + if instance.GetVcpuCount() != int8(ncpu) || instance.GetVmemSizeMB() != vmem { + flavorId, err := instance.host.zone.region.syncFlavor("", ncpu, vmem, 40) + if err != nil { + return err + } + return instance.host.zone.region.ChangeConfig(instance, flavorId) + } + return nil } func (instance *SInstance) ChangeConfig2(ctx context.Context, instanceType string) error { - return cloudprovider.ErrNotImplemented + if instance.GetInstanceType() != instanceType { + flavorId, err := instance.host.zone.region.syncFlavor(instanceType, 0, 0, 0) + if err != nil { + return err + } + return instance.host.zone.region.ChangeConfig(instance, flavorId) + } + return nil +} + +func (region *SRegion) ChangeConfig(instance *SInstance, flavorId string) error { + params := map[string]map[string]string{ + "resize": { + "flavorRef": flavorId, + }, + } + _, maxVersion, _ := region.GetVersion("compute") + _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), maxVersion, jsonutils.Marshal(params)) + return err } func (instance *SInstance) AttachDisk(ctx context.Context, diskId string) error { @@ -401,40 +431,86 @@ func (region *SRegion) CreateInstance(name string, imageId string, instanceType return "", cloudprovider.ErrNotImplemented } -func (region *SRegion) doStartVM(instanceId string) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) instanceOperation(instanceId, operate string) error { + params := jsonutils.Marshal(map[string]string{operate: ""}) + _, maxVersion, _ := region.GetVersion("compute") + _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, params) + return err } func (region *SRegion) doStopVM(instanceId string, isForce bool) error { - return cloudprovider.ErrNotImplemented + return region.instanceOperation(instanceId, "os-stop") } func (region *SRegion) doDeleteVM(instanceId string) error { - return cloudprovider.ErrNotImplemented + return region.instanceOperation(instanceId, "forceDelete") } func (region *SRegion) StartVM(instanceId string) error { - return cloudprovider.ErrNotImplemented + return region.instanceOperation(instanceId, "os-start") } func (region *SRegion) StopVM(instanceId string, isForce bool) error { - return cloudprovider.ErrNotImplemented + return region.doStopVM(instanceId, isForce) } func (region *SRegion) DeleteVM(instanceId string) error { - return cloudprovider.ErrNotImplemented + instance, err := region.GetInstance(instanceId) + if err != nil { + if err == cloudprovider.ErrNotFound { + return nil + } + log.Errorf("failed to get instance %s %v", instanceId, err) + return err + } + status := instance.GetStatus() + log.Debugf("Instance status on delete is %s", status) + if status != models.VM_READY { + log.Warningf("DeleteVM: vm status is %s expect %s", status, models.VM_READY) + } + return region.doDeleteVM(instanceId) } func (region *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error { - return cloudprovider.ErrNotImplemented + if len(password) > 0 { + params := map[string]map[string]string{ + "changePassword": { + "adminPass": password, + }, + } + _, maxVersion, _ := region.GetVersion("compute") + _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, jsonutils.Marshal(params)) + return err + } + return nil } func (instance *SInstance) DeleteVM(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.DeleteVM(instance.ID) } -func (region *SRegion) ReplaceSystemDisk(instanceId string, imageId string, passwd string, keypairName string, sysDiskSizeGB int) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) ReplaceSystemDisk(instanceId string, imageId string, passwd string, publicKey string, sysDiskSizeGB int) error { + params := map[string]map[string]string{ + "rebuild": { + "imageRef": imageId, + }, + } + + if len(publicKey) > 0 { + keypairName, err := region.syncKeypair(instanceId, publicKey) + if err != nil { + return err + } + params["rebuild"]["key_name"] = keypairName + } + + if len(passwd) > 0 { + params["rebuild"]["adminPass"] = passwd + } + + _, maxVersion, _ := region.GetVersion("compute") + _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, jsonutils.Marshal(params)) + return err } func (region *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { @@ -446,27 +522,78 @@ func (region *SRegion) ChangeVMConfig2(zoneId string, instanceId string, instanc } func (region *SRegion) DetachDisk(instanceId string, diskId string) error { - return cloudprovider.ErrNotImplemented + _, err := region.Delete("compute", fmt.Sprintf("/servers/%s/os-volume_attachments/%s", instanceId, diskId), "") + return err } func (region *SRegion) AttachDisk(instanceId string, diskId string) error { - return cloudprovider.ErrNotImplemented + params := map[string]map[string]string{ + "volumeAttachment": { + "volumeId": diskId, + }, + } + _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/os-volume_attachments", instanceId), "", jsonutils.Marshal(params)) + return err } func (instance *SInstance) AssignSecurityGroup(secgroupId string) error { - return cloudprovider.ErrNotImplemented + secgroup, err := instance.host.zone.region.GetSecurityGroup(secgroupId) + if err != nil { + return err + } + params := map[string]map[string]string{ + "addSecurityGroup": { + "name": secgroup.Name, + }, + } + _, _, err = instance.host.zone.region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), "", jsonutils.Marshal(params)) + return err } -func (instance *SInstance) AssignSecurityGroups(secgroupIds []string) error { - return cloudprovider.ErrNotImplemented +func (instance *SInstance) RevokeSecurityGroup(secgroupId string) error { + secgroup, err := instance.host.zone.region.GetSecurityGroup(secgroupId) + if err != nil { + return err + } + params := map[string]map[string]string{ + "removeSecurityGroup": { + "name": secgroup.Name, + }, + } + _, _, err = instance.host.zone.region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), "", jsonutils.Marshal(params)) + return err +} + +func (instance *SInstance) SetSecurityGroups(secgroupIds []string) error { + secgroups, err := instance.host.zone.region.GetSecurityGroupsByInstance(instance.ID) + if err != nil { + return err + } + originIds := []string{} + for _, secgroup := range secgroups { + if !utils.IsInStringArray(secgroup.ID, secgroupIds) { + if err := instance.RevokeSecurityGroup(secgroup.ID); err != nil { + return err + } + } + originIds = append(originIds, secgroup.ID) + } + for _, secgroupId := range secgroupIds { + if !utils.IsInStringArray(secgroupId, originIds) { + if err := instance.AssignSecurityGroup(secgroupId); err != nil { + return err + } + } + } + return nil } func (instance *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { - return nil, cloudprovider.ErrNotImplemented + return nil, cloudprovider.ErrNotSupported } func (instance *SInstance) GetBillingType() string { - return models.BILLING_TYPE_PREPAID + return models.BILLING_TYPE_POSTPAID } func (instance *SInstance) GetExpiredAt() time.Time { @@ -482,9 +609,9 @@ func (instance *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid stri } func (instance *SInstance) Renew(bc billing.SBillingCycle) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } func (region *SRegion) RenewInstances(instanceId []string, bc billing.SBillingCycle) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } diff --git a/pkg/util/openstack/keypaire.go b/pkg/util/openstack/keypaire.go new file mode 100644 index 0000000000..10f68d414f --- /dev/null +++ b/pkg/util/openstack/keypaire.go @@ -0,0 +1,92 @@ +package openstack + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/version" + "yunion.io/x/pkg/utils" +) + +type SKeypair struct { + Fingerprint string + Name string + Type string + PublicKey string +} + +type SKeyPair struct { + Keypair SKeypair +} + +func (region *SRegion) GetKeypairs() ([]SKeyPair, error) { + _, resp, err := region.List("compute", "/os-keypairs", "", nil) + if err != nil { + return nil, err + } + keypairs := []SKeyPair{} + return keypairs, resp.Unmarshal(&keypairs, "keypairs") +} + +func (region *SRegion) CreateKeypair(name, publicKey, Type string) (*SKeyPair, error) { + if len(Type) > 0 && !utils.IsInStringArray(Type, []string{"ssh", "x509"}) { + return nil, fmt.Errorf("only support ssh or x509 type") + } + params := map[string]map[string]string{ + "keypair": { + "name": name, + "public_key": publicKey, + }, + } + _, maxVersion, _ := region.GetVersion("compute") + if len(Type) > 0 && version.GE(maxVersion, "2.2") { + params["keypair"]["type"] = Type + } + _, resp, err := region.Post("compute", "/os-keypairs", maxVersion, jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + keypair := &SKeyPair{} + return keypair, resp.Unmarshal(keypair) +} + +func (region *SRegion) DeleteKeypair(name string) error { + _, err := region.Delete("compute", "/os-keypairs/"+name, "") + return err +} + +func (region *SRegion) GetKeypair(name string) (*SKeyPair, error) { + _, resp, err := region.Get("compute", "/os-keypairs/"+name, "", nil) + if err != nil { + return nil, err + } + keypair := &SKeyPair{} + return keypair, resp.Unmarshal(keypair) +} + +func (region *SRegion) syncKeypair(namePrefix, publicKey string) (string, error) { + keypairs, err := region.GetKeypairs() + if err != nil { + return "", err + } + + for _, keypair := range keypairs { + if keypair.Keypair.PublicKey == publicKey { + return keypair.Keypair.Name, nil + } + } + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%s-%d", namePrefix, i) + if _, err := region.GetKeypair(name); err != nil { + if err == cloudprovider.ErrNotFound { + keypair, err := region.CreateKeypair(name, publicKey, "ssh") + if err != nil { + return "", err + } + return keypair.Keypair.Name, nil + } + } + } + return "", fmt.Errorf("failed to find uniq name for keypair") +} diff --git a/pkg/util/openstack/network.go b/pkg/util/openstack/network.go index b7891dc572..43512a7e3a 100644 --- a/pkg/util/openstack/network.go +++ b/pkg/util/openstack/network.go @@ -73,8 +73,9 @@ func (network *SNetwork) Delete() error { return network.wire.zone.region.DeleteNetwork(network.ID) } -func (network *SRegion) DeleteNetwork(networkId string) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) DeleteNetwork(networkId string) error { + _, err := region.Delete("network", "/v2.0/subnets/"+networkId, "") + return err } func (network *SNetwork) GetIWire() cloudprovider.ICloudWire { @@ -126,7 +127,7 @@ func (region *SRegion) GetNetwork(networkId string) (*SNetwork, error) { } func (region *SRegion) GetNetworks(vpcId string) ([]SNetwork, error) { - _, resp, err := region.Get("network", "/v2.0/subnets", "", nil) + _, resp, err := region.List("network", "/v2.0/subnets", "", nil) if err != nil { return nil, err } @@ -152,6 +153,19 @@ func (network *SNetwork) Refresh() error { return jsonutils.Update(network, new) } -func (network *SRegion) CreateNetwork(zoneId string, vpcId string, name string, cidr string, desc string) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (region *SRegion) CreateNetwork(vpcId string, name string, cidr string, desc string) (string, error) { + params := map[string]map[string]interface{}{ + "subnet": { + "name": name, + "network_id": vpcId, + "cidr": cidr, + "description": desc, + "ip_version": 4, + }, + } + _, resp, err := region.Post("network", "/v2.0/subnets", "", jsonutils.Marshal(params)) + if err != nil { + return "", err + } + return resp.GetString("subnet", "id") } diff --git a/pkg/util/openstack/openstack.go b/pkg/util/openstack/openstack.go index 8dd02d8b7f..c5c71f40f1 100644 --- a/pkg/util/openstack/openstack.go +++ b/pkg/util/openstack/openstack.go @@ -3,11 +3,12 @@ package openstack import ( "context" "fmt" + "io" "net/http" "strings" "yunion.io/x/jsonutils" - + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" @@ -68,7 +69,37 @@ func (cli *SOpenStackClient) Request(region, service, method string, url string, } ctx := context.Background() session := cli.client.NewSession(ctx, region, "", "internal", cli.tokenCredential, "") - return session.JSONRequest(service, "", httputils.THttpMethod(method), url, header, body) + header, resp, err := session.JSONRequest(service, "", httputils.THttpMethod(method), url, header, body) + if err != nil && body != nil { + uri, _ := session.GetServiceURL(service, "") + log.Errorf("microversion %s url: %s, params: %s", microversion, uri+url, body.PrettyString()) + } + return header, resp, err +} + +func (cli *SOpenStackClient) RawRequest(region, service, method string, url string, microversion string, body jsonutils.JSONObject) (*http.Response, error) { + header := http.Header{} + if len(microversion) > 0 { + header.Set("X-Openstack-Nova-API-Version", microversion) + } + ctx := context.Background() + session := cli.client.NewSession(ctx, region, "", "internal", cli.tokenCredential, "") + data := strings.NewReader("") + if body != nil { + data = strings.NewReader(body.String()) + } + return session.RawRequest(service, "", httputils.THttpMethod(method), url, header, data) +} + +func (cli *SOpenStackClient) StreamRequest(region, service, method string, url string, microversion string, body io.Reader) (*http.Response, error) { + header := http.Header{} + if len(microversion) > 0 { + header.Set("X-Openstack-Nova-API-Version", microversion) + } + header.Set("Content-Type", "application/octet-stream") + ctx := context.Background() + session := cli.client.NewSession(ctx, region, "", "internal", cli.tokenCredential, "") + return session.RawRequest(service, "", httputils.THttpMethod(method), url, header, body) } func (cli *SOpenStackClient) getVersion(region string, service string) (string, string, error) { diff --git a/pkg/util/openstack/port.go b/pkg/util/openstack/port.go index 13b8923d2e..68841312fb 100644 --- a/pkg/util/openstack/port.go +++ b/pkg/util/openstack/port.go @@ -60,7 +60,7 @@ func (region *SRegion) GetPorts(macAddress string) ([]SPort, error) { params.Set("mac_address", macAddress) } url := fmt.Sprintf("%s?%s", base, params.Encode()) - _, resp, err := region.Get("network", url, "", nil) + _, resp, err := region.List("network", url, "", nil) if err != nil { return nil, err } diff --git a/pkg/util/openstack/provider/provider.go b/pkg/util/openstack/provider/provider.go index 44f5067de8..d6315ec53f 100644 --- a/pkg/util/openstack/provider/provider.go +++ b/pkg/util/openstack/provider/provider.go @@ -41,6 +41,10 @@ func (self *SOpenStackProviderFactory) IsSupportPrepaidResources() bool { return false } +func (self *SOpenStackProviderFactory) NeedSyncSkuFromCloud() bool { + return true +} + func (self *SOpenStackProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { projectName, _ := data.GetString("project_name") if len(projectName) == 0 { diff --git a/pkg/util/openstack/quota.go b/pkg/util/openstack/quota.go new file mode 100644 index 0000000000..a911749590 --- /dev/null +++ b/pkg/util/openstack/quota.go @@ -0,0 +1,61 @@ +package openstack + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/util/version" +) + +type SQuota struct { + FixedIps int + Floatingips int + Networks int + Port int + RbacPolicy int + Router int + SecurityGroups int + SecurityGroupRules int +} + +func (region *SRegion) GetQuota() (*SQuota, error) { + _, resp, err := region.Get("compute", "/os-quota-sets/"+region.client.tokenCredential.GetTenantId(), "", nil) + if err != nil { + return nil, err + } + quota := &SQuota{} + return quota, resp.Unmarshal(quota, "quota_set") +} + +func (region *SRegion) SetQuota(quota *SQuota) error { + _, maxVersion, _ := region.GetVersion("compute") + params := map[string]map[string]interface{}{ + "quota_set": { + "force": "True", + }, + } + + if version.GE(maxVersion, "2.35") { + if quota.Floatingips > 0 { + params["quota_set"]["floating_ips"] = quota.Floatingips + } + + if quota.SecurityGroups > 0 { + params["quota_set"]["security_group"] = quota.SecurityGroups + } + + if quota.SecurityGroupRules > 0 { + params["quota_set"]["security_group_rules"] = quota.SecurityGroupRules + } + + if quota.FixedIps > 0 { + params["quota_set"]["fixed_ips"] = quota.FixedIps + } + + if quota.Networks > 0 { + params["quota_set"]["networks"] = quota.Networks + } + + } + + _, _, err := region.Update("compute", "/os-quota-sets/"+region.client.tokenCredential.GetTenantId(), maxVersion, jsonutils.Marshal(params)) + return err +} diff --git a/pkg/util/openstack/region.go b/pkg/util/openstack/region.go index adfedb0131..cc194037aa 100644 --- a/pkg/util/openstack/region.go +++ b/pkg/util/openstack/region.go @@ -3,11 +3,13 @@ package openstack import ( "fmt" "net/http" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SRegion struct { @@ -63,7 +65,25 @@ func (region *SRegion) Refresh() error { } func (region *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) { - return nil, cloudprovider.ErrNotImplemented + params := map[string]map[string]string{ + "network": { + "name": name, + "description": desc, + }, + } + _, resp, err := region.Post("network", "/v2.0/networks", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + err = region.fetchInfrastructure() + if err != nil { + return nil, err + } + vpcId, err := resp.GetString("network", "id") + if err != nil { + return nil, err + } + return region.GetIVpcById(vpcId) } func (region *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) { @@ -173,7 +193,7 @@ func (region *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) } func (region *SRegion) fetchZones() error { - _, resp, err := region.Get("compute", "/os-availability-zone", "", jsonutils.NewDict()) + _, resp, err := region.List("compute", "/os-availability-zone", "", jsonutils.NewDict()) if err != nil { return err } @@ -227,24 +247,107 @@ func (region *SRegion) fetchInfrastructure() error { } func (region *SRegion) Get(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - return region.client.Request(region.Name, service, "GET", url, microversion, body) + if strings.HasSuffix(url, "/") { + return nil, nil, cloudprovider.ErrNotFound + } + header, resp, err := region.client.Request(region.Name, service, "GET", url, microversion, body) + if err != nil { + if jsonErr, ok := err.(*httputils.JSONClientError); ok { + if jsonErr.Code == 404 || strings.HasSuffix(jsonErr.Class, "NotFound") { + return nil, nil, cloudprovider.ErrNotFound + } + } + return nil, nil, err + } + return header, resp, nil +} + +func (region *SRegion) List(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + header, resp, err := region.client.Request(region.Name, service, "GET", url, microversion, body) + if err != nil { + if jsonErr, ok := err.(*httputils.JSONClientError); ok { + if jsonErr.Code == 404 || strings.HasSuffix(jsonErr.Class, "NotFound") { + return nil, nil, cloudprovider.ErrNotFound + } + } + return nil, nil, err + } + return header, resp, nil } func (region *SRegion) Post(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { return region.client.Request(region.Name, service, "POST", url, microversion, body) } -func (region *SRegion) CinderGet(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { +func (region *SRegion) Update(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + return region.client.Request(region.Name, service, "PUT", url, microversion, body) +} + +func (region *SRegion) Delete(service, url string, microversion string) (*http.Response, error) { + return region.client.RawRequest(region.Name, service, "DELETE", url, microversion, nil) +} + +func (region *SRegion) CinderList(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { for _, service := range []string{"volumev3", "volumev2", "volume"} { header, resp, err := region.Get(service, url, microversion, body) if err == nil { return header, resp, nil } + log.Debugf("failed to list %s by service %s error: %v, try another", url, service, err) + } + return nil, nil, fmt.Errorf("failed to get %s by cinder service", url) +} + +func (region *SRegion) CinderGet(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + if strings.HasSuffix(url, "/") { + return nil, nil, cloudprovider.ErrNotFound + } + for _, service := range []string{"volumev3", "volumev2", "volume"} { + header, resp, err := region.Get(service, url, microversion, body) + if err == nil || err == cloudprovider.ErrNotFound { + return header, resp, err + } log.Debugf("failed to get %s by service %s error: %v, try another", url, service, err) } return nil, nil, fmt.Errorf("failed to get %s by cinder service", url) } +func (region *SRegion) CinderCreate(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + for _, service := range []string{"volumev3", "volumev2", "volume"} { + header, resp, err := region.Post(service, url, microversion, body) + if err == nil { + return header, resp, nil + } + log.Debugf("failed to create %s by service %s error: %v, try another", url, service, err) + } + return nil, nil, fmt.Errorf("failed to create %s by cinder service", url) +} + +func (region *SRegion) CinderDelete(url string, microversion string) (*http.Response, error) { + if strings.HasSuffix(url, "/") { + return nil, cloudprovider.ErrNotFound + } + for _, service := range []string{"volumev3", "volumev2", "volume"} { + resp, err := region.Delete(service, url, microversion) + if err == nil { + return resp, nil + } + log.Debugf("failed to delete %s by service %s error: %v, try another", url, service, err) + } + return nil, fmt.Errorf("failed to delete %s by cinder service", url) +} + +func (region *SRegion) CinderAction(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + for _, service := range []string{"volumev3", "volumev2", "volume"} { + header, resp, err := region.Post(service, url, microversion, body) + if err == nil { + return header, resp, nil + } + log.Debugf("failed to operate %s by service %s error: %v, try another", url, service, err) + } + return nil, nil, fmt.Errorf("failed to operate %s by cinder service", url) +} + func (region *SRegion) ProjectId() string { return region.client.tokenCredential.GetProjectId() } @@ -270,7 +373,7 @@ func (region *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) { } func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { - return nil, cloudprovider.ErrNotImplemented + return nil, cloudprovider.ErrNotSupported } func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { @@ -316,3 +419,16 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { return nil, cloudprovider.ErrNotImplemented } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + flavors, err := region.GetFlavors() + if err != nil { + return nil, err + } + iskus := make([]cloudprovider.ICloudSku, len(flavors)) + for i := 0; i < len(flavors); i++ { + flavors[i].region = region + iskus[i] = &flavors[i] + } + return iskus, nil +} diff --git a/pkg/util/openstack/securitygroup.go b/pkg/util/openstack/securitygroup.go index f737a7c6e8..b90b112e48 100644 --- a/pkg/util/openstack/securitygroup.go +++ b/pkg/util/openstack/securitygroup.go @@ -1,11 +1,14 @@ package openstack import ( + "fmt" "net" + "sort" "strings" "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" @@ -45,6 +48,20 @@ type SSecurityGroup struct { TenantID string } +type SecurigyGroupRuleSet []SSecurityGroupRule + +func (v SecurigyGroupRuleSet) Len() int { + return len(v) +} + +func (v SecurigyGroupRuleSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurigyGroupRuleSet) Less(i, j int) bool { + return strings.Compare(v[i].String(), v[j].String()) <= 0 +} + func (region *SRegion) GetSecurityGroup(secgroupId string) (*SSecurityGroup, error) { _, resp, err := region.Get("network", "/v2.0/security-groups/"+secgroupId, "", nil) if err != nil { @@ -55,7 +72,7 @@ func (region *SRegion) GetSecurityGroup(secgroupId string) (*SSecurityGroup, err } func (region *SRegion) GetSecurityGroups() ([]SSecurityGroup, error) { - _, resp, err := region.Get("network", "/v2.0/security-groups", "", nil) + _, resp, err := region.List("network", "/v2.0/security-groups", "", nil) if err != nil { return nil, err } @@ -172,14 +189,165 @@ func (secgroup *SSecurityGroup) Refresh() error { return jsonutils.Update(secgroup, new) } -func (secgroup *SRegion) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (region *SRegion) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) { + if len(secgroupId) > 0 { + _, err := region.GetSecurityGroup(secgroupId) + if err != nil { + if err != cloudprovider.ErrNotFound { + return "", err + } + secgroupId = "" + } + } + if len(secgroupId) == 0 { + secgroups, err := region.GetSecurityGroups() + if err != nil { + return "", err + } + + secgroupNames := []string{} + for _, secgroup := range secgroups { + secgroupNames = append(secgroupNames, strings.ToLower(secgroup.Name)) + } + + uniqName := strings.ToLower(name) + if utils.IsInStringArray(uniqName, secgroupNames) { + for i := 0; i < 20; i++ { + uniqName = fmt.Sprintf("%s-%d", strings.ToLower(name), i) + if !utils.IsInStringArray(uniqName, secgroupNames) { + break + } + } + } + log.Errorf("create secgroup %s", uniqName) + secgroup, err := region.CreateSecurityGroup(uniqName, desc) + if err != nil { + return "", err + } + secgroupId = secgroup.ID + } + return region.syncSecgroupRules(secgroupId, rules) +} + +func (region *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) (string, error) { + secgroup, err := region.GetSecurityGroup(secgroupId) + if err != nil { + return "", err + } + + sort.Sort(secrules.SecurityRuleSet(rules)) + sort.Sort(SecurigyGroupRuleSet(secgroup.SecurityGroupRules)) + + delSecgroupRuleIds := []string{} + addSecgroupRules := []secrules.SecurityRule{} + addSecgroupRuleStrings := []string{} + + i, j := 0, 0 + for i < len(rules) || j < len(secgroup.SecurityGroupRules) { + if i < len(rules) && j < len(secgroup.SecurityGroupRules) { + secruleStr := secgroup.SecurityGroupRules[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(secruleStr, ruleStr) + if cmp == 0 { + i++ + j++ + } else if cmp > 0 { + delSecgroupRuleIds = append(delSecgroupRuleIds, secgroup.SecurityGroupRules[j].ID) + j++ + } else { + if !utils.IsInStringArray(ruleStr, addSecgroupRuleStrings) { + addSecgroupRules = append(addSecgroupRules, rules[i]) + addSecgroupRuleStrings = append(addSecgroupRuleStrings, ruleStr) + } + i++ + } + } else if i >= len(rules) { + delSecgroupRuleIds = append(delSecgroupRuleIds, secgroup.SecurityGroupRules[j].ID) + j++ + } else if j >= len(secgroup.SecurityGroupRules) { + ruleStr := rules[i].String() + if !utils.IsInStringArray(ruleStr, addSecgroupRuleStrings) { + addSecgroupRules = append(addSecgroupRules, rules[i]) + addSecgroupRuleStrings = append(addSecgroupRuleStrings, ruleStr) + } + i++ + } + } + + for _, ruleId := range delSecgroupRuleIds { + if err := region.delSecurityGroupRule(ruleId); err != nil { + log.Errorf("delSecurityGroupRule error %v", err) + return "", err + } + } + for i := 0; i < len(addSecgroupRules); i++ { + if err := region.addSecurityGroupRules(secgroupId, &addSecgroupRules[i]); err != nil { + log.Errorf("addSecurityGroupRule error %v", rules[i]) + return "", err + } + } + + return secgroupId, nil +} + +func (region *SRegion) delSecurityGroupRule(ruleId string) error { + _, err := region.Delete("network", "/v2.0/security-group-rules/"+ruleId, "") + return err +} + +func (region *SRegion) addSecurityGroupRules(secgroupId string, rule *secrules.SecurityRule) error { + direction := "ingress" + if rule.Direction == secrules.SecurityRuleEgress { + direction = "egress" + } + + if rule.Protocol == secrules.PROTO_ANY { + rule.Protocol = "0" + } + + params := map[string]map[string]interface{}{ + "security_group_rule": { + "direction": direction, + "protocol": rule.Protocol, + "security_group_id": secgroupId, + "remote_ip_prefix": rule.IPNet.String(), + }, + } + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + params["security_group_rule"]["port_range_max"] = port + params["security_group_rule"]["port_range_min"] = port + _, _, err := region.Post("network", "/v2.0/security-group-rules", "", jsonutils.Marshal(params)) + if err != nil { + return err + } + } + return nil + } + if rule.PortEnd > 0 && rule.PortStart > 0 { + params["security_group_rule"]["port_range_min"] = rule.PortStart + params["security_group_rule"]["port_range_max"] = rule.PortEnd + } + _, _, err := region.Post("network", "/v2.0/security-group-rules", "", jsonutils.Marshal(params)) + return err } func (region *SRegion) DeleteSecurityGroup(vpcId, secGroupId string) error { - return cloudprovider.ErrNotImplemented + _, err := region.Delete("network", "/v2.0/security-groups/"+secGroupId, "") + return err } func (region *SRegion) CreateSecurityGroup(name, description string) (*SSecurityGroup, error) { - return nil, cloudprovider.ErrNotImplemented + params := map[string]map[string]interface{}{ + "security_group": { + "name": name, + "description": description, + }, + } + _, resp, err := region.Post("network", "/v2.0/security-groups", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + secgroup := &SSecurityGroup{} + return secgroup, resp.Unmarshal(secgroup, "security_group") } diff --git a/pkg/util/openstack/shell/disk.go b/pkg/util/openstack/shell/disk.go index fd66d361c0..dcb2b29abc 100644 --- a/pkg/util/openstack/shell/disk.go +++ b/pkg/util/openstack/shell/disk.go @@ -18,11 +18,11 @@ func init() { return nil }) - type DiskShowOptions struct { - ID string `help:"Storage type for disk"` + type DiskOptions struct { + ID string `help:"ID of disk"` } - shellutils.R(&DiskShowOptions{}, "disk-show", "Show disk", func(cli *openstack.SRegion, args *DiskShowOptions) error { + shellutils.R(&DiskOptions{}, "disk-show", "Show disk", func(cli *openstack.SRegion, args *DiskOptions) error { disk, err := cli.GetDisk(args.ID) if err != nil { return err @@ -30,4 +30,43 @@ func init() { printObject(disk) return nil }) + + shellutils.R(&DiskOptions{}, "disk-delete", "Delete disk", func(cli *openstack.SRegion, args *DiskOptions) error { + return cli.DeleteDisk(args.ID) + }) + + type DiskCreateOptions struct { + ZONE string `help:"Zone name"` + CATEGORY string `help:"Disk category"` + NAME string `help:"Disk Name"` + SIZE int `help:"Disk Size GB"` + Desc string `help:"Description of disk"` + } + shellutils.R(&DiskCreateOptions{}, "disk-create", "Create disk", func(cli *openstack.SRegion, args *DiskCreateOptions) error { + disk, err := cli.CreateDisk(args.ZONE, args.CATEGORY, args.NAME, args.SIZE, args.Desc) + if err != nil { + return err + } + printObject(disk) + return nil + }) + + type DiskResetOptions struct { + DISK string `help:"ID of disk"` + SNAPSHOT string `help:"ID of snapshot"` + } + + shellutils.R(&DiskResetOptions{}, "disk-reset", "Reset disk", func(cli *openstack.SRegion, args *DiskResetOptions) error { + return cli.ResetDisk(args.DISK, args.SNAPSHOT) + }) + + type DiskResizeOptions struct { + DISK string `help:"ID of disk"` + SIZE int64 `help:"Disk size GB"` + } + + shellutils.R(&DiskResizeOptions{}, "disk-resize", "Resize disk", func(cli *openstack.SRegion, args *DiskResizeOptions) error { + return cli.ResizeDisk(args.DISK, args.SIZE*1024) + }) + } diff --git a/pkg/util/openstack/shell/flavor.go b/pkg/util/openstack/shell/flavor.go new file mode 100644 index 0000000000..dc83807c5f --- /dev/null +++ b/pkg/util/openstack/shell/flavor.go @@ -0,0 +1,53 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/openstack" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type FlavorkListOptions struct { + } + shellutils.R(&FlavorkListOptions{}, "flavor-list", "List flavors", func(cli *openstack.SRegion, args *FlavorkListOptions) error { + flavors, err := cli.GetFlavors() + if err != nil { + return err + } + printList(flavors, 0, 0, 0, []string{}) + return nil + }) + + type FlavorOptions struct { + ID string `help:"ID of flavor"` + } + + shellutils.R(&FlavorOptions{}, "flavor-show", "Show flavor", func(cli *openstack.SRegion, args *FlavorOptions) error { + flavor, err := cli.GetFlavor(args.ID) + if err != nil { + return err + } + printObject(flavor) + return nil + }) + + shellutils.R(&FlavorOptions{}, "flavor-delete", "Delete flavor", func(cli *openstack.SRegion, args *FlavorOptions) error { + return cli.DeleteFlavor(args.ID) + }) + + type FlavorCreateOptions struct { + NAME string `help:"Name of flavor"` + CPU int `help:"Core num of cpu"` + MEMORY_MB int `help:"Memory of flavor"` + DISK int `help:"Disk size of flavor"` + } + + shellutils.R(&FlavorCreateOptions{}, "flavor-create", "Create flavor", func(cli *openstack.SRegion, args *FlavorCreateOptions) error { + flavor, err := cli.CreateFlavor(args.NAME, args.CPU, args.MEMORY_MB, args.DISK) + if err != nil { + return err + } + printObject(flavor) + return nil + }) + +} diff --git a/pkg/util/openstack/shell/image.go b/pkg/util/openstack/shell/image.go new file mode 100644 index 0000000000..7e2371e743 --- /dev/null +++ b/pkg/util/openstack/shell/image.go @@ -0,0 +1,53 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/openstack" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ImageListOptions struct { + Name string + Ids []string + Status string + } + shellutils.R(&ImageListOptions{}, "image-list", "List images", func(cli *openstack.SRegion, args *ImageListOptions) error { + images, err := cli.GetImages(args.Name, args.Status, args.Ids) + if err != nil { + return err + } + printList(images, 0, 0, 0, []string{}) + return nil + }) + + type ImageOptions struct { + ID string + } + + shellutils.R(&ImageOptions{}, "image-show", "Show image", func(cli *openstack.SRegion, args *ImageOptions) error { + image, err := cli.GetImages("", "", []string{args.ID}) + if err != nil { + return err + } + printObject(image[0]) + return nil + }) + + shellutils.R(&ImageOptions{}, "image-delete", "Delete image", func(cli *openstack.SRegion, args *ImageOptions) error { + return cli.DeleteImage(args.ID) + }) + + type ImageCreateOptions struct { + NAME string + } + + shellutils.R(&ImageCreateOptions{}, "image-create", "Create image", func(cli *openstack.SRegion, args *ImageCreateOptions) error { + image, err := cli.CreateImage(args.NAME) + if err != nil { + return err + } + printObject(image) + return nil + }) + +} diff --git a/pkg/util/openstack/shell/instance.go b/pkg/util/openstack/shell/instance.go index fa00647313..0d1663045e 100644 --- a/pkg/util/openstack/shell/instance.go +++ b/pkg/util/openstack/shell/instance.go @@ -42,4 +42,23 @@ func init() { return nil }) + type InstanceDeployOptions struct { + ID string `help:"Instance ID"` + Password string `help:"Instance password"` + Name string `help:"Instance name"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy instance", func(cli *openstack.SRegion, args *InstanceDeployOptions) error { + return cli.DeployVM(args.ID, args.Name, args.Password, "", false, "") + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"Instance ID"` + FLAVOR_ID string `help:"Flavor ID"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Change instance config", func(cli *openstack.SRegion, args *InstanceChangeConfigOptions) error { + return cli.ChageConfig(args.ID, args.FLAVOR_ID) + }) + } diff --git a/pkg/util/openstack/shell/keypair.go b/pkg/util/openstack/shell/keypair.go new file mode 100644 index 0000000000..8916f75e7c --- /dev/null +++ b/pkg/util/openstack/shell/keypair.go @@ -0,0 +1,52 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/openstack" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type KeypairListOptions struct { + } + shellutils.R(&KeypairListOptions{}, "keypair-list", "List keypairs", func(cli *openstack.SRegion, args *KeypairListOptions) error { + keypairs, err := cli.GetKeypairs() + if err != nil { + return err + } + printList(keypairs, 0, 0, 0, []string{}) + return nil + }) + + type KeypairCreateOptions struct { + NAME string + PublicKey string + Type string `help:"keypair type" choices:"ssh|x509"` + } + + shellutils.R(&KeypairCreateOptions{}, "keypair-create", "Create keypair", func(cli *openstack.SRegion, args *KeypairCreateOptions) error { + keypair, err := cli.CreateKeypair(args.NAME, args.PublicKey, args.Type) + if err != nil { + return err + } + printObject(keypair) + return nil + }) + + type KeypairOptions struct { + NAME string `help:"Keypair name"` + } + + shellutils.R(&KeypairOptions{}, "keypair-show", "Show keypair", func(cli *openstack.SRegion, args *KeypairOptions) error { + keypair, err := cli.GetKeypair(args.NAME) + if err != nil { + return err + } + printObject(keypair) + return nil + }) + + shellutils.R(&KeypairOptions{}, "keypair-delete", "Delete keypair", func(cli *openstack.SRegion, args *KeypairOptions) error { + return cli.DeleteKeypair(args.NAME) + }) + +} diff --git a/pkg/util/openstack/shell/quota.go b/pkg/util/openstack/shell/quota.go new file mode 100644 index 0000000000..485fad384d --- /dev/null +++ b/pkg/util/openstack/shell/quota.go @@ -0,0 +1,24 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/openstack" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type QuotaOptions struct { + } + shellutils.R(&QuotaOptions{}, "quota-show", "Show quota", func(cli *openstack.SRegion, args *QuotaOptions) error { + quota, err := cli.GetQuota() + if err != nil { + return err + } + printObject(quota) + return nil + }) + + shellutils.R(&openstack.SQuota{}, "quota-set", "Set quota", func(cli *openstack.SRegion, args *openstack.SQuota) error { + return cli.SetQuota(args) + }) + +} diff --git a/pkg/util/openstack/shell/securitygroup.go b/pkg/util/openstack/shell/securitygroup.go index ca85e82b7b..2c3470f708 100644 --- a/pkg/util/openstack/shell/securitygroup.go +++ b/pkg/util/openstack/shell/securitygroup.go @@ -29,4 +29,18 @@ func init() { return nil }) + type SecurityGroupCreateOptions struct { + NAME string `help:"Name of security group"` + Desc string `help:"Description of security group"` + } + + shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *openstack.SRegion, args *SecurityGroupCreateOptions) error { + secgroup, err := cli.CreateSecurityGroup(args.NAME, args.Desc) + if err != nil { + return err + } + printObject(secgroup) + return nil + }) + } diff --git a/pkg/util/openstack/shell/snapshot.go b/pkg/util/openstack/shell/snapshot.go index 79097fa691..1f92e68548 100644 --- a/pkg/util/openstack/shell/snapshot.go +++ b/pkg/util/openstack/shell/snapshot.go @@ -18,11 +18,11 @@ func init() { return nil }) - type SnapshotShowOptions struct { + type SnapshotOptions struct { ID string `help:"ID of snapshot"` } - shellutils.R(&SnapshotShowOptions{}, "snapshot-show", "Show snapshot", func(cli *openstack.SRegion, args *SnapshotShowOptions) error { + shellutils.R(&SnapshotOptions{}, "snapshot-show", "Show snapshot", func(cli *openstack.SRegion, args *SnapshotOptions) error { snapshot, err := cli.GetISnapshotById(args.ID) if err != nil { return err @@ -31,4 +31,23 @@ func init() { return nil }) + shellutils.R(&SnapshotOptions{}, "snapshot-delete", "Delete snapshot", func(cli *openstack.SRegion, args *SnapshotOptions) error { + return cli.DeleteSnapshot(args.ID) + }) + + type SnapshotCreateOptions struct { + DISKID string `help:"Disk ID"` + Name string `help:"Disk Name"` + Desc string `help:"Disk description"` + } + + shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *openstack.SRegion, args *SnapshotCreateOptions) error { + snapshot, err := cli.CreateSnapshot(args.DISKID, args.Name, args.Desc) + if err != nil { + return err + } + printObject(snapshot) + return nil + }) + } diff --git a/pkg/util/openstack/snapshot.go b/pkg/util/openstack/snapshot.go index 090aa76d94..265daeef8e 100644 --- a/pkg/util/openstack/snapshot.go +++ b/pkg/util/openstack/snapshot.go @@ -43,7 +43,7 @@ func (region *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloud if err != nil { return nil, err } - snapshot := SSnapshot{} + snapshot := SSnapshot{region: region} if err := resp.Unmarshal(&snapshot, "snapshot"); err != nil { return nil, err } @@ -78,7 +78,7 @@ func (snapshot *SSnapshot) Refresh() error { } func (region *SRegion) GetSnapshots(diskId string) ([]cloudprovider.ICloudSnapshot, error) { - _, resp, err := region.CinderGet("/snapshots/detail", "", nil) + _, resp, err := region.CinderList("/snapshots/detail", "", nil) if err != nil { return nil, err } @@ -121,6 +121,9 @@ func (snapshot *SSnapshot) GetGlobalId() string { } func (snapshot *SSnapshot) GetName() string { + if len(snapshot.Name) == 0 { + return snapshot.ID + } return snapshot.Name } @@ -139,10 +142,24 @@ func (snapshot *SSnapshot) GetDiskType() string { return models.DISK_TYPE_DATA } -func (snapshot *SRegion) DeleteSnapshot(snapshotId string) error { - return cloudprovider.ErrNotImplemented +func (region *SRegion) DeleteSnapshot(snapshotId string) error { + _, err := region.CinderDelete("/snapshots/"+snapshotId, "") + return err } -func (snapshot *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (region *SRegion) CreateSnapshot(diskId, name, desc string) (*SSnapshot, error) { + params := map[string]map[string]interface{}{ + "snapshot": { + "volume_id": diskId, + "name": name, + "description": desc, + "force": true, + }, + } + _, resp, err := region.CinderCreate("/snapshots", "", jsonutils.Marshal(params)) + if err != nil { + return nil, err + } + snapshot := &SSnapshot{region: region} + return snapshot, resp.Unmarshal(snapshot, "snapshot") } diff --git a/pkg/util/openstack/storage.go b/pkg/util/openstack/storage.go index d395372473..0827d20d58 100644 --- a/pkg/util/openstack/storage.go +++ b/pkg/util/openstack/storage.go @@ -2,8 +2,10 @@ package openstack import ( "strings" + "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -94,7 +96,13 @@ func (storage *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { } func (storage *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotImplemented + disk, err := storage.zone.region.CreateDisk(storage.zone.ZoneName, storage.Name, name, sizeGb, desc) + if err != nil { + log.Errorf("createDisk fail %v", err) + return nil, err + } + disk.storage = storage + return disk, cloudprovider.WaitStatus(disk, models.DISK_READY, time.Second*5, time.Minute*5) } func (storage *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { diff --git a/pkg/util/openstack/storagecache.go b/pkg/util/openstack/storagecache.go index 65f5a03a10..331c264422 100644 --- a/pkg/util/openstack/storagecache.go +++ b/pkg/util/openstack/storagecache.go @@ -3,11 +3,18 @@ package openstack import ( "context" "fmt" + "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/image/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/qemuimg" ) type SStoragecache struct { @@ -49,7 +56,16 @@ func (cache *SStoragecache) GetManagerId() string { } func (cache *SStoragecache) fetchImages() error { - return cloudprovider.ErrNotImplemented + images, err := cache.region.GetImages("", ACTIVE, []string{}) + if err != nil { + return err + } + cache.iimages = make([]cloudprovider.ICloudImage, len(images)) + for i := 0; i < len(images); i++ { + images[i].storageCache = cache + cache.iimages[i] = &images[i] + } + return nil } func (cache *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { @@ -62,7 +78,12 @@ func (cache *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { } func (cache *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { - return nil, cloudprovider.ErrNotImplemented + image, err := cache.region.GetImage(extId) + if err != nil { + return nil, err + } + image.storageCache = cache + return image, nil } func (cache *SStoragecache) GetPath() string { @@ -70,11 +91,62 @@ func (cache *SStoragecache) GetPath() string { } func (cache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist, osVersion string, extId string, isForce bool) (string, error) { - return "", cloudprovider.ErrNotImplemented + if len(extId) > 0 { + log.Debugf("UploadImage: Image external ID exists %s", extId) + + statsu, err := cache.region.GetImageStatus(extId) + if err != nil { + log.Errorf("GetImageStatus error %s", err) + } + if statsu == ACTIVE && !isForce { + return extId, nil + } + } + log.Debugf("UploadImage: no external ID") + return cache.uploadImage(ctx, userCred, imageId, osArch, osType, osDist, osVersion, isForce) } -func (cache *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, isForce bool) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist, osVersion string, isForce bool) (string, error) { + s := auth.GetAdminSession(ctx, options.Options.Region, "") + + meta, reader, err := modules.Images.Download(s, imageId, string(qemuimg.VMDK), false) + if err != nil { + return "", err + } + log.Infof("meta data %s", meta) + + imageBaseName := imageId + if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { + imageBaseName = fmt.Sprintf("img%s", imageId) + } + imageName := imageBaseName + nameIdx := 1 + + for { + _, err = cache.region.GetImageByName(imageName) + if err != nil { + if err == cloudprovider.ErrNotFound { + break + } else { + return "", err + } + } + imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx) + nameIdx++ + } + + image, err := cache.region.CreateImage(imageName) + if err != nil { + return "", err + } + + image.storageCache = cache + + _, err = cache.region.client.StreamRequest(cache.region.Name, "image", "PUT", fmt.Sprintf("/v2/images/%s/file", image.ID), "", reader) + if err != nil { + return "", err + } + return image.ID, cloudprovider.WaitStatus(image, models.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second) } func (cache *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { diff --git a/pkg/util/openstack/vpc.go b/pkg/util/openstack/vpc.go index 43e739de86..1ecfab8245 100644 --- a/pkg/util/openstack/vpc.go +++ b/pkg/util/openstack/vpc.go @@ -96,7 +96,8 @@ func (vpc *SVpc) Delete() error { } func (region *SRegion) DeleteVpc(vpcId string) error { - return cloudprovider.ErrNotImplemented + _, err := region.Delete("network", "/v2.0/networks/"+vpcId, "") + return err } func (vpc *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { @@ -191,7 +192,7 @@ func (region *SRegion) GetVpc(vpcId string) (*SVpc, error) { } func (region *SRegion) GetVpcs() ([]SVpc, error) { - _, resp, err := region.Get("network", "/v2.0/networks", "", nil) + _, resp, err := region.List("network", "/v2.0/networks", "", nil) if err != nil { return nil, err } diff --git a/pkg/util/openstack/wire.go b/pkg/util/openstack/wire.go index 8673f2d7dc..b7ffa9d6f1 100644 --- a/pkg/util/openstack/wire.go +++ b/pkg/util/openstack/wire.go @@ -4,6 +4,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -56,7 +57,13 @@ func (wire *SWire) GetBandwidth() int { } func (wire *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - return nil, cloudprovider.ErrNotImplemented + networkId, err := wire.zone.region.CreateNetwork(wire.vpc.ID, name, cidr, desc) + if err != nil { + log.Errorf("CreateNetwork error %s", err) + return nil, err + } + wire.inetworks = nil + return wire.GetINetworkById(networkId) } func (wire *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, error) { diff --git a/pkg/util/openstack/zone.go b/pkg/util/openstack/zone.go index ec3f8af98a..07c2353887 100644 --- a/pkg/util/openstack/zone.go +++ b/pkg/util/openstack/zone.go @@ -2,6 +2,7 @@ package openstack import ( "fmt" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -76,7 +77,7 @@ func (zone *SZone) getStorageByCategory(category string) (*SStorage, error) { } for i := 0; i < len(storages); i++ { storage := storages[i].(*SStorage) - if storage.Name == category { + if strings.ToLower(storage.Name) == strings.ToLower(category) { return storage, nil } } @@ -94,7 +95,7 @@ func (zone *SZone) fetchStorages() error { zone.istorages = []cloudprovider.ICloudStorage{} for _, service := range []string{"volumev3", "volumev2", "volume"} { - _, resp, err := zone.region.Get(service, "/types", "", nil) + _, resp, err := zone.region.List(service, "/types", "", nil) if err == nil { storages := []SStorage{} if err := resp.Unmarshal(&storages, "volume_types"); err != nil { @@ -142,7 +143,7 @@ func (zone *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { hosts := []SHost{} _, maxVersion, err := zone.region.GetVersion("compute") if err == nil && version.GE(maxVersion, HYPERVISORS_VERSION) { - _, resp, err := zone.region.Get("compute", "/os-hypervisors/detail", maxVersion, nil) + _, resp, err := zone.region.List("compute", "/os-hypervisors/detail", maxVersion, nil) if err != nil { return nil, err } @@ -156,7 +157,7 @@ func (zone *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { return ihosts, nil } - _, resp, err := zone.region.Get("compute", "/os-hosts", "", nil) + _, resp, err := zone.region.List("compute", "/os-hosts", "", nil) if err != nil { return nil, err } diff --git a/pkg/util/qcloud/instance.go b/pkg/util/qcloud/instance.go index dd5c841c34..8feea35642 100644 --- a/pkg/util/qcloud/instance.go +++ b/pkg/util/qcloud/instance.go @@ -749,7 +749,7 @@ func (self *SInstance) AssignSecurityGroup(secgroupId string) error { return self.host.zone.region.instanceOperation(self.InstanceId, "ModifyInstancesAttribute", params) } -func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { params := map[string]string{} for i := 0; i < len(secgroupIds); i++ { params[fmt.Sprintf("SecurityGroups.%d", i)] = secgroupIds[i] diff --git a/pkg/util/qcloud/provider/provider.go b/pkg/util/qcloud/provider/provider.go index f15dc80ba6..d7bbf95255 100644 --- a/pkg/util/qcloud/provider/provider.go +++ b/pkg/util/qcloud/provider/provider.go @@ -44,6 +44,10 @@ func (self *SQcloudProviderFactory) IsSupportPrepaidResources() bool { return true } +func (self *SQcloudProviderFactory) NeedSyncSkuFromCloud() bool { + return false +} + func (self *SQcloudProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error { appID, _ := data.GetString("app_id") if len(appID) == 0 { diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index 95bc0dffaf..d7be925eed 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -627,3 +627,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { return nil, cloudprovider.ErrNotImplemented } + +func (region *SRegion) GetSkus(zoneId string) ([]cloudprovider.ICloudSku, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/sysutils/sysutils.go b/pkg/util/sysutils/sysutils.go index 8b4d4eff88..63d9ae6731 100644 --- a/pkg/util/sysutils/sysutils.go +++ b/pkg/util/sysutils/sysutils.go @@ -3,6 +3,8 @@ package sysutils import ( "fmt" "net" + "os" + "os/exec" "strconv" "strings" @@ -231,3 +233,20 @@ func GetSerialPorts(lines []string) []string { } return ret } + +func Start(closeFd bool, args ...string) (p *os.Process, err error) { + if args[0], err = exec.LookPath(args[0]); err == nil { + var procAttr os.ProcAttr + if closeFd { + procAttr.Files = []*os.File{nil, nil, nil} + } else { + procAttr.Files = []*os.File{os.Stdin, + os.Stdout, os.Stderr} + } + p, err := os.StartProcess(args[0], args, &procAttr) + if err == nil { + return p, nil + } + } + return nil, err +} diff --git a/pkg/yunionconf/service/service.go b/pkg/yunionconf/service/service.go index 2f9f2aee95..48914b08bd 100644 --- a/pkg/yunionconf/service/service.go +++ b/pkg/yunionconf/service/service.go @@ -30,7 +30,6 @@ func StartService() { } cloudcommon.InitDB(dbOpts) - defer cloudcommon.CloseDB() app := cloudcommon.InitApp(commonOpts, true) yunionconf.InitHandlers(app) @@ -38,7 +37,9 @@ func StartService() { if db.CheckSync(opts.AutoSyncTable) { err := models.InitDB() if err == nil { - cloudcommon.ServeForever(app, commonOpts) + cloudcommon.ServeForeverWithCleanup(app, commonOpts, func() { + cloudcommon.CloseDB() + }) } else { log.Errorf("InitDB fail: %s", err) }