diff --git a/Gopkg.lock b/Gopkg.lock index 32230c7ea1..4c92a38082 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1331,7 +1331,7 @@ [[projects]] branch = "master" - digest = "1:bf987f904fd821c6395fefdab956d070596eb401657fff40420014a3a8e3f45c" + digest = "1:8c4d781aafefb667511de3e0e9fe02170519f63d75a69e34045bd417f25d5523" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1355,7 +1355,6 @@ "util/secrules", "util/sets", "util/stringutils", - "util/sysutils", "util/timeutils", "util/trace", "util/ttlpool", @@ -1365,7 +1364,7 @@ "utils", ] pruneopts = "UT" - revision = "7614d751299a6703a05f757a13e6dc900332e31b" + revision = "883ea57a4b57d551bbf2fb7d1876d6ab9355c486" [[projects]] branch = "master" @@ -1377,11 +1376,11 @@ [[projects]] branch = "master" - digest = "1:f07a1ef9758f56186dd9039a8608bc9d537070a7c04479dbab6aeb42501001b7" + digest = "1:bbaf572e68e5dad4045e1e424af29838882c620df391fa1631dd8dc851c87a95" name = "yunion.io/x/structarg" packages = ["."] pruneopts = "UT" - revision = "e0cc2c73375327d401b100bd8988fa340c2687ee" + revision = "c95bf78846decd8dea379e925410f2895fb20076" [solve-meta] analyzer-name = "dep" @@ -1496,7 +1495,6 @@ "yunion.io/x/pkg/util/secrules", "yunion.io/x/pkg/util/sets", "yunion.io/x/pkg/util/stringutils", - "yunion.io/x/pkg/util/sysutils", "yunion.io/x/pkg/util/timeutils", "yunion.io/x/pkg/util/trace", "yunion.io/x/pkg/util/ttlpool", diff --git a/cmd/climc/shell/storages.go b/cmd/climc/shell/storages.go index de96c3bf29..5392b09073 100644 --- a/cmd/climc/shell/storages.go +++ b/cmd/climc/shell/storages.go @@ -99,17 +99,21 @@ func init() { }) type StorageCreateOptions struct { - NAME string `help:"Name of the Storage"` - Capacity int64 `help:"Capacity of the Storage"` - MediumType string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"` - StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|baremetal"` - MonHost string `helo:"Ceph mon_host config"` - Key string `helo:"Ceph key config"` - Pool string `helo:"Ceph Poll Name"` + NAME string `help:"Name of the Storage"` + ZONE string `help:"Zone id of storage"` + Capacity int64 `help:"Capacity of the Storage"` + MediumType string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"` + StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|baremetal"` + MonHost string `help:"Ceph mon_host config"` + Key string `help:"Ceph key config"` + Pool string `help:"Ceph Poll Name"` + NfsHost string `help:"NFS host"` + NfsSharedDir string `help:"NFS shared dir"` } R(&StorageCreateOptions{}, "storage-create", "Create a Storage", func(s *mcclient.ClientSession, args *StorageCreateOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.ZONE), "zone") params.Add(jsonutils.NewInt(args.Capacity), "capacity") params.Add(jsonutils.NewString(args.StorageType), "storage_type") params.Add(jsonutils.NewString(args.MediumType), "medium_type") @@ -120,6 +124,12 @@ func init() { params.Add(jsonutils.NewString(args.MonHost), "rbd_mon_host") params.Add(jsonutils.NewString(args.Key), "rbd_key") params.Add(jsonutils.NewString(args.Pool), "rbd_pool") + } else if args.StorageType == "nfs" { + if len(args.NfsHost) == 0 || len(args.NfsSharedDir) == 0 { + return fmt.Errorf("Storage type nfs missing conf host or shared dir") + } + params.Add(jsonutils.NewString(args.NfsHost), "nfs_host") + params.Add(jsonutils.NewString(args.NfsSharedDir), "nfs_shared_dir") } storage, err := modules.Storages.Create(s, params) if err != nil { @@ -168,6 +178,24 @@ func init() { return nil }) + R(&StorageShowOptions{}, "storage-online", "Online a storage", func(s *mcclient.ClientSession, args *StorageShowOptions) error { + result, err := modules.Storages.PerformAction(s, args.ID, "online", nil) + if err != nil { + return err + } + printObject(result) + return nil + }) + + R(&StorageShowOptions{}, "storage-offline", "Offline a storage", func(s *mcclient.ClientSession, args *StorageShowOptions) error { + result, err := modules.Storages.PerformAction(s, args.ID, "offline", nil) + if err != nil { + return err + } + printObject(result) + return nil + }) + type StorageCacheImageActionOptions struct { ID string `help:"ID or name of storage"` IMAGE string `help:"ID or name of image"` diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index cdcb19335f..7e9556c0e3 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -9,7 +9,6 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -74,7 +73,7 @@ func (self *SQcloudGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m if !utils.IsInStringArray(guest.Status, []string{models.VM_READY, models.VM_RUNNING}) { return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) } - if utils.IsInStringArray(storage.StorageType, []string{sysutils.STORAGE_LOCAL_BASIC, sysutils.STORAGE_LOCAL_SSD}) { + if utils.IsInStringArray(storage.StorageType, []string{models.STORAGE_LOCAL_BASIC, models.STORAGE_LOCAL_SSD}) { return fmt.Errorf("Cannot resize %s disk", storage.StorageType) } return nil diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 55338fbfb5..2319762a2b 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -15,7 +15,6 @@ import ( "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -1030,7 +1029,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info diskConfig.Mountpoint = p } else if p == "autoextend" { diskConfig.Size = -1 - } else if utils.IsInStringArray(p, sysutils.STORAGE_TYPES) { + } else if utils.IsInStringArray(p, STORAGE_TYPES) { diskConfig.Backend = p } else if len(p) > 0 { if userCred == nil { diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index cb5834f328..0e61957918 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -146,7 +146,8 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } } storage := disk.GetStorage() - if host.HostType == HOST_TYPE_HYPERVISOR && disk.IsLocal() || (storage != nil && storage.StorageType == STORAGE_RBD) { + // XXX ??? + if host.HostType == HOST_TYPE_HYPERVISOR { desc.Add(jsonutils.NewString(disk.StorageId), "storage_id") localpath := disk.GetPathAtHost(host) if len(localpath) == 0 { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 92a2bf8257..9213938a3a 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -19,7 +19,6 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/secrules" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -2306,7 +2305,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token for _, guestdisk := range self.GetDisks() { disk := guestdisk.GetDisk() storage := disk.GetStorage() - if utils.IsInStringArray(storage.StorageType, sysutils.LOCAL_STORAGE_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { + if utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { disk.DoPendingDelete(ctx, userCred) } else { self.DetachDisk(ctx, disk, userCred) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 52bc0c742d..c05f30ab93 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -17,7 +17,6 @@ import ( "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -467,7 +466,7 @@ func (self *SHost) GetBaremetalstorage() *SHoststorage { q := hoststorages.Query() q = q.Join(storages, sqlchemy.AND(sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), sqlchemy.IsFalse(storages.Field("deleted")))) - q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), sysutils.STORAGE_BAREMETAL)) + q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), STORAGE_BAREMETAL)) q = q.Filter(sqlchemy.Equals(hoststorages.Field("host_id"), self.Id)) if q.Count() == 1 { hs := SHoststorage{} diff --git a/pkg/compute/models/hoststorages.go b/pkg/compute/models/hoststorages.go index 8da2193ddb..2b234dd825 100644 --- a/pkg/compute/models/hoststorages.go +++ b/pkg/compute/models/hoststorages.go @@ -3,12 +3,16 @@ package models import ( "context" "fmt" + "net/http" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" ) @@ -97,12 +101,89 @@ func (manager *SHoststorageManager) ValidateCreateData(ctx context.Context, user return manager.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) } +func (self *SHoststorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + self.SHostJointsBase.PostCreate(ctx, userCred, ownerProjId, query, data) + storage := self.GetStorage() + if !utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) { + host := storage.GetMasterHost() + log.Infof("Attach SharedStorage[%s] on host %s ...", storage.Name, host.Name) + url := fmt.Sprintf("%s/storages/attach", host.ManagerUri) + headers := http.Header{} + headers.Set("X-Auth-Token", userCred.GetTokenString()) + body := jsonutils.NewDict() + body.Set("mount_point", jsonutils.NewString(self.MountPoint)) + body.Set("name", jsonutils.NewString(storage.Name)) + body.Set("storage_id", jsonutils.NewString(storage.Id)) + body.Set("storage_conf", storage.StorageConf) + body.Set("storage_type", jsonutils.NewString(storage.StorageType)) + if len(storage.StoragecacheId) > 0 { + storagecache := StoragecacheManager.FetchStoragecacheById(storage.StoragecacheId) + if storagecache != nil { + body.Set("imagecache_path", jsonutils.NewString( + storage.GetStorageCachePath(self.MountPoint, storagecache.Path))) + body.Set("storagecache_id", jsonutils.NewString(storagecache.Id)) + } + } + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), + ctx, "POST", url, headers, body, false) + if err != nil { + log.Errorf("Host Storage Post Create Error: %s", err) + // panic(err) ??? + } + self.SyncStorageStatus() + } +} + +func (self *SHoststorage) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) { + storage := self.GetStorage() + if !utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) { + host := storage.GetMasterHost() + log.Infof("Attach SharedStorage[%s] on host %s ...", storage.Name, host.Name) + url := fmt.Sprintf("%s/storages/detach", host.ManagerUri) + headers := http.Header{} + headers.Set("X-Auth-Token", userCred.GetTokenString()) + body := jsonutils.NewDict() + body.Set("mount_point", jsonutils.NewString(self.MountPoint)) + body.Set("name", jsonutils.NewString(storage.Name)) + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), + ctx, "POST", url, headers, body, false) + if err != nil { + log.Errorf("Host Storage Post Create Error: %s", err) + // panic(err) ??? + } + self.SyncStorageStatus() + } +} + +func (self *SHoststorage) SyncStorageStatus() { + storage := self.GetStorage() + hostQuery := HostManager.Query().SubQuery() + count := HoststorageManager.Query().Join(hostQuery, + sqlchemy.AND(sqlchemy.Equals(hostQuery.Field("id"), self.HostId), + sqlchemy.Equals(hostQuery.Field("host_status"), "online"))).Count() + status := storage.Status + if count >= 1 { + status = STORAGE_ONLINE + } else { + status = STORAGE_OFFLINE + } + if status != storage.Status { + storage.GetModelManager().TableSpec().Update(storage, func() error { + storage.Status = status + return nil + }) + } +} + func (self *SHoststorage) getExtraDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { host := self.GetHost() extra.Add(jsonutils.NewString(host.Name), "host") storage := self.GetStorage() extra.Add(jsonutils.NewString(storage.Name), "storage") extra.Add(jsonutils.NewInt(int64(storage.Capacity)), "capacity") + if storage.StorageConf != nil { + extra.Set("storage_conf", storage.StorageConf) + } used := storage.GetUsedCapacity(tristate.True) wasted := storage.GetUsedCapacity(tristate.False) extra.Add(jsonutils.NewInt(int64(used)), "used_capacity") @@ -110,12 +191,16 @@ func (self *SHoststorage) getExtraDetails(extra *jsonutils.JSONDict) *jsonutils. extra.Add(jsonutils.NewInt(int64(storage.Capacity-used-wasted)), "free_capacity") extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") - if storage.Enabled { - extra.Add(jsonutils.JSONTrue, "enabled") - } else { - extra.Add(jsonutils.JSONFalse, "enabled") - } + extra.Add(jsonutils.NewBool(storage.Enabled), "enabled") extra.Add(jsonutils.NewFloat(float64(storage.GetOvercommitBound())), "cmtbound") + extra.Add(jsonutils.NewInt(int64(self.GetGuestDiskCount())), "guest_disk_count") + if len(storage.StoragecacheId) > 0 { + storagecache := StoragecacheManager.FetchStoragecacheById(storage.StoragecacheId) + if storagecache != nil { + extra.Set("imagecache_path", jsonutils.NewString(storage.GetStorageCachePath(self.MountPoint, storagecache.Path))) + extra.Set("storagecache_id", jsonutils.NewString(storagecache.Id)) + } + } return extra } diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 240a99782b..5d2d07624a 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -284,6 +284,23 @@ func (manager *SStoragecacheManager) ListItemFilter(ctx context.Context, q *sqlc return q, nil } +func (manager *SStoragecacheManager) FetchStoragecacheById(storageCacheId string) *SStoragecache { + iStorageCache, _ := manager.FetchById(storageCacheId) + if iStorageCache == nil { + return nil + } + return iStorageCache.(*SStoragecache) +} + +func (manager *SStoragecacheManager) GetCachePathById(storageCacheId string) string { + iStorageCache, _ := manager.FetchById(storageCacheId) + if iStorageCache == nil { + return "" + } + sc := iStorageCache.(*SStoragecache) + return sc.Path +} + func (self *SStoragecache) ValidateDeleteCondition(ctx context.Context) error { if self.getCachedImageCount() > 0 { return httperrors.NewNotEmptyError("storage cache not empty") diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index b7be546d7d..37ca701ead 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -2,6 +2,10 @@ package models import ( "context" + "fmt" + "path" + "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -10,31 +14,51 @@ import ( "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_NFS = "nfs" + STORAGE_PUBLIC_CLOUD = "cloud" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_SSD = "sc1" // aws Cold HDD - STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes + STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 + //Azure hdd and ssd storagetype + STORAGE_STANDARD_GRS = "standard_grs" + STORAGE_STANDARD_LRS = "standard_lrs" + STORAGE_STANDARD_RAGRS = "standard_ragrs" + STORAGE_STANDARD_ZRS = "standard_zrs" + STORAGE_PREMIUM_LRS = "premium_lrs" + + // aws storage type + STORAGE_GP2_SSD = "gp2" // aws general purpose ssd + STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD + STORAGE_SC1_SSD = "sc1" // aws Cold HDD + STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes + + // qcloud storage type + // STORAGE_CLOUD_SSD ="cloud_ssd" + STORAGE_LOCAL_BASIC = "local_basic" + STORAGE_LOCAL_SSD = "local_ssd" + STORAGE_CLOUD_BASIC = "cloud_basic" + STORAGE_CLOUD_PERMIUM = "cloud_permium" +) + +const ( STORAGE_ENABLED = "enabled" STORAGE_DISABLED = "disabled" STORAGE_OFFLINE = "offline" @@ -47,17 +71,26 @@ const ( var ( DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} - STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS} + STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES STORAGE_ALL_TYPES = []string{ STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_NFS, } - STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD} + STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, + STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS, + STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD, + STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM, + } + + STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} ) type SStorageManager struct { - db.SEnabledStatusStandaloneResourceBaseManager + db.SStandaloneResourceBaseManager SInfrastructureManager } @@ -65,7 +98,7 @@ var StorageManager *SStorageManager func init() { StorageManager = &SStorageManager{ - SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( SStorage{}, "storages_tbl", "storage", @@ -75,7 +108,7 @@ func init() { } type SStorage struct { - db.SEnabledStatusStandaloneResourceBase + db.SStandaloneResourceBase SInfrastructure SManagedResourceBase @@ -89,6 +122,9 @@ type SStorage struct { ZoneId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"admin_required"` StoragecacheId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin"` + + Enabled bool `nullable:"false" default:"true" list:"user" create:"optional"` + Status string `width:"36" charset:"ascii" nullable:"false" default:"offline" list:"user" create:"optional"` } func (manager *SStorageManager) GetContextManager() []db.IModelManager { @@ -98,7 +134,12 @@ func (manager *SStorageManager) GetContextManager() []db.IModelManager { func (manager *SStorageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { storageType, _ := data.GetString("storage_type") mediumType, _ := data.GetString("medium_type") - if !utils.IsInStringArray(storageType, STORAGE_ALL_TYPES) { + capacity, _ := data.Int("capacity") + if capacity <= 0 { + return nil, httperrors.NewInputParameterError("Invalid capacity") + } + data.Set("capacity", jsonutils.NewInt(capacity)) + if !utils.IsInStringArray(storageType, STORAGE_TYPES) { return nil, httperrors.NewInputParameterError("Invalid storage type %s", storageType) } if !utils.IsInStringArray(mediumType, DISK_TYPES) { @@ -113,27 +154,196 @@ func (manager *SStorageManager) ValidateCreateData(ctx context.Context, userCred return nil, httperrors.NewResourceNotFoundError("zone %s", zoneId) } data.Set("zone_id", jsonutils.NewString(zone.GetId())) - // TODO: ValidateRdbConfData - // if storageType == STORAGE_RBD { - // conf := jsonutils.NewDict() - // for k, v := range data.Value() { - // if strings.HasPrefix(k, fmt.Sprintf("%s_", storageType)) { - // k = k[len(storageType)+1:] - // if len(k) > 0 { - // conf.Set(k, v) - // } - // } - // } - // data.Set("capacity", manager.ValidateRdbConfData(conf)) - // } - return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) + if storageType == STORAGE_RBD { + conf, err := manager.ValidateRbdConfData(data) + if err != nil { + return nil, httperrors.NewBadRequestError("Vaildata rbd conf error: %s", err.Error()) + } + data.Set("storage_conf", conf) + // data.Set("capacity", rbdConf) + } else if storageType == STORAGE_NFS { + conf, err := manager.ValidataNfsConfdata(data) + if err != nil { + return nil, httperrors.NewBadRequestError("Vaildata nfs conf error: %s", err.Error()) + } + data.Set("storage_conf", conf) + } + return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) +} + +func (manager *SStorageManager) ValidataNfsConfdata(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + conf := jsonutils.NewDict() + if nfsHost, err := data.GetString("nfs_host"); err != nil { + return nil, httperrors.NewInputParameterError("Get nfs conf host error: %s", err.Error()) + } else { + conf.Set("nfs_host", jsonutils.NewString(nfsHost)) + } + if nfsSharedDir, err := data.GetString("nfs_shared_dir"); err != nil { + return nil, httperrors.NewInputParameterError("Get nfs conf shared dir error: %s", err.Error()) + } else { + conf.Set("nfs_shared_dir", jsonutils.NewString(nfsSharedDir)) + } + return conf, nil +} + +func (manager *SStorageManager) ValidateRbdConfData(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + conf := jsonutils.NewDict() + for k, v := range data.Value() { + if strings.HasPrefix(k, fmt.Sprintf("%s_", STORAGE_RBD)) { + k = k[len(STORAGE_RBD)+1:] + if len(k) > 0 { + conf.Set(k, v) + } + } + } + requireFields := []string{"mon_host", "key", "pool"} + for _, field := range requireFields { + if !conf.Contains(field) { + return nil, httperrors.NewMissingParameterError(field) + } + } + storages := make([]SStorage, 0) + err := manager.Query().Equals("storage_type", STORAGE_RBD).All(&storages) + if err != nil { + return nil, err + } + for i := 0; i < len(storages); i++ { + if conf.Equals(storages[i].StorageConf) { + return nil, httperrors.NewDuplicateResourceError("This RBD Storage[%s/%s] has already exist", storages[i].Name, conf.String()) + } + } + // TODO??? ensure rbd pool can use and get capacity + return conf, nil } func (self *SStorage) ValidateDeleteCondition(ctx context.Context) error { if self.GetHostCount() > 0 || self.GetDiskCount() > 0 || self.GetSnapshotCount() > 0 { return httperrors.NewNotEmptyError("Not an empty storage provider") } - return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx) + return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx) +} + +func (self *SStorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + self.SStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) + if self.StorageType == STORAGE_RBD { + // TODO + } else if self.StorageType == STORAGE_NFS { + sc := &SStoragecache{} + sc.Path = options.Options.NfsDefaultImageCacheDir + sc.ExternalId = self.Id + sc.Name = "nfs-" + self.Name + time.Now().Format("2006-01-02 15:04:05") + err := StoragecacheManager.TableSpec().Insert(sc) + if err != nil { + log.Errorln(err) + return + } + err = StoragecacheManager.Query().Equals("external_id", self.Id).First(sc) + if err != nil { + log.Errorln(err) + return + } + _, err = self.GetModelManager().TableSpec().Update(self, func() error { + self.StoragecacheId = sc.Id + self.Status = STORAGE_ONLINE + return nil + }) + if err != nil { + log.Errorln(err) + } + } +} + +func (self *SStorage) SetStatus(userCred mcclient.TokenCredential, status string, reason string) error { + if self.Status == status { + return nil + } + oldStatus := self.Status + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.Status = status + return nil + }) + if err != nil { + return err + } + if userCred != nil { + notes := fmt.Sprintf("%s=>%s", oldStatus, status) + if len(reason) > 0 { + notes = fmt.Sprintf("%s: %s", notes, reason) + } + db.OpsLog.LogEvent(self, db.ACT_UPDATE_STATUS, notes, userCred) + if strings.Contains(notes, "fail") { + logclient.AddActionLog(self, logclient.ACT_VM_SYNC_STATUS, notes, userCred, false) + } + } + return nil +} + +func (self *SStorage) AllowPerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SStorage) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !self.Enabled { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.Enabled = true + return nil + }) + if err != nil { + log.Errorf("PerformEnable save update fail %s", err) + return nil, err + } + db.OpsLog.LogEvent(self, db.ACT_ENABLE, "", userCred) + } + return nil, nil +} + +func (self *SStorage) AllowPerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SStorage) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Enabled { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.Enabled = false + return nil + }) + if err != nil { + log.Errorf("PerformDisable save update fail %s", err) + return nil, err + } + db.OpsLog.LogEvent(self, db.ACT_DISABLE, "", userCred) + } + return nil, nil +} + +func (self *SStorage) AllowPerformOnline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SStorage) PerformOnline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status != STORAGE_ONLINE { + err := self.SetStatus(userCred, STORAGE_ONLINE, "") + if err != nil { + return nil, err + } + db.OpsLog.LogEvent(self, db.ACT_ONLINE, "", userCred) + } + return nil, nil +} + +func (self *SStorage) AllowPerformOffline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SStorage) PerformOffline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status != STORAGE_OFFLINE { + err := self.SetStatus(userCred, STORAGE_OFFLINE, "") + if err != nil { + return nil, err + } + db.OpsLog.LogEvent(self, db.ACT_OFFLINE, "", userCred) + } + return nil, nil } func (self *SStorage) GetHostCount() int { @@ -156,6 +366,14 @@ func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcc return true } +func (self *SStorage) GetStorageCachePath(mountPoint, imageCachePath string) string { + if self.StorageType == STORAGE_NFS { + return path.Join(mountPoint, imageCachePath) + } else { + return imageCachePath + } +} + func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { used := self.GetUsedCapacity(tristate.True) waste := self.GetUsedCapacity(tristate.False) @@ -176,7 +394,7 @@ func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND } func (self *SStorage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { - extra := self.SEnabledStatusStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query) + extra := self.SStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query) return self.getMoreDetails(extra) } @@ -761,7 +979,7 @@ func (manager *SStorageManager) InitializeData() error { } func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { - q, err := manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { return nil, err } @@ -777,11 +995,11 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. } if jsonutils.QueryBoolean(query, "share", false) { - q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), STORAGE_LOCAL_TYPES)) } if jsonutils.QueryBoolean(query, "local", false) { - q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.In(q.Field("storage_type"), STORAGE_LOCAL_TYPES)) } if jsonutils.QueryBoolean(query, "usable", false) { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 25d5937c55..b357663c43 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -65,6 +65,8 @@ type ComputeOptions struct { ConvertEsxiDefaultTemplate string `default:"Default template" help:"ESXI baremetal convert option"` ConvertKubeletDockerVolumeSize string `default:"256g" help:"Docker volume size"` + NfsDefaultImageCacheDir string `default:"image_cache"` + cloudcommon.DBOptions } diff --git a/pkg/compute/tasks/guest_detach_all_disks_task.go b/pkg/compute/tasks/guest_detach_all_disks_task.go index cb5fa8f9ee..c5c6a7d8ac 100644 --- a/pkg/compute/tasks/guest_detach_all_disks_task.go +++ b/pkg/compute/tasks/guest_detach_all_disks_task.go @@ -43,13 +43,7 @@ func (self *GuestDetachAllDisksTask) OnDiskDeleteComplete(ctx context.Context, o if jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { taskData.Add(jsonutils.JSONTrue, "override_pending_delete") } - disk := guestdisk.GetDisk() - storage := disk.GetStorage() - if storage.IsLocal() { - taskData.Add(jsonutils.JSONFalse, "keep_disk") - } else { - taskData.Add(jsonutils.JSONTrue, "keep_disk") - } + taskData.Add(jsonutils.JSONFalse, "keep_disk") task, err := taskman.TaskManager.NewTask(ctx, "GuestDetachDiskTask", guest, self.UserCred, taskData, self.GetTaskId(), "", nil) if err != nil { self.SetStageFailed(ctx, err.Error()) diff --git a/pkg/compute/tasks/guest_detach_disk_task.go b/pkg/compute/tasks/guest_detach_disk_task.go index 8b976b0444..676c12972a 100644 --- a/pkg/compute/tasks/guest_detach_disk_task.go +++ b/pkg/compute/tasks/guest_detach_disk_task.go @@ -91,15 +91,12 @@ func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) disk.RealDelete(ctx, self.UserCred) self.SetStageComplete(ctx, nil) - return - } - if !keepDisk && disk.GetGuestDiskCount() == 0 && disk.AutoDelete { + } else if (disk.Status != models.DISK_READY || !keepDisk) && disk.GetGuestDiskCount() == 0 && disk.AutoDelete { self.SetStage("on_disk_delete_complete", nil) db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) err := guest.GetDriver().RequestDeleteDetachedDisk(ctx, disk, self, purge) if err != nil { self.OnTaskFail(ctx, guest, disk, err) - return } } else { self.SetStageComplete(ctx, nil) diff --git a/vendor/yunion.io/x/pkg/prettytable/prettytable.go b/vendor/yunion.io/x/pkg/prettytable/prettytable.go index f5ee4f34f1..feba929161 100644 --- a/vendor/yunion.io/x/pkg/prettytable/prettytable.go +++ b/vendor/yunion.io/x/pkg/prettytable/prettytable.go @@ -3,6 +3,7 @@ package prettytable import ( "bytes" "strings" + "unicode" ) type AlignmentType uint8 @@ -102,6 +103,17 @@ func textLine(buf *bytes.Buffer, columns []ptColumn, widths []int) { } } +func runeDisplayWidth(r rune) int { + const puncts = "。，；：（）、？《》" + if unicode.Is(unicode.Han, r) { + return 2 + } + if strings.ContainsRune(puncts, r) { + return 2 + } + return 1 +} + // cellDisplayWidth returns display width of the cell when printed as the // nthCol. prevWidth is the total display width (as return by this same func) // of previous cells in the same line @@ -116,7 +128,7 @@ func cellDisplayWidth(cell string, nthCol int, prevWidth int) int { for _, c := range line { incr := 0 if c != '\t' { - incr = 1 + incr = runeDisplayWidth(c) } else { // terminal with have the char TabWidth aligned incr = TabWidth - (x & (TabWidth - 1)) diff --git a/vendor/yunion.io/x/pkg/util/sysutils/doc.go b/vendor/yunion.io/x/pkg/util/sysutils/doc.go deleted file mode 100644 index 8294259502..0000000000 --- a/vendor/yunion.io/x/pkg/util/sysutils/doc.go +++ /dev/null @@ -1 +0,0 @@ -package sysutils // import "yunion.io/x/pkg/util/sysutils" diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go deleted file mode 100644 index 3f9de2fe1b..0000000000 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ /dev/null @@ -1,49 +0,0 @@ -package sysutils - -const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - - STORAGE_CLOUD = "cloud" - STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 - STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" - - //Azure hdd and ssd storagetype - STORAGE_STANDARD_GRS = "standard_grs" - STORAGE_STANDARD_LRS = "standard_lrs" - STORAGE_STANDARD_RAGRS = "standard_ragrs" - STORAGE_STANDARD_ZRS = "standard_zrs" - STORAGE_PREMIUM_LRS = "premium_lrs" - - // aws storage type - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_SSD = "sc1" // aws Cold HDD - STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes - - // qcloud storage type - // STORAGE_CLOUD_SSD ="cloud_ssd" - STORAGE_LOCAL_BASIC = "local_basic" - STORAGE_LOCAL_SSD = "local_ssd" - STORAGE_CLOUD_BASIC = "cloud_basic" - STORAGE_CLOUD_PERMIUM = "cloud_permium" -) - -var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, - STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY, - STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS, - STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD, - STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM, -} - -var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} - -// var SUPPORT = []string {support = [STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS] diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index e09e97cdad..ff749484c7 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -187,56 +187,6 @@ func (this *ArgumentParser) addStructArgument(tp reflect.Type, val reflect.Value return nil } -/*func findWord(str []byte, offset int) (string, int) { - var buffer bytes.Buffer - i := skipEmpty(str, offset) - if i >= len(str) { - return "", i - } - var endstr string - quote := false - if str[i] == '"' { - quote = true - endstr = "\"" - i++ - } else if str[i] == '\'' { - quote = true - endstr = "'" - i++ - } else { - endstr = " :,\t\n}]" - } - for i < len(str) { - if quote && str[i] == '\\' { - if i+1 < len(str) { - i++ - switch str[i] { - case 'n': - buffer.WriteByte('\n') - case 'r': - buffer.WriteByte('\r') - case 't': - buffer.WriteByte('\t') - default: - buffer.WriteByte(str[i]) - } - i++ - } else { - break - } - } else if strings.IndexByte(endstr, str[i]) >= 0 { // end - if quote { - i++ - } - break - } else { - buffer.WriteByte(str[i]) - i++ - } - } - return buffer.String(), i -}*/ - func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) error { tagMap := utils.TagMap(f.Tag) help := tagMap[TAG_HELP] @@ -901,6 +851,14 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error return err } +func isQuotedByChar(str string, quoteChar byte) bool { + return str[0] == quoteChar && str[len(str)-1] == quoteChar +} + +func isQuoted(str string) bool { + return isQuotedByChar(str, '"') || isQuotedByChar(str, '\'') +} + func (this *ArgumentParser) parseKeyValue(key, value string) error { arg := this.findOptionalArgument(key) if arg != nil { @@ -914,7 +872,15 @@ func (this *ArgumentParser) parseKeyValue(key, value string) error { } } } else { - return arg.SetValue(value) + if !isQuoted(value) { + value = fmt.Sprintf("\"%s\"", value) + } + values := utils.FindWords([]byte(value), 0) + if len(values) == 1 { + return arg.SetValue(values[0]) + } else { + log.Warningf("too many arguments %#v for %s", values, key) + } } } else { log.Warningf("Cannot find argument %s", key) @@ -964,7 +930,7 @@ func (this *ArgumentParser) ParseFile(filepath string) error { for scanner.Scan() { line := scanner.Text() line = strings.TrimSpace(removeComments(line)) - line = removeCharacters(line, `"'`) + // line = removeCharacters(line, `"'`) if len(line) > 0 { key, val, e := line2KeyValue(line) if e == nil {