Merge pull request #556 in YUNIONIO/onecloud from ~WANYAOQI/onecloud:feature/wyq/storage-nfs-support to release/2.4.0

* commit '71bca9927977f5cfa33f97e6d975176dc7c7bca8':
  make dep
  support nfs storage

Gopkg.lock

@@ -1331,7 +1331,7 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:bf987f904fd821c6395fefdab956d070596eb401657fff40420014a3a8e3f45c"
+  digest = "1:8c4d781aafefb667511de3e0e9fe02170519f63d75a69e34045bd417f25d5523"
   name = "yunion.io/x/pkg"
   packages = [
     "gotypes",
@@ -1355,7 +1355,6 @@
     "util/secrules",
     "util/sets",
     "util/stringutils",
-    "util/sysutils",
     "util/timeutils",
     "util/trace",
     "util/ttlpool",
@@ -1365,7 +1364,7 @@
     "utils",
   ]
   pruneopts = "UT"
-  revision = "7614d751299a6703a05f757a13e6dc900332e31b"
+  revision = "883ea57a4b57d551bbf2fb7d1876d6ab9355c486"
 
 [[projects]]
   branch = "master"
@@ -1377,11 +1376,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:f07a1ef9758f56186dd9039a8608bc9d537070a7c04479dbab6aeb42501001b7"
+  digest = "1:bbaf572e68e5dad4045e1e424af29838882c620df391fa1631dd8dc851c87a95"
   name = "yunion.io/x/structarg"
   packages = ["."]
   pruneopts = "UT"
-  revision = "e0cc2c73375327d401b100bd8988fa340c2687ee"
+  revision = "c95bf78846decd8dea379e925410f2895fb20076"
 
 [solve-meta]
   analyzer-name = "dep"
@@ -1496,7 +1495,6 @@
     "yunion.io/x/pkg/util/secrules",
     "yunion.io/x/pkg/util/sets",
     "yunion.io/x/pkg/util/stringutils",
-    "yunion.io/x/pkg/util/sysutils",
     "yunion.io/x/pkg/util/timeutils",
     "yunion.io/x/pkg/util/trace",
     "yunion.io/x/pkg/util/ttlpool",

cmd/climc/shell/storages.go

@@ -99,17 +99,21 @@ func init() {
 	})
 
 	type StorageCreateOptions struct {
-		NAME        string `help:"Name of the Storage"`
-		Capacity    int64  `help:"Capacity of the Storage"`
-		MediumType  string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"`
-		StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|baremetal"`
-		MonHost     string `helo:"Ceph mon_host config"`
-		Key         string `helo:"Ceph key config"`
-		Pool        string `helo:"Ceph Poll Name"`
+		NAME         string `help:"Name of the Storage"`
+		ZONE         string `help:"Zone id of storage"`
+		Capacity     int64  `help:"Capacity of the Storage"`
+		MediumType   string `help:"Medium type, either ssd or rotate" choices:"ssd|rotate"`
+		StorageType  string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|baremetal"`
+		MonHost      string `help:"Ceph mon_host config"`
+		Key          string `help:"Ceph key config"`
+		Pool         string `help:"Ceph Poll Name"`
+		NfsHost      string `help:"NFS host"`
+		NfsSharedDir string `help:"NFS shared dir"`
 	}
 	R(&StorageCreateOptions{}, "storage-create", "Create a Storage", func(s *mcclient.ClientSession, args *StorageCreateOptions) error {
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewString(args.NAME), "name")
+		params.Add(jsonutils.NewString(args.ZONE), "zone")
 		params.Add(jsonutils.NewInt(args.Capacity), "capacity")
 		params.Add(jsonutils.NewString(args.StorageType), "storage_type")
 		params.Add(jsonutils.NewString(args.MediumType), "medium_type")
@@ -120,6 +124,12 @@ func init() {
 			params.Add(jsonutils.NewString(args.MonHost), "rbd_mon_host")
 			params.Add(jsonutils.NewString(args.Key), "rbd_key")
 			params.Add(jsonutils.NewString(args.Pool), "rbd_pool")
+		} else if args.StorageType == "nfs" {
+			if len(args.NfsHost) == 0 || len(args.NfsSharedDir) == 0 {
+				return fmt.Errorf("Storage type nfs missing conf host or shared dir")
+			}
+			params.Add(jsonutils.NewString(args.NfsHost), "nfs_host")
+			params.Add(jsonutils.NewString(args.NfsSharedDir), "nfs_shared_dir")
 		}
 		storage, err := modules.Storages.Create(s, params)
 		if err != nil {
@@ -168,6 +178,24 @@ func init() {
 		return nil
 	})
 
+	R(&StorageShowOptions{}, "storage-online", "Online a storage", func(s *mcclient.ClientSession, args *StorageShowOptions) error {
+		result, err := modules.Storages.PerformAction(s, args.ID, "online", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&StorageShowOptions{}, "storage-offline", "Offline a storage", func(s *mcclient.ClientSession, args *StorageShowOptions) error {
+		result, err := modules.Storages.PerformAction(s, args.ID, "offline", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type StorageCacheImageActionOptions struct {
 		ID    string `help:"ID or name of storage"`
 		IMAGE string `help:"ID or name of image"`

pkg/compute/guestdrivers/qcloud.go

@@ -9,7 +9,6 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -74,7 +73,7 @@ func (self *SQcloudGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	if !utils.IsInStringArray(guest.Status, []string{models.VM_READY, models.VM_RUNNING}) {
 		return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status)
 	}
-	if utils.IsInStringArray(storage.StorageType, []string{sysutils.STORAGE_LOCAL_BASIC, sysutils.STORAGE_LOCAL_SSD}) {
+	if utils.IsInStringArray(storage.StorageType, []string{models.STORAGE_LOCAL_BASIC, models.STORAGE_LOCAL_SSD}) {
 		return fmt.Errorf("Cannot resize %s disk", storage.StorageType)
 	}
 	return nil

pkg/compute/models/disks.go

@@ -15,7 +15,6 @@ import (
 	"yunion.io/x/pkg/util/fileutils"
 	"yunion.io/x/pkg/util/osprofile"
 	"yunion.io/x/pkg/util/regutils"
-	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
@@ -1030,7 +1029,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info
 			diskConfig.Mountpoint = p
 		} else if p == "autoextend" {
 			diskConfig.Size = -1
-		} else if utils.IsInStringArray(p, sysutils.STORAGE_TYPES) {
+		} else if utils.IsInStringArray(p, STORAGE_TYPES) {
 			diskConfig.Backend = p
 		} else if len(p) > 0 {
 			if userCred == nil {

pkg/compute/models/guestdisks.go

@@ -146,7 +146,8 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject {
 		}
 	}
 	storage := disk.GetStorage()
-	if host.HostType == HOST_TYPE_HYPERVISOR && disk.IsLocal() || (storage != nil && storage.StorageType == STORAGE_RBD) {
+	// XXX ???
+	if host.HostType == HOST_TYPE_HYPERVISOR {
 		desc.Add(jsonutils.NewString(disk.StorageId), "storage_id")
 		localpath := disk.GetPathAtHost(host)
 		if len(localpath) == 0 {

pkg/compute/models/guests.go

@@ -19,7 +19,6 @@ import (
 	"yunion.io/x/pkg/util/osprofile"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
-	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
@@ -2306,7 +2305,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token
 	for _, guestdisk := range self.GetDisks() {
 		disk := guestdisk.GetDisk()
 		storage := disk.GetStorage()
-		if utils.IsInStringArray(storage.StorageType, sysutils.LOCAL_STORAGE_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) {
+		if utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) {
 			disk.DoPendingDelete(ctx, userCred)
 		} else {
 			self.DetachDisk(ctx, disk, userCred)

pkg/compute/models/hosts.go

@@ -17,7 +17,6 @@ import (
 	"yunion.io/x/pkg/util/fileutils"
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/regutils"
-	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -467,7 +466,7 @@ func (self *SHost) GetBaremetalstorage() *SHoststorage {
 	q := hoststorages.Query()
 	q = q.Join(storages, sqlchemy.AND(sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")),
 		sqlchemy.IsFalse(storages.Field("deleted"))))
-	q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), sysutils.STORAGE_BAREMETAL))
+	q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), STORAGE_BAREMETAL))
 	q = q.Filter(sqlchemy.Equals(hoststorages.Field("host_id"), self.Id))
 	if q.Count() == 1 {
 		hs := SHoststorage{}

pkg/compute/models/hoststorages.go

@@ -3,12 +3,16 @@ package models
 import (
 	"context"
 	"fmt"
+	"net/http"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 )
 
@@ -97,12 +101,89 @@ func (manager *SHoststorageManager) ValidateCreateData(ctx context.Context, user
 	return manager.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
+func (self *SHoststorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	self.SHostJointsBase.PostCreate(ctx, userCred, ownerProjId, query, data)
+	storage := self.GetStorage()
+	if !utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) {
+		host := storage.GetMasterHost()
+		log.Infof("Attach SharedStorage[%s] on host %s ...", storage.Name, host.Name)
+		url := fmt.Sprintf("%s/storages/attach", host.ManagerUri)
+		headers := http.Header{}
+		headers.Set("X-Auth-Token", userCred.GetTokenString())
+		body := jsonutils.NewDict()
+		body.Set("mount_point", jsonutils.NewString(self.MountPoint))
+		body.Set("name", jsonutils.NewString(storage.Name))
+		body.Set("storage_id", jsonutils.NewString(storage.Id))
+		body.Set("storage_conf", storage.StorageConf)
+		body.Set("storage_type", jsonutils.NewString(storage.StorageType))
+		if len(storage.StoragecacheId) > 0 {
+			storagecache := StoragecacheManager.FetchStoragecacheById(storage.StoragecacheId)
+			if storagecache != nil {
+				body.Set("imagecache_path", jsonutils.NewString(
+					storage.GetStorageCachePath(self.MountPoint, storagecache.Path)))
+				body.Set("storagecache_id", jsonutils.NewString(storagecache.Id))
+			}
+		}
+		_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(),
+			ctx, "POST", url, headers, body, false)
+		if err != nil {
+			log.Errorf("Host Storage Post Create Error: %s", err)
+			// panic(err) ???
+		}
+		self.SyncStorageStatus()
+	}
+}
+
+func (self *SHoststorage) PreDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	storage := self.GetStorage()
+	if !utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) {
+		host := storage.GetMasterHost()
+		log.Infof("Attach SharedStorage[%s] on host %s ...", storage.Name, host.Name)
+		url := fmt.Sprintf("%s/storages/detach", host.ManagerUri)
+		headers := http.Header{}
+		headers.Set("X-Auth-Token", userCred.GetTokenString())
+		body := jsonutils.NewDict()
+		body.Set("mount_point", jsonutils.NewString(self.MountPoint))
+		body.Set("name", jsonutils.NewString(storage.Name))
+		_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(),
+			ctx, "POST", url, headers, body, false)
+		if err != nil {
+			log.Errorf("Host Storage Post Create Error: %s", err)
+			// panic(err) ???
+		}
+		self.SyncStorageStatus()
+	}
+}
+
+func (self *SHoststorage) SyncStorageStatus() {
+	storage := self.GetStorage()
+	hostQuery := HostManager.Query().SubQuery()
+	count := HoststorageManager.Query().Join(hostQuery,
+		sqlchemy.AND(sqlchemy.Equals(hostQuery.Field("id"), self.HostId),
+			sqlchemy.Equals(hostQuery.Field("host_status"), "online"))).Count()
+	status := storage.Status
+	if count >= 1 {
+		status = STORAGE_ONLINE
+	} else {
+		status = STORAGE_OFFLINE
+	}
+	if status != storage.Status {
+		storage.GetModelManager().TableSpec().Update(storage, func() error {
+			storage.Status = status
+			return nil
+		})
+	}
+}
+
 func (self *SHoststorage) getExtraDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	host := self.GetHost()
 	extra.Add(jsonutils.NewString(host.Name), "host")
 	storage := self.GetStorage()
 	extra.Add(jsonutils.NewString(storage.Name), "storage")
 	extra.Add(jsonutils.NewInt(int64(storage.Capacity)), "capacity")
+	if storage.StorageConf != nil {
+		extra.Set("storage_conf", storage.StorageConf)
+	}
 	used := storage.GetUsedCapacity(tristate.True)
 	wasted := storage.GetUsedCapacity(tristate.False)
 	extra.Add(jsonutils.NewInt(int64(used)), "used_capacity")
@@ -110,12 +191,16 @@ func (self *SHoststorage) getExtraDetails(extra *jsonutils.JSONDict) *jsonutils.
 	extra.Add(jsonutils.NewInt(int64(storage.Capacity-used-wasted)), "free_capacity")
 	extra.Add(jsonutils.NewString(storage.StorageType), "storage_type")
 	extra.Add(jsonutils.NewString(storage.MediumType), "medium_type")
-	if storage.Enabled {
-		extra.Add(jsonutils.JSONTrue, "enabled")
-	} else {
-		extra.Add(jsonutils.JSONFalse, "enabled")
-	}
+	extra.Add(jsonutils.NewBool(storage.Enabled), "enabled")
 	extra.Add(jsonutils.NewFloat(float64(storage.GetOvercommitBound())), "cmtbound")
+	extra.Add(jsonutils.NewInt(int64(self.GetGuestDiskCount())), "guest_disk_count")
+	if len(storage.StoragecacheId) > 0 {
+		storagecache := StoragecacheManager.FetchStoragecacheById(storage.StoragecacheId)
+		if storagecache != nil {
+			extra.Set("imagecache_path", jsonutils.NewString(storage.GetStorageCachePath(self.MountPoint, storagecache.Path)))
+			extra.Set("storagecache_id", jsonutils.NewString(storagecache.Id))
+		}
+	}
 	return extra
 }
 

pkg/compute/models/storagecaches.go

@@ -284,6 +284,23 @@ func (manager *SStoragecacheManager) ListItemFilter(ctx context.Context, q *sqlc
 	return q, nil
 }
 
+func (manager *SStoragecacheManager) FetchStoragecacheById(storageCacheId string) *SStoragecache {
+	iStorageCache, _ := manager.FetchById(storageCacheId)
+	if iStorageCache == nil {
+		return nil
+	}
+	return iStorageCache.(*SStoragecache)
+}
+
+func (manager *SStoragecacheManager) GetCachePathById(storageCacheId string) string {
+	iStorageCache, _ := manager.FetchById(storageCacheId)
+	if iStorageCache == nil {
+		return ""
+	}
+	sc := iStorageCache.(*SStoragecache)
+	return sc.Path
+}
+
 func (self *SStoragecache) ValidateDeleteCondition(ctx context.Context) error {
 	if self.getCachedImageCount() > 0 {
 		return httperrors.NewNotEmptyError("storage cache not empty")

pkg/compute/models/storages.go

@@ -2,6 +2,10 @@ package models
 
 import (
 	"context"
+	"fmt"
+	"path"
+	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -10,31 +14,51 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
-	"yunion.io/x/pkg/util/sysutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 )
 
 const (
-	STORAGE_LOCAL            = "local"
-	STORAGE_BAREMETAL        = "baremetal"
-	STORAGE_SHEEPDOG         = "sheepdog"
-	STORAGE_RBD              = "rbd"
-	STORAGE_DOCKER           = "docker"
-	STORAGE_NAS              = "nas"
-	STORAGE_VSAN             = "vsan"
+	STORAGE_LOCAL     = "local"
+	STORAGE_BAREMETAL = "baremetal"
+	STORAGE_SHEEPDOG  = "sheepdog"
+	STORAGE_RBD       = "rbd"
+	STORAGE_DOCKER    = "docker"
+	STORAGE_NAS       = "nas"
+	STORAGE_VSAN      = "vsan"
+	STORAGE_NFS       = "nfs"
+
 	STORAGE_PUBLIC_CLOUD     = "cloud"
 	STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency"
 	STORAGE_CLOUD_SSD        = "cloud_ssd"
-	STORAGE_EPHEMERAL_SSD    = "ephemeral_ssd"
-	STORAGE_GP2_SSD          = "gp2"      // aws general purpose ssd
-	STORAGE_IO1_SSD          = "io1"      // aws Provisioned IOPS SSD
-	STORAGE_ST1_HDD          = "st1"      // aws Throughput Optimized HDD
-	STORAGE_SC1_SSD          = "sc1"      // aws Cold HDD
-	STORAGE_STANDARD_SSD     = "standard" // aws Magnetic volumes
+	STORAGE_CLOUD_ESSD       = "cloud_essd" //增强型(Enhanced)SSD 云盘
 
+	//Azure hdd and ssd storagetype
+	STORAGE_STANDARD_GRS   = "standard_grs"
+	STORAGE_STANDARD_LRS   = "standard_lrs"
+	STORAGE_STANDARD_RAGRS = "standard_ragrs"
+	STORAGE_STANDARD_ZRS   = "standard_zrs"
+	STORAGE_PREMIUM_LRS    = "premium_lrs"
+
+	// aws storage type
+	STORAGE_GP2_SSD      = "gp2"      // aws general purpose ssd
+	STORAGE_IO1_SSD      = "io1"      // aws Provisioned IOPS SSD
+	STORAGE_ST1_HDD      = "st1"      // aws Throughput Optimized HDD
+	STORAGE_SC1_SSD      = "sc1"      // aws Cold HDD
+	STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes
+
+	// qcloud storage type
+	// STORAGE_CLOUD_SSD ="cloud_ssd"
+	STORAGE_LOCAL_BASIC   = "local_basic"
+	STORAGE_LOCAL_SSD     = "local_ssd"
+	STORAGE_CLOUD_BASIC   = "cloud_basic"
+	STORAGE_CLOUD_PERMIUM = "cloud_permium"
+)
+
+const (
 	STORAGE_ENABLED  = "enabled"
 	STORAGE_DISABLED = "disabled"
 	STORAGE_OFFLINE  = "offline"
@@ -47,17 +71,26 @@ const (
 
 var (
 	DISK_TYPES            = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID}
-	STORAGE_LOCAL_TYPES   = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS}
+	STORAGE_LOCAL_TYPES   = []string{STORAGE_LOCAL, STORAGE_BAREMETAL}
 	STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES
 	STORAGE_ALL_TYPES     = []string{
 		STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN,
+		STORAGE_NFS,
 	}
-	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD}
+	STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
+		STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS,
+		STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY,
+		STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS,
+		STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD,
+		STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM,
+	}
+
+	STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS}
 )
 
 type SStorageManager struct {
-	db.SEnabledStatusStandaloneResourceBaseManager
+	db.SStandaloneResourceBaseManager
 	SInfrastructureManager
 }
 
@@ -65,7 +98,7 @@ var StorageManager *SStorageManager
 
 func init() {
 	StorageManager = &SStorageManager{
-		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
+		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
 			SStorage{},
 			"storages_tbl",
 			"storage",
@@ -75,7 +108,7 @@ func init() {
 }
 
 type SStorage struct {
-	db.SEnabledStatusStandaloneResourceBase
+	db.SStandaloneResourceBase
 	SInfrastructure
 	SManagedResourceBase
 
@@ -89,6 +122,9 @@ type SStorage struct {
 	ZoneId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"admin_required"`
 
 	StoragecacheId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin"`
+
+	Enabled bool   `nullable:"false" default:"true" list:"user" create:"optional"`
+	Status  string `width:"36" charset:"ascii" nullable:"false" default:"offline" list:"user" create:"optional"`
 }
 
 func (manager *SStorageManager) GetContextManager() []db.IModelManager {
@@ -98,7 +134,12 @@ func (manager *SStorageManager) GetContextManager() []db.IModelManager {
 func (manager *SStorageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	storageType, _ := data.GetString("storage_type")
 	mediumType, _ := data.GetString("medium_type")
-	if !utils.IsInStringArray(storageType, STORAGE_ALL_TYPES) {
+	capacity, _ := data.Int("capacity")
+	if capacity <= 0 {
+		return nil, httperrors.NewInputParameterError("Invalid capacity")
+	}
+	data.Set("capacity", jsonutils.NewInt(capacity))
+	if !utils.IsInStringArray(storageType, STORAGE_TYPES) {
 		return nil, httperrors.NewInputParameterError("Invalid storage type %s", storageType)
 	}
 	if !utils.IsInStringArray(mediumType, DISK_TYPES) {
@@ -113,27 +154,196 @@ func (manager *SStorageManager) ValidateCreateData(ctx context.Context, userCred
 		return nil, httperrors.NewResourceNotFoundError("zone %s", zoneId)
 	}
 	data.Set("zone_id", jsonutils.NewString(zone.GetId()))
-	// TODO: ValidateRdbConfData
-	// if storageType == STORAGE_RBD {
-	// 	conf := jsonutils.NewDict()
-	// 	for k, v := range data.Value() {
-	// 		if strings.HasPrefix(k, fmt.Sprintf("%s_", storageType)) {
-	// 			k = k[len(storageType)+1:]
-	// 			if len(k) > 0 {
-	// 				conf.Set(k, v)
-	// 			}
-	// 		}
-	// 	}
-	// 	data.Set("capacity", manager.ValidateRdbConfData(conf))
-	// }
-	return manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+	if storageType == STORAGE_RBD {
+		conf, err := manager.ValidateRbdConfData(data)
+		if err != nil {
+			return nil, httperrors.NewBadRequestError("Vaildata rbd conf error: %s", err.Error())
+		}
+		data.Set("storage_conf", conf)
+		// data.Set("capacity", rbdConf)
+	} else if storageType == STORAGE_NFS {
+		conf, err := manager.ValidataNfsConfdata(data)
+		if err != nil {
+			return nil, httperrors.NewBadRequestError("Vaildata nfs conf error: %s", err.Error())
+		}
+		data.Set("storage_conf", conf)
+	}
+	return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
+}
+
+func (manager *SStorageManager) ValidataNfsConfdata(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	conf := jsonutils.NewDict()
+	if nfsHost, err := data.GetString("nfs_host"); err != nil {
+		return nil, httperrors.NewInputParameterError("Get nfs conf host error: %s", err.Error())
+	} else {
+		conf.Set("nfs_host", jsonutils.NewString(nfsHost))
+	}
+	if nfsSharedDir, err := data.GetString("nfs_shared_dir"); err != nil {
+		return nil, httperrors.NewInputParameterError("Get nfs conf shared dir error: %s", err.Error())
+	} else {
+		conf.Set("nfs_shared_dir", jsonutils.NewString(nfsSharedDir))
+	}
+	return conf, nil
+}
+
+func (manager *SStorageManager) ValidateRbdConfData(data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	conf := jsonutils.NewDict()
+	for k, v := range data.Value() {
+		if strings.HasPrefix(k, fmt.Sprintf("%s_", STORAGE_RBD)) {
+			k = k[len(STORAGE_RBD)+1:]
+			if len(k) > 0 {
+				conf.Set(k, v)
+			}
+		}
+	}
+	requireFields := []string{"mon_host", "key", "pool"}
+	for _, field := range requireFields {
+		if !conf.Contains(field) {
+			return nil, httperrors.NewMissingParameterError(field)
+		}
+	}
+	storages := make([]SStorage, 0)
+	err := manager.Query().Equals("storage_type", STORAGE_RBD).All(&storages)
+	if err != nil {
+		return nil, err
+	}
+	for i := 0; i < len(storages); i++ {
+		if conf.Equals(storages[i].StorageConf) {
+			return nil, httperrors.NewDuplicateResourceError("This RBD Storage[%s/%s] has already exist", storages[i].Name, conf.String())
+		}
+	}
+	// TODO??? ensure rbd pool can use and get capacity
+	return conf, nil
 }
 
 func (self *SStorage) ValidateDeleteCondition(ctx context.Context) error {
 	if self.GetHostCount() > 0 || self.GetDiskCount() > 0 || self.GetSnapshotCount() > 0 {
 		return httperrors.NewNotEmptyError("Not an empty storage provider")
 	}
-	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
+	return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx)
+}
+
+func (self *SStorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	self.SStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
+	if self.StorageType == STORAGE_RBD {
+		// TODO
+	} else if self.StorageType == STORAGE_NFS {
+		sc := &SStoragecache{}
+		sc.Path = options.Options.NfsDefaultImageCacheDir
+		sc.ExternalId = self.Id
+		sc.Name = "nfs-" + self.Name + time.Now().Format("2006-01-02 15:04:05")
+		err := StoragecacheManager.TableSpec().Insert(sc)
+		if err != nil {
+			log.Errorln(err)
+			return
+		}
+		err = StoragecacheManager.Query().Equals("external_id", self.Id).First(sc)
+		if err != nil {
+			log.Errorln(err)
+			return
+		}
+		_, err = self.GetModelManager().TableSpec().Update(self, func() error {
+			self.StoragecacheId = sc.Id
+			self.Status = STORAGE_ONLINE
+			return nil
+		})
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+}
+
+func (self *SStorage) SetStatus(userCred mcclient.TokenCredential, status string, reason string) error {
+	if self.Status == status {
+		return nil
+	}
+	oldStatus := self.Status
+	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
+		self.Status = status
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if userCred != nil {
+		notes := fmt.Sprintf("%s=>%s", oldStatus, status)
+		if len(reason) > 0 {
+			notes = fmt.Sprintf("%s: %s", notes, reason)
+		}
+		db.OpsLog.LogEvent(self, db.ACT_UPDATE_STATUS, notes, userCred)
+		if strings.Contains(notes, "fail") {
+			logclient.AddActionLog(self, logclient.ACT_VM_SYNC_STATUS, notes, userCred, false)
+		}
+	}
+	return nil
+}
+
+func (self *SStorage) AllowPerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (self *SStorage) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if !self.Enabled {
+		_, err := self.GetModelManager().TableSpec().Update(self, func() error {
+			self.Enabled = true
+			return nil
+		})
+		if err != nil {
+			log.Errorf("PerformEnable save update fail %s", err)
+			return nil, err
+		}
+		db.OpsLog.LogEvent(self, db.ACT_ENABLE, "", userCred)
+	}
+	return nil, nil
+}
+
+func (self *SStorage) AllowPerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (self *SStorage) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Enabled {
+		_, err := self.GetModelManager().TableSpec().Update(self, func() error {
+			self.Enabled = false
+			return nil
+		})
+		if err != nil {
+			log.Errorf("PerformDisable save update fail %s", err)
+			return nil, err
+		}
+		db.OpsLog.LogEvent(self, db.ACT_DISABLE, "", userCred)
+	}
+	return nil, nil
+}
+
+func (self *SStorage) AllowPerformOnline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (self *SStorage) PerformOnline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Status != STORAGE_ONLINE {
+		err := self.SetStatus(userCred, STORAGE_ONLINE, "")
+		if err != nil {
+			return nil, err
+		}
+		db.OpsLog.LogEvent(self, db.ACT_ONLINE, "", userCred)
+	}
+	return nil, nil
+}
+
+func (self *SStorage) AllowPerformOffline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsSystemAdmin()
+}
+
+func (self *SStorage) PerformOffline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Status != STORAGE_OFFLINE {
+		err := self.SetStatus(userCred, STORAGE_OFFLINE, "")
+		if err != nil {
+			return nil, err
+		}
+		db.OpsLog.LogEvent(self, db.ACT_OFFLINE, "", userCred)
+	}
+	return nil, nil
 }
 
 func (self *SStorage) GetHostCount() int {
@@ -156,6 +366,14 @@ func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcc
 	return true
 }
 
+func (self *SStorage) GetStorageCachePath(mountPoint, imageCachePath string) string {
+	if self.StorageType == STORAGE_NFS {
+		return path.Join(mountPoint, imageCachePath)
+	} else {
+		return imageCachePath
+	}
+}
+
 func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
 	used := self.GetUsedCapacity(tristate.True)
 	waste := self.GetUsedCapacity(tristate.False)
@@ -176,7 +394,7 @@ func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND
 }
 
 func (self *SStorage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
-	extra := self.SEnabledStatusStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	extra := self.SStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query)
 	return self.getMoreDetails(extra)
 }
 
@@ -761,7 +979,7 @@ func (manager *SStorageManager) InitializeData() error {
 }
 
 func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
-	q, err := manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
 	if err != nil {
 		return nil, err
 	}
@@ -777,11 +995,11 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 	}
 
 	if jsonutils.QueryBoolean(query, "share", false) {
-		q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES))
+		q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), STORAGE_LOCAL_TYPES))
 	}
 
 	if jsonutils.QueryBoolean(query, "local", false) {
-		q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES))
+		q = q.Filter(sqlchemy.In(q.Field("storage_type"), STORAGE_LOCAL_TYPES))
 	}
 
 	if jsonutils.QueryBoolean(query, "usable", false) {

pkg/compute/options/options.go

@@ -65,6 +65,8 @@ type ComputeOptions struct {
 	ConvertEsxiDefaultTemplate       string `default:"Default template" help:"ESXI baremetal convert option"`
 	ConvertKubeletDockerVolumeSize   string `default:"256g" help:"Docker volume size"`
 
+	NfsDefaultImageCacheDir string `default:"image_cache"`
+
 	cloudcommon.DBOptions
 }
 

pkg/compute/tasks/guest_detach_all_disks_task.go

@@ -43,13 +43,7 @@ func (self *GuestDetachAllDisksTask) OnDiskDeleteComplete(ctx context.Context, o
 		if jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) {
 			taskData.Add(jsonutils.JSONTrue, "override_pending_delete")
 		}
-		disk := guestdisk.GetDisk()
-		storage := disk.GetStorage()
-		if storage.IsLocal() {
-			taskData.Add(jsonutils.JSONFalse, "keep_disk")
-		} else {
-			taskData.Add(jsonutils.JSONTrue, "keep_disk")
-		}
+		taskData.Add(jsonutils.JSONFalse, "keep_disk")
 		task, err := taskman.TaskManager.NewTask(ctx, "GuestDetachDiskTask", guest, self.UserCred, taskData, self.GetTaskId(), "", nil)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())

pkg/compute/tasks/guest_detach_disk_task.go

@@ -91,15 +91,12 @@ func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest
 		db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred)
 		disk.RealDelete(ctx, self.UserCred)
 		self.SetStageComplete(ctx, nil)
-		return
-	}
-	if !keepDisk && disk.GetGuestDiskCount() == 0 && disk.AutoDelete {
+	} else if (disk.Status != models.DISK_READY || !keepDisk) && disk.GetGuestDiskCount() == 0 && disk.AutoDelete {
 		self.SetStage("on_disk_delete_complete", nil)
 		db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred)
 		err := guest.GetDriver().RequestDeleteDetachedDisk(ctx, disk, self, purge)
 		if err != nil {
 			self.OnTaskFail(ctx, guest, disk, err)
-			return
 		}
 	} else {
 		self.SetStageComplete(ctx, nil)

vendor/yunion.io/x/pkg/prettytable/prettytable.go

@@ -3,6 +3,7 @@ package prettytable
 import (
 	"bytes"
 	"strings"
+	"unicode"
 )
 
 type AlignmentType uint8
@@ -102,6 +103,17 @@ func textLine(buf *bytes.Buffer, columns []ptColumn, widths []int) {
 	}
 }
 
+func runeDisplayWidth(r rune) int {
+	const puncts = "。，；：（）、？《》"
+	if unicode.Is(unicode.Han, r) {
+		return 2
+	}
+	if strings.ContainsRune(puncts, r) {
+		return 2
+	}
+	return 1
+}
+
 // cellDisplayWidth returns display width of the cell when printed as the
 // nthCol.  prevWidth is the total display width (as return by this same func)
 // of previous cells in the same line
@@ -116,7 +128,7 @@ func cellDisplayWidth(cell string, nthCol int, prevWidth int) int {
 		for _, c := range line {
 			incr := 0
 			if c != '\t' {
-				incr = 1
+				incr = runeDisplayWidth(c)
 			} else {
 				// terminal with have the char TabWidth aligned
 				incr = TabWidth - (x & (TabWidth - 1))

vendor/yunion.io/x/pkg/util/sysutils/doc.go

@@ -1 +0,0 @@
-package sysutils // import "yunion.io/x/pkg/util/sysutils"

vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go

@@ -1,49 +0,0 @@
-package sysutils
-
-const (
-	STORAGE_LOCAL     = "local"
-	STORAGE_BAREMETAL = "baremetal"
-	STORAGE_SHEEPDOG  = "sheepdog"
-	STORAGE_RBD       = "rbd"
-	STORAGE_DOCKER    = "docker"
-	STORAGE_NAS       = "nas"
-	STORAGE_VSAN      = "vsan"
-
-	STORAGE_CLOUD            = "cloud"
-	STORAGE_CLOUD_SSD        = "cloud_ssd"
-	STORAGE_CLOUD_ESSD       = "cloud_essd" //增强型(Enhanced)SSD 云盘
-	STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency"
-
-	//Azure hdd and ssd storagetype
-	STORAGE_STANDARD_GRS   = "standard_grs"
-	STORAGE_STANDARD_LRS   = "standard_lrs"
-	STORAGE_STANDARD_RAGRS = "standard_ragrs"
-	STORAGE_STANDARD_ZRS   = "standard_zrs"
-	STORAGE_PREMIUM_LRS    = "premium_lrs"
-
-	// aws storage type
-	STORAGE_GP2_SSD      = "gp2"      // aws general purpose ssd
-	STORAGE_IO1_SSD      = "io1"      // aws Provisioned IOPS SSD
-	STORAGE_ST1_HDD      = "st1"      // aws Throughput Optimized HDD
-	STORAGE_SC1_SSD      = "sc1"      // aws Cold HDD
-	STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes
-
-	// qcloud storage type
-	// STORAGE_CLOUD_SSD ="cloud_ssd"
-	STORAGE_LOCAL_BASIC   = "local_basic"
-	STORAGE_LOCAL_SSD     = "local_ssd"
-	STORAGE_CLOUD_BASIC   = "cloud_basic"
-	STORAGE_CLOUD_PERMIUM = "cloud_permium"
-)
-
-var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
-	STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN,
-	STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY,
-	STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS,
-	STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD,
-	STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM,
-}
-
-var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL}
-
-// var SUPPORT = []string {support = [STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS]

vendor/yunion.io/x/structarg/structarg.go

@@ -187,56 +187,6 @@ func (this *ArgumentParser) addStructArgument(tp reflect.Type, val reflect.Value
 	return nil
 }
 
-/*func findWord(str []byte, offset int) (string, int) {
-	var buffer bytes.Buffer
-	i := skipEmpty(str, offset)
-	if i >= len(str) {
-		return "", i
-	}
-	var endstr string
-	quote := false
-	if str[i] == '"' {
-		quote = true
-		endstr = "\""
-		i++
-	} else if str[i] == '\'' {
-		quote = true
-		endstr = "'"
-		i++
-	} else {
-		endstr = " :,\t\n}]"
-	}
-	for i < len(str) {
-		if quote && str[i] == '\\' {
-			if i+1 < len(str) {
-				i++
-				switch str[i] {
-				case 'n':
-					buffer.WriteByte('\n')
-				case 'r':
-					buffer.WriteByte('\r')
-				case 't':
-					buffer.WriteByte('\t')
-				default:
-					buffer.WriteByte(str[i])
-				}
-				i++
-			} else {
-				break
-			}
-		} else if strings.IndexByte(endstr, str[i]) >= 0 { // end
-			if quote {
-				i++
-			}
-			break
-		} else {
-			buffer.WriteByte(str[i])
-			i++
-		}
-	}
-	return buffer.String(), i
-}*/
-
 func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) error {
 	tagMap := utils.TagMap(f.Tag)
 	help := tagMap[TAG_HELP]
@@ -901,6 +851,14 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error
 	return err
 }
 
+func isQuotedByChar(str string, quoteChar byte) bool {
+	return str[0] == quoteChar && str[len(str)-1] == quoteChar
+}
+
+func isQuoted(str string) bool {
+	return isQuotedByChar(str, '"') || isQuotedByChar(str, '\'')
+}
+
 func (this *ArgumentParser) parseKeyValue(key, value string) error {
 	arg := this.findOptionalArgument(key)
 	if arg != nil {
@@ -914,7 +872,15 @@ func (this *ArgumentParser) parseKeyValue(key, value string) error {
 				}
 			}
 		} else {
-			return arg.SetValue(value)
+			if !isQuoted(value) {
+				value = fmt.Sprintf("\"%s\"", value)
+			}
+			values := utils.FindWords([]byte(value), 0)
+			if len(values) == 1 {
+				return arg.SetValue(values[0])
+			} else {
+				log.Warningf("too many arguments %#v for %s", values, key)
+			}
 		}
 	} else {
 		log.Warningf("Cannot find argument %s", key)
@@ -964,7 +930,7 @@ func (this *ArgumentParser) ParseFile(filepath string) error {
 	for scanner.Scan() {
 		line := scanner.Text()
 		line = strings.TrimSpace(removeComments(line))
-		line = removeCharacters(line, `"'`)
+		// line = removeCharacters(line, `"'`)
 		if len(line) > 0 {
 			key, val, e := line2KeyValue(line)
 			if e == nil {