Merge branch 'master' into hotfix/qx-sku-validate

cmd/climc/shell/networks.go

@@ -35,6 +35,7 @@ func init() {
 		Vpc        string `help:"search networks belongs to a VPC"`
 		Region     string `help:"search networks belongs to a CloudRegion" json:"cloudregion"`
 		City       string `help:"search networks belongs to a city"`
+		Usable     *bool  `help:"search usable networks"`
 		ServerType string `help:"search networks belongs to a ServerType" choices:"guest|baremetal|container|pxe|ipmi"`
 	}
 	R(&NetworkListOptions{}, "network-list", "List networks", func(s *mcclient.ClientSession, opts *NetworkListOptions) error {

go.mod

@@ -153,6 +153,6 @@ require (
 	yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
 	yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30
-	yunion.io/x/sqlchemy v0.0.0-20190701085445-12b2b4318f7d
+	yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda
 	yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe
 )

go.sum

@@ -482,7 +482,7 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRi
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30 h1:6CkrwtX4xeYFqqpdWtQPAVqEnD3aEivPLOLzNKNPAB0=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/sqlchemy v0.0.0-20190701085445-12b2b4318f7d h1:b6gMLwNLzovRgmENv2tBJZKCf3gydQqnJmDd9QeVNrg=
-yunion.io/x/sqlchemy v0.0.0-20190701085445-12b2b4318f7d/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda h1:i+/3Hh+kVmPZM1P+j3wfViEzb0XlttWyNvDSYA5DDDU=
+yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe h1:Wr2EXv72ynwtW+x0VCqWwpsfUBWecaZWnLMgaP3UiTo=
 yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/guest_const.go

@@ -38,6 +38,8 @@ const (
 	VM_STOP_FAILED     = "stop_fail" // # = running
 	VM_RENEWING        = "renewing"
 	VM_RENEW_FAILED    = "renew_failed"
+	VM_ATTACH_DISK     = "attach_disk"
+	VM_DETACH_DISK     = "detach_disk"
 
 	VM_BACKUP_STARTING         = "backup_starting"
 	VM_BACKUP_CREATING         = "backup_creating"

pkg/cloudcommon/db/metadata.go

@@ -158,13 +158,13 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	conditions := []sqlchemy.ICondition{}
 	admin := jsonutils.QueryBoolean(query, "admin", false)
 	for _, resource := range resources {
-		if manager, ok := ResourceMap[resource]; ok {
-			resourceView := manager.Query().SubQuery()
-			prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", manager.Keyword()))
-			field := sqlchemy.CONCAT(manager.Keyword(), prefix, resourceView.Field("id"))
+		if man, ok := ResourceMap[resource]; ok {
+			resourceView := man.Query().SubQuery()
+			prefix := sqlchemy.NewStringField(fmt.Sprintf("%s::", man.Keyword()))
+			field := sqlchemy.CONCAT(man.Keyword(), prefix, resourceView.Field("id"))
 			sq := resourceView.Query(field)
-			if !admin || !IsAllowList(rbacutils.ScopeSystem, userCred, manager) {
-				sq = manager.FilterByOwner(sq, userCred, manager.ResourceScope())
+			if !admin || !IsAllowList(rbacutils.ScopeSystem, userCred, man) {
+				sq = man.FilterByOwner(sq, userCred, man.ResourceScope())
 			}
 			conditions = append(conditions, sqlchemy.In(q.Field("id"), sq))
 		} else {
@@ -179,6 +179,18 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy
 			q = q.Filter(sqlchemy.Startswith(q.Field("key"), prefix))
 		}
 	}
+
+	withConditions := []sqlchemy.ICondition{}
+	for args, prefix := range map[string]string{"with_sys_meta": SYS_TAG_PREFIX, "with_cloud_meta": CLOUD_TAG_PREFIX, "with_user_meta": USER_TAG_PREFIX} {
+		if jsonutils.QueryBoolean(query, args, false) {
+			withConditions = append(withConditions, sqlchemy.Startswith(q.Field("key"), prefix))
+		}
+	}
+
+	if len(withConditions) > 0 {
+		q = q.Filter(sqlchemy.OR(withConditions...))
+	}
+
 	return q, nil
 }
 

pkg/cloudcommon/db/quotas/handler.go

@@ -116,7 +116,9 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, scope rbacutil
 
 	ret.Update(quota.ToJSON(""))
 	ret.Update(usage.ToJSON("usage"))
-	ret.Update(pending.ToJSON("pending"))
+	if !pending.IsEmpty() {
+		ret.Update(pending.ToJSON("pending"))
+	}
 
 	if scope == rbacutils.ScopeDomain {
 		total, err := manager.getDomainTotalQuota(ctx, ownerId.GetProjectDomainId(), nil)

pkg/cloudcommon/db/standalone.go

@@ -17,7 +17,6 @@ package db
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -160,7 +159,7 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex
 	}
 
 	if len(tags) > 0 {
-		metadataView := Metadata.Query("id")
+		metadataView := Metadata.Query()
 		idx := 0
 		for k, v := range tags {
 			if idx == 0 {
@@ -178,25 +177,26 @@ func (manager *SStandaloneResourceBaseManager) ListItemFilter(ctx context.Contex
 			}
 			idx++
 		}
-		metadataView = metadataView.Filter(sqlchemy.Like(metadataView.Field("id"), manager.Keyword()+"::%")).Distinct()
-		resourceIds := []string{}
-		rows, err := metadataView.Rows()
-		if err != nil {
-			log.Errorf("query metadata ids error: %v", err)
-			return nil, err
-		}
-		defer rows.Close()
-		for rows.Next() {
-			var metadataID string
-			err = rows.Scan(&metadataID)
-			if err != nil {
-				log.Errorf("get metadata id scan error: %v", err)
-				return nil, err
-			}
-			resourceIds = append(resourceIds, strings.TrimPrefix(metadataID, manager.Keyword()+"::"))
-		}
-		q = q.Filter(sqlchemy.In(q.Field("id"), resourceIds))
+		metadatas := metadataView.SubQuery()
+		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
+		metadataSQ := metadatas.Query(
+			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
+		)
+		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("id"), manager.Keyword()+"::%")).Distinct()
+		q = q.Filter(sqlchemy.In(q.Field("id"), sq))
 	}
+
+	if withoutUserMeta, _ := query.Bool("without_user_meta"); withoutUserMeta {
+		metadatas := Metadata.Query().SubQuery()
+		fieldName := fmt.Sprintf("%s_id", manager.Keyword())
+		metadataSQ := metadatas.Query(
+			sqlchemy.REPLACE(fieldName, metadatas.Field("id"), manager.Keyword()+"::", ""),
+		)
+		sq := metadataSQ.Filter(sqlchemy.Like(metadatas.Field("key"), USER_TAG_PREFIX+"%")).Distinct()
+
+		q.Filter(sqlchemy.NotIn(q.Field("id"), sq))
+	}
+
 	return q, nil
 }
 

pkg/cloudprovider/waitstatus.go

@@ -77,6 +77,21 @@ func WaitDeleted(res ICloudResource, interval time.Duration, timeout time.Durati
 	return ErrTimeout
 }
 
+func Wait(interval time.Duration, timeout time.Duration, callback func() (bool, error)) error {
+	startTime := time.Now()
+	for time.Now().Sub(startTime) < timeout {
+		ok, err := callback()
+		if err != nil {
+			return err
+		}
+		if ok {
+			return nil
+		}
+		time.Sleep(interval)
+	}
+	return ErrTimeout
+}
+
 func WaitCreated(interval time.Duration, timeout time.Duration, callback func() bool) error {
 	startTime := time.Now()
 	for time.Now().Sub(startTime) < timeout {

pkg/compute/guestdrivers/managedvirtual.go

@@ -161,6 +161,34 @@ func (self *SManagedVirtualizedGuestDriver) RequestDetachDisk(ctx context.Contex
 		if err != nil {
 			return nil, errors.Wrapf(err, "iVM.DetachDisk")
 		}
+
+		err = cloudprovider.Wait(time.Second*5, time.Minute*3, func() (bool, error) {
+			err := iVM.Refresh()
+			if err != nil {
+				return false, errors.Wrapf(err, "iVM.Refresh")
+			}
+			iDisks, err := iVM.GetIDisks()
+			if err != nil {
+				return false, errors.Wrapf(err, "RequestDetachDisk.iVM.GetIDisks")
+			}
+
+			exist := false
+			for i := 0; i < len(iDisks); i++ {
+				if iDisks[i].GetGlobalId() == disk.ExternalId {
+					exist = true
+				}
+			}
+
+			if !exist {
+				return true, nil
+			}
+			return false, nil
+		})
+
+		if err != nil {
+			return nil, errors.Wrapf(err, "RequestDetachDisk.Wait")
+		}
+
 		return nil, nil
 	})
 	return nil
@@ -179,6 +207,31 @@ func (self *SManagedVirtualizedGuestDriver) RequestAttachDisk(ctx context.Contex
 		if err != nil {
 			return nil, errors.Wrapf(err, "iVM.AttachDisk")
 		}
+
+		err = cloudprovider.Wait(time.Second*5, time.Minute*3, func() (bool, error) {
+			err := iVM.Refresh()
+			if err != nil {
+				return false, errors.Wrapf(err, "iVM.Refresh")
+			}
+
+			iDisks, err := iVM.GetIDisks()
+			if err != nil {
+				return false, errors.Wrapf(err, "RequestAttachDisk.iVM.GetIDisks")
+			}
+
+			for i := 0; i < len(iDisks); i++ {
+				if iDisks[i].GetGlobalId() == disk.ExternalId {
+					return true, nil
+				}
+			}
+
+			return false, nil
+		})
+
+		if err != nil {
+			return nil, errors.Wrapf(err, "RequestAttachDisk.Wait")
+		}
+
 		return nil, nil
 	})
 	return nil

pkg/compute/models/cloudproviders.go

@@ -1088,10 +1088,6 @@ func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.To
 		SnapshotPolicyManager,
 		StorageManager,
 		StoragecacheManager,
-		LoadbalancerListenerRuleManager,
-		LoadbalancerListenerManager,
-		LoadbalancerBackendManager,
-		LoadbalancerBackendGroupManager,
 		LoadbalancerManager,
 		LoadbalancerAclManager,
 		LoadbalancerCertificateManager,

pkg/compute/models/guest_actions.go

@@ -582,6 +582,8 @@ func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.Tok
 	taskData := data.(*jsonutils.JSONDict)
 	taskData.Set("disk_id", jsonutils.NewString(disk.GetId()))
 
+	self.SetStatus(userCred, api.VM_ATTACH_DISK, "")
+
 	if err := self.GetDriver().StartGuestAttachDiskTask(ctx, userCred, self, taskData, ""); err != nil {
 		return nil, err
 	}
@@ -1422,6 +1424,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok
 				return nil, httperrors.NewInputParameterError("Cannot keep detached disk")
 			}
 			if utils.IsInStringArray(self.Status, detachDiskStatus) {
+				self.SetStatus(userCred, api.VM_DETACH_DISK, "")
 				err = self.StartGuestDetachdiskTask(ctx, userCred, disk, keepDisk, "")
 				return nil, err
 			} else {

pkg/compute/models/loadbalancerbackendgroups.go

@@ -175,6 +175,16 @@ func (man *SLoadbalancerBackendGroupManager) ValidateCreateData(ctx context.Cont
 	return region.GetDriver().ValidateCreateLoadbalancerBackendGroupData(ctx, userCred, data, lb, backends)
 }
 
+func (lbbg *SLoadbalancerBackendGroup) GetLoadbalancerListeners() ([]SLoadbalancerListener, error) {
+	q := LoadbalancerListenerManager.Query().Equals("backend_group_id", lbbg.Id).IsFalse("pending_deleted")
+	listeners := []SLoadbalancerListener{}
+	err := db.FetchModelObjects(LoadbalancerListenerManager, q, &listeners)
+	if err != nil {
+		return nil, err
+	}
+	return listeners, nil
+}
+
 func (lbbg *SLoadbalancerBackendGroup) GetLoadbalancer() *SLoadbalancer {
 	lb, err := LoadbalancerManager.FetchById(lbbg.LoadbalancerId)
 	if err != nil {
@@ -256,11 +266,7 @@ func (lbbg *SLoadbalancerBackendGroup) ValidateDeleteCondition(ctx context.Conte
 		}
 	}
 
-	region := lbbg.GetRegion()
-	if region == nil {
-		return nil
-	}
-	return region.GetDriver().ValidateDeleteLoadbalancerBackendGroupCondition(ctx, lbbg)
+	return nil
 }
 
 func (lbbg *SLoadbalancerBackendGroup) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {

pkg/compute/models/loadbalancerbackends.go

@@ -426,14 +426,7 @@ func (man *SLoadbalancerBackendManager) getLoadbalancerBackendsByLoadbalancerBac
 }
 
 func (lbb *SLoadbalancerBackend) ValidateDeleteCondition(ctx context.Context) error {
-	if err := lbb.SVirtualResourceBase.ValidateDeleteCondition(ctx); err != nil {
-		return err
-	}
-	region := lbb.GetRegion()
-	if region == nil {
-		return nil
-	}
-	return region.GetDriver().ValidateDeleteLoadbalancerBackendCondition(ctx, lbb)
+	return lbb.SVirtualResourceBase.ValidateDeleteCondition(ctx)
 }
 
 func (man *SLoadbalancerBackendManager) SyncLoadbalancerBackends(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, loadbalancerBackendgroup *SLoadbalancerBackendGroup, lbbs []cloudprovider.ICloudLoadbalancerBackend, syncRange *SSyncRange) compare.SyncResult {

pkg/compute/models/loadbalancerlisteners.go

@@ -678,6 +678,16 @@ func (lblis *SLoadbalancerListener) GetLoadbalancerListenerParams() (*cloudprovi
 	return listener, nil
 }
 
+func (lblis *SLoadbalancerListener) GetLoadbalancerListenerRules() ([]SLoadbalancerListenerRule, error) {
+	q := LoadbalancerListenerRuleManager.Query().Equals("listener_id", lblis.Id).IsFalse("pending_deleted")
+	rules := []SLoadbalancerListenerRule{}
+	err := db.FetchModelObjects(LoadbalancerListenerRuleManager, q, &rules)
+	if err != nil {
+		return nil, err
+	}
+	return rules, nil
+}
+
 func (lblis *SLoadbalancerListener) GetLoadbalancerCertificate() *SLoadbalancerCertificate {
 	if len(lblis.CertificateId) == 0 {
 		return nil

pkg/compute/models/loadbalancers.go

@@ -519,7 +519,7 @@ func (lb *SLoadbalancer) GetLoadbalancerListeners() ([]SLoadbalancerListener, er
 func (lb *SLoadbalancer) GetLoadbalancerBackendgroups() ([]SLoadbalancerBackendGroup, error) {
 	lbbgs := []SLoadbalancerBackendGroup{}
 	q := LoadbalancerBackendGroupManager.Query().Equals("loadbalancer_id", lb.Id).IsFalse("pending_deleted")
-	if err := db.FetchModelObjects(LoadbalancerListenerManager, q, &lbbgs); err != nil {
+	if err := db.FetchModelObjects(LoadbalancerBackendGroupManager, q, &lbbgs); err != nil {
 		return nil, err
 	}
 	return lbbgs, nil

pkg/compute/models/networks.go

@@ -1539,6 +1539,24 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy.
 		q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery()))
 	}
 
+	if jsonutils.QueryBoolean(query, "usable", false) {
+		wires := WireManager.Query().SubQuery()
+		zones := ZoneManager.Query().SubQuery()
+		vpcs := VpcManager.Query().SubQuery()
+		regions := CloudregionManager.Query().SubQuery()
+
+		sq := wires.Query(wires.Field("id")).
+			Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))).
+			Join(zones, sqlchemy.Equals(wires.Field("zone_id"), zones.Field("id"))).
+			Join(regions, sqlchemy.Equals(zones.Field("cloudregion_id"), regions.Field("id"))).
+			Filter(sqlchemy.AND(
+				sqlchemy.Equals(vpcs.Field("status"), api.VPC_STATUS_AVAILABLE),
+				sqlchemy.Equals(zones.Field("status"), api.ZONE_ENABLE),
+				sqlchemy.Equals(regions.Field("status"), api.CLOUD_REGION_STATUS_INSERVER),
+			))
+		q = q.In("wire_id", sq.SubQuery()).Equals("status", api.NETWORK_STATUS_AVAILABLE)
+	}
+
 	hostStr, _ := query.GetString("host")
 	if len(hostStr) > 0 {
 		hostObj, err := HostManager.FetchByIdOrName(userCred, hostStr)

pkg/compute/models/purge.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
@@ -247,11 +248,59 @@ func (lb *SLoadbalancer) detachAllNetworks(ctx context.Context, userCred mcclien
 	return nil
 }
 
+func (lb *SLoadbalancer) purgeBackendGroups(ctx context.Context, userCred mcclient.TokenCredential) error {
+	backendGroups, err := lb.GetLoadbalancerBackendgroups()
+	if err != nil {
+		return err
+	}
+	for i := range backendGroups {
+		err = backendGroups[i].purge(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (lb *SLoadbalancer) purgeListeners(ctx context.Context, userCred mcclient.TokenCredential) error {
+	listeners, err := lb.GetLoadbalancerListeners()
+	if err != nil {
+		return err
+	}
+	for i := range listeners {
+		err = listeners[i].purge(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (lb *SLoadbalancer) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, lb)
 	defer lockman.ReleaseObject(ctx, lb)
 
-	err := lb.detachAllNetworks(ctx, userCred)
+	_, err := db.UpdateWithLock(ctx, lb, func() error {
+		//避免 purge backendgroups 时循环依赖
+		lb.BackendGroupId = ""
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("loadbalancer %s(%s): clear up backend group error: %v", lb.Name, lb.Id, err)
+	}
+
+	err = lb.detachAllNetworks(ctx, userCred)
+	if err != nil {
+		return err
+	}
+
+	err = lb.purgeBackendGroups(ctx, userCred)
+	if err != nil {
+		return err
+	}
+
+	err = lb.purgeListeners(ctx, userCred)
 	if err != nil {
 		return err
 	}
@@ -265,14 +314,14 @@ func (lb *SLoadbalancer) purge(ctx context.Context, userCred mcclient.TokenCrede
 	return nil
 }
 
-func (manager *SLoadbalancerListenerManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
-	lbls := make([]SLoadbalancerListener, 0)
-	err := fetchByManagerId(manager, providerId, &lbls)
+func (lbl *SLoadbalancerListener) purgeListenerRules(ctx context.Context, userCred mcclient.TokenCredential) error {
+	listenerRules, err := lbl.GetLoadbalancerListenerRules()
 	if err != nil {
 		return err
 	}
-	for i := range lbls {
-		err := lbls[i].purge(ctx, userCred)
+
+	for i := range listenerRules {
+		err = listenerRules[i].purge(ctx, userCred)
 		if err != nil {
 			return err
 		}
@@ -284,7 +333,12 @@ func (lbl *SLoadbalancerListener) purge(ctx context.Context, userCred mcclient.T
 	lockman.LockObject(ctx, lbl)
 	defer lockman.ReleaseObject(ctx, lbl)
 
-	err := lbl.ValidateDeleteCondition(ctx)
+	err := lbl.purgeListenerRules(ctx, userCred)
+	if err != nil {
+		return err
+	}
+
+	err = lbl.ValidateDeleteCondition(ctx)
 	if err != nil {
 		return err
 	}
@@ -293,21 +347,6 @@ func (lbl *SLoadbalancerListener) purge(ctx context.Context, userCred mcclient.T
 	return nil
 }
 
-func (manager *SLoadbalancerListenerRuleManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
-	lblrs := make([]SLoadbalancerListenerRule, 0)
-	err := fetchByManagerId(manager, providerId, &lblrs)
-	if err != nil {
-		return err
-	}
-	for i := range lblrs {
-		err := lblrs[i].purge(ctx, userCred)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (lblr *SLoadbalancerListenerRule) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, lblr)
 	defer lockman.ReleaseObject(ctx, lblr)
@@ -319,14 +358,27 @@ func (lblr *SLoadbalancerListenerRule) purge(ctx context.Context, userCred mccli
 	return lblr.DoPendingDelete(ctx, userCred)
 }
 
-func (manager *SLoadbalancerBackendGroupManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
-	lbbgs := make([]SLoadbalancerBackendGroup, 0)
-	err := fetchByManagerId(manager, providerId, &lbbgs)
+func (lbbg *SLoadbalancerBackendGroup) purgeBackends(ctx context.Context, userCred mcclient.TokenCredential) error {
+	backends, err := lbbg.GetBackends()
 	if err != nil {
 		return err
 	}
-	for i := range lbbgs {
-		err := lbbgs[i].purge(ctx, userCred)
+	for i := range backends {
+		err = backends[i].purge(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (lbbg *SLoadbalancerBackendGroup) purgeListeners(ctx context.Context, userCred mcclient.TokenCredential) error {
+	listeners, err := lbbg.GetLoadbalancerListeners()
+	if err != nil {
+		return err
+	}
+	for i := range listeners {
+		err = listeners[i].purge(ctx, userCred)
 		if err != nil {
 			return err
 		}
@@ -338,7 +390,17 @@ func (lbbg *SLoadbalancerBackendGroup) purge(ctx context.Context, userCred mccli
 	lockman.LockObject(ctx, lbbg)
 	defer lockman.ReleaseObject(ctx, lbbg)
 
-	err := lbbg.ValidateDeleteCondition(ctx)
+	err := lbbg.purgeBackends(ctx, userCred)
+	if err != nil {
+		return err
+	}
+
+	err = lbbg.purgeListeners(ctx, userCred)
+	if err != nil {
+		return err
+	}
+
+	err = lbbg.ValidateDeleteCondition(ctx)
 	if err != nil {
 		return err
 	}
@@ -347,21 +409,6 @@ func (lbbg *SLoadbalancerBackendGroup) purge(ctx context.Context, userCred mccli
 	return nil
 }
 
-func (manager *SLoadbalancerBackendManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
-	lbbs := make([]SLoadbalancerBackend, 0)
-	err := fetchByManagerId(manager, providerId, &lbbs)
-	if err != nil {
-		return err
-	}
-	for i := range lbbs {
-		err := lbbs[i].purge(ctx, userCred)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (lbb *SLoadbalancerBackend) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
 	lockman.LockObject(ctx, lbb)
 	defer lockman.ReleaseObject(ctx, lbb)

pkg/compute/models/quotas.go

@@ -296,41 +296,41 @@ func keyName(prefix, name string) string {
 
 func (self *SQuota) ToJSON(prefix string) jsonutils.JSONObject {
 	ret := jsonutils.NewDict()
-	if self.Cpu > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Cpu)), keyName(prefix, "cpu"))
-	}
-	if self.Memory > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Memory)), keyName(prefix, "memory"))
-	}
-	if self.Storage > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Storage)), keyName(prefix, "storage"))
-	}
-	if self.Port > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Port)), keyName(prefix, "port"))
-	}
-	if self.Eip > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Eip)), keyName(prefix, "eip"))
-	}
-	if self.Eport > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Eport)), keyName(prefix, "eport"))
-	}
-	if self.Bw > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw"))
-	}
-	if self.Ebw > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw"))
-	}
-	if self.Group > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Group)), keyName(prefix, "group"))
-	}
-	if self.Secgroup > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Secgroup)), keyName(prefix, "secgroup"))
-	}
-	if self.IsolatedDevice > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.IsolatedDevice)), keyName(prefix, "isolated_device"))
-	}
-	if self.Snapshot > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot"))
-	}
+	// if self.Cpu > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Cpu)), keyName(prefix, "cpu"))
+	//}
+	// if self.Memory > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Memory)), keyName(prefix, "memory"))
+	//}
+	//if self.Storage > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Storage)), keyName(prefix, "storage"))
+	//}
+	//if self.Port > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Port)), keyName(prefix, "port"))
+	//}
+	//if self.Eip > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Eip)), keyName(prefix, "eip"))
+	//}
+	//if self.Eport > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Eport)), keyName(prefix, "eport"))
+	//}
+	//if self.Bw > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Bw)), keyName(prefix, "bw"))
+	//}
+	//if self.Ebw > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Ebw)), keyName(prefix, "ebw"))
+	//}
+	//if self.Group > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Group)), keyName(prefix, "group"))
+	//}
+	//if self.Secgroup > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Secgroup)), keyName(prefix, "secgroup"))
+	//}
+	//if self.IsolatedDevice > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.IsolatedDevice)), keyName(prefix, "isolated_device"))
+	//}
+	//if self.Snapshot > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot"))
+	//}
 	return ret
 }

pkg/compute/models/regiondrivers.go

@@ -50,7 +50,6 @@ type IRegionDriver interface {
 	ValidateCreateLoadbalancerBackendGroupData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, lb *SLoadbalancer, backends []cloudprovider.SLoadbalancerBackend) (*jsonutils.JSONDict, error)
 	RequestCreateLoadbalancerBackendGroup(ctx context.Context, userCred mcclient.TokenCredential, lbbg *SLoadbalancerBackendGroup, backends []cloudprovider.SLoadbalancerBackend, task taskman.ITask) error
 	RequestDeleteLoadbalancerBackendGroup(ctx context.Context, userCred mcclient.TokenCredential, lbbg *SLoadbalancerBackendGroup, task taskman.ITask) error
-	ValidateDeleteLoadbalancerBackendGroupCondition(ctx context.Context, lbbb *SLoadbalancerBackendGroup) error
 	GetBackendStatusForAdd() []string
 
 	ValidateCreateLoadbalancerBackendData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, backendType string, lb *SLoadbalancer, backendGroup *SLoadbalancerBackendGroup, backend db.IModel) (*jsonutils.JSONDict, error)
@@ -58,7 +57,6 @@ type IRegionDriver interface {
 	RequestCreateLoadbalancerBackend(ctx context.Context, userCred mcclient.TokenCredential, lbb *SLoadbalancerBackend, task taskman.ITask) error
 	RequestDeleteLoadbalancerBackend(ctx context.Context, userCred mcclient.TokenCredential, lbb *SLoadbalancerBackend, task taskman.ITask) error
 	RequestSyncLoadbalancerBackend(ctx context.Context, userCred mcclient.TokenCredential, lbb *SLoadbalancerBackend, task taskman.ITask) error
-	ValidateDeleteLoadbalancerBackendCondition(ctx context.Context, lbb *SLoadbalancerBackend) error
 
 	ValidateCreateLoadbalancerListenerData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, backendGroup db.IModel) (*jsonutils.JSONDict, error)
 	ValidateUpdateLoadbalancerListenerData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, lblist *SLoadbalancerListener, backendGroup db.IModel) (*jsonutils.JSONDict, error)

pkg/compute/regiondrivers/aliyun.go

@@ -79,21 +79,6 @@ func (self *SAliyunRegionDriver) ValidateUpdateLoadbalancerCertificateData(ctx c
 	return data, nil
 }
 
-func (self *SAliyunRegionDriver) ValidateDeleteLoadbalancerBackendCondition(ctx context.Context, lbb *models.SLoadbalancerBackend) error {
-	backendGroup := lbb.GetLoadbalancerBackendGroup()
-	if backendGroup.Type == api.LB_BACKENDGROUP_TYPE_MASTER_SLAVE {
-		return httperrors.NewUnsupportOperationError("backend %s belong master slave backendgroup, not allow delete", lbb.Name)
-	}
-	return nil
-}
-
-func (self *SAliyunRegionDriver) ValidateDeleteLoadbalancerBackendGroupCondition(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup) error {
-	if lbbg.Type == api.LB_BACKENDGROUP_TYPE_DEFAULT {
-		return httperrors.NewUnsupportOperationError("not allow to delete default backend group")
-	}
-	return nil
-}
-
 func (self *SAliyunRegionDriver) GetBackendStatusForAdd() []string {
 	return []string{api.VM_RUNNING}
 }

pkg/compute/regiondrivers/kvm.go

@@ -236,14 +236,6 @@ func (self *SKVMRegionDriver) RequestCreateLoadbalancerBackend(ctx context.Conte
 	return nil
 }
 
-func (self *SKVMRegionDriver) ValidateDeleteLoadbalancerBackendCondition(ctx context.Context, lbb *models.SLoadbalancerBackend) error {
-	return nil
-}
-
-func (self *SKVMRegionDriver) ValidateDeleteLoadbalancerBackendGroupCondition(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup) error {
-	return nil
-}
-
 func (self *SKVMRegionDriver) GetBackendStatusForAdd() []string {
 	return []string{api.VM_RUNNING, api.VM_READY}
 }

pkg/compute/regiondrivers/managedvirtual.go

@@ -152,14 +152,6 @@ func (self *SManagedVirtualizationRegionDriver) ValidateUpdateLoadbalancerListen
 	return data, nil
 }
 
-func (self *SManagedVirtualizationRegionDriver) ValidateDeleteLoadbalancerBackendCondition(ctx context.Context, lbb *models.SLoadbalancerBackend) error {
-	return nil
-}
-
-func (self *SManagedVirtualizationRegionDriver) ValidateDeleteLoadbalancerBackendGroupCondition(ctx context.Context, lbbg *models.SLoadbalancerBackendGroup) error {
-	return nil
-}
-
 func (self *SManagedVirtualizationRegionDriver) GetBackendStatusForAdd() []string {
 	return []string{api.VM_RUNNING}
 }

pkg/compute/tasks/guest_attach_disk_task.go

@@ -76,7 +76,7 @@ func (self *GuestAttachDiskTask) OnSyncConfigComplete(ctx context.Context, guest
 		self.OnTaskFail(ctx, guest, nil, fmt.Errorf("Connot find disk %s", diskId))
 		return
 	}
-	disk.SetStatus(self.UserCred, api.DISK_READY, "")
+	disk.SetDiskReady(ctx, self.UserCred, "")
 	self.SetStageComplete(ctx, nil)
 	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_ATTACH_DISK, nil, self.UserCred, true)
 }

pkg/compute/tasks/guest_deploy_task.go

@@ -150,6 +150,8 @@ func (self *GuestDeployTask) OnDeployGuestCompleteFailed(ctx context.Context, ob
 	}
 	guest.SetStatus(self.UserCred, api.VM_DEPLOY_FAILED, data.String())
 	self.SetStageFailed(ctx, data.String())
+	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_DEPLOY, data, self.UserCred, false)
+	db.OpsLog.LogEvent(guest, db.ACT_VM_DEPLOY_FAIL, data, self.UserCred)
 }
 
 func (self *GuestDeployTask) OnDeployStartGuestComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/tasks/guest_detach_disk_task.go

@@ -22,7 +22,6 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -67,20 +66,16 @@ func (self *GuestDetachDiskTask) OnInit(ctx context.Context, obj db.IStandaloneM
 	if host != nil && host.Status == api.HOST_DISABLED && jsonutils.QueryBoolean(self.Params, "purge", false) {
 		purge = true
 	}
-	detachStatus, err := guest.GetDriver().GetDetachDiskStatus()
-	if err != nil {
-		self.OnTaskFail(ctx, guest, disk, err)
-		return
-	}
-	if utils.IsInStringArray(guest.Status, detachStatus) && !purge {
-		self.SetStage("on_sync_config_complete", nil)
+
+	if !purge {
+		self.SetStage("OnDetachDiskComplete", nil)
 		guest.GetDriver().RequestDetachDisk(ctx, guest, disk, self)
 	} else {
-		self.OnSyncConfigComplete(ctx, guest, nil)
+		self.OnDetachDiskComplete(ctx, guest, nil)
 	}
 }
 
-func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+func (self *GuestDetachDiskTask) OnDetachDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	diskId, _ := self.Params.GetString("disk_id")
 	objDisk, err := models.DiskManager.FetchById(diskId)
 	if err != nil {
@@ -92,7 +87,7 @@ func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest
 		self.OnTaskFail(ctx, guest, nil, fmt.Errorf("Connot find disk %s", diskId))
 		return
 	}
-	disk.SetDiskReady(ctx, self.UserCred, "")
+	disk.SetStatus(self.UserCred, api.DISK_READY, "")
 	keepDisk := jsonutils.QueryBoolean(self.Params, "keep_disk", true)
 	host := guest.GetHost()
 	purge := false
@@ -114,7 +109,7 @@ func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest
 	self.OnDiskDeleteComplete(ctx, guest, nil)
 }
 
-func (self *GuestDetachDiskTask) OnSyncConfigCompleteFailed(ctx context.Context, obj db.IStandaloneModel, reason jsonutils.JSONObject) {
+func (self *GuestDetachDiskTask) OnDetachDiskCompleteFailed(ctx context.Context, obj db.IStandaloneModel, reason jsonutils.JSONObject) {
 	guest := obj.(*models.SGuest)
 	driver, _ := self.Params.GetString("driver")
 	cache, _ := self.Params.GetString("cache")
@@ -128,7 +123,7 @@ func (self *GuestDetachDiskTask) OnSyncConfigCompleteFailed(ctx context.Context,
 	}
 	disk := objDisk.(*models.SDisk)
 	db.OpsLog.LogEvent(disk, db.ACT_DETACH, reason.String(), self.UserCred)
-	disk.SetDiskReady(ctx, self.UserCred, "")
+	disk.SetStatus(self.UserCred, api.DISK_READY, "")
 	err = guest.AttachDisk(ctx, disk, self.UserCred, driver, cache, mountpoint)
 	if err != nil {
 		log.Warningf("recover attach disk %s(%s) for guest %s(%s) error: %v", disk.Name, disk.Id, guest.Name, guest.Id, err)
@@ -138,7 +133,7 @@ func (self *GuestDetachDiskTask) OnSyncConfigCompleteFailed(ctx context.Context,
 
 func (self *GuestDetachDiskTask) OnTaskFail(ctx context.Context, guest *models.SGuest, disk *models.SDisk, err error) {
 	if disk != nil {
-		disk.SetDiskReady(ctx, self.UserCred, "")
+		disk.SetStatus(self.UserCred, api.DISK_READY, "")
 	}
 	guest.SetStatus(self.UserCred, api.VM_DETACH_DISK_FAILED, err.Error())
 	self.SetStageFailed(ctx, err.Error())
@@ -147,6 +142,16 @@ func (self *GuestDetachDiskTask) OnTaskFail(ctx context.Context, guest *models.S
 }
 
 func (self *GuestDetachDiskTask) OnDiskDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	self.SetStage("OnSyncstatusComplete", nil)
+	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+}
+
+func (self *GuestDetachDiskTask) OnSyncstatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
 	self.SetStageComplete(ctx, nil)
 	logclient.AddActionLogWithStartable(self, obj, logclient.ACT_VM_DETACH_DISK, nil, self.UserCred, true)
 }
+
+func (self *GuestDetachDiskTask) OnSyncstatusCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	self.OnSyncstatusComplete(ctx, obj, data)
+}

pkg/image/models/quotas.go

@@ -108,8 +108,8 @@ func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
 
 func (self *SQuota) ToJSON(prefix string) jsonutils.JSONObject {
 	ret := jsonutils.NewDict()
-	if self.Image > 0 {
-		ret.Add(jsonutils.NewInt(int64(self.Image)), quotas.KeyName(prefix, "image"))
-	}
+	// if self.Image > 0 {
+	ret.Add(jsonutils.NewInt(int64(self.Image)), quotas.KeyName(prefix, "image"))
+	// }
 	return ret
 }

pkg/mcclient/options/metadata.go

@@ -20,6 +20,10 @@ type MetadataListOptions struct {
 	CloudMeta *bool    `help:"Show cloud metadata olny"`
 	UserMeta  *bool    `help:"Show user metadata olny"`
 	Admin     *bool    `help:"Show all metadata"`
+
+	WithSysMeta   *bool `help:"Show sys metadata"`
+	WithCloudMeta *bool `help:"Show cloud metadata"`
+	WithUserMeta  *bool `help:"Show user metadata"`
 }
 
 type TagListOptions MetadataListOptions

pkg/mcclient/options/servers.go

@@ -48,6 +48,7 @@ type ServerListOptions struct {
 	OrderByHost        string `help:"Order by host name" choices:"asc|desc"`
 	Vpc                string `help:"Vpc id or name"`
 	UsableServerForEip string `help:"Eip id or name"`
+	WithoutUserMeta    *bool  `help:"Show Servers without user metadata"`
 
 	ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"`
 

pkg/util/openstack/shell/instance.go

@@ -78,4 +78,12 @@ func init() {
 		return cli.ChangeConfig(instance, args.FLAVOR_ID)
 	})
 
+	type InstanceDiskOptions struct {
+		ID   string `help:"Instance ID"`
+		DISK string `help:"DiskId"`
+	}
+
+	shellutils.R(&InstanceDiskOptions{}, "instance-detach-disk", "Detach instance disk", func(cli *openstack.SRegion, args *InstanceDiskOptions) error {
+		return cli.DetachDisk(args.ID, args.DISK)
+	})
 }

vendor/modules.txt

@@ -757,7 +757,7 @@ yunion.io/x/pkg/util/prometheus
 yunion.io/x/pkg/prettytable
 yunion.io/x/pkg/util/runtime
 yunion.io/x/pkg/util/clock
-# yunion.io/x/sqlchemy v0.0.0-20190701085445-12b2b4318f7d
+# yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda
 yunion.io/x/sqlchemy
 # yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe
 yunion.io/x/structarg

vendor/yunion.io/x/sqlchemy/functions.go

@@ -85,6 +85,11 @@ func GROUP_CONCAT(name string, field IQueryField) IQueryField {
 	return &ff
 }
 
+func REPLACE(name string, field IQueryField, old string, new string) IQueryField {
+	ff := NewFunctionField(name, fmt.Sprintf(`REPLACE(%s, "%s", "%s")`, "%s", old, new), field)
+	return &ff
+}
+
 type SStringField struct {
 	strConst string
 	alias    string