Merge pull request #1222 in YUNIONIO/onecloud from ~QIUJIAN/onecloud:hotfix/qj-msyd-bugfix-20190312 to release/2.6.0

* commit '1ca00c14c6d0e3e9342f8e2a0a5586f6b2fc7a2c':
  fix: find image peer host with local storage only
  fix: filter cached images by storage status
  remove debug logs
  fix: usage port state not correct
  fix: 1. usage hide system servers 2. host undo convert shows wrong status

pkg/cloudcommon/db/taskman/tasks.go

@@ -10,17 +10,17 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/onecloud/pkg/appctx"
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -447,6 +447,14 @@ func (task *STask) ScheduleRun(data jsonutils.JSONObject) {
 	runTask(task.Id, data)
 }
 
+func (self *STask) HasParentTask() bool {
+	parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY)
+	if len(parentTaskId) > 0 {
+		return true
+	}
+	return false
+}
+
 func (self *STask) GetParentTask() *STask {
 	parentTaskId, _ := self.Params.GetString(PARENT_TASK_ID_KEY)
 	if len(parentTaskId) > 0 {

pkg/compute/models/cachedimages.go

@@ -336,14 +336,17 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 
 	scimgs := make([]SStoragecachedimage, 0)
 	q := storageCachedImage.Query().
-		Join(storage, sqlchemy.AND(sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id")))).
-		Join(hostStorage, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id")))).
-		Join(host, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id")))).
+		Join(storage, sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id"))).
+		Join(hostStorage, sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id"))).
+		Join(host, sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id"))).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)).
 		Filter(sqlchemy.Equals(storageCachedImage.Field("status"), CACHED_IMAGE_STATUS_READY)).
 		Filter(sqlchemy.Equals(host.Field("status"), HOST_STATUS_RUNNING)).
 		Filter(sqlchemy.IsTrue(host.Field("enabled"))).
-		Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE))
+		Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)).
+		Filter(sqlchemy.IsTrue(storage.Field("enabled"))).
+		Filter(sqlchemy.In(storage.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})).
+		Filter(sqlchemy.Equals(storage.Field("storage_type"), STORAGE_LOCAL))
 
 	if len(excludes) > 0 {
 		q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))

pkg/compute/models/networks.go

@@ -633,7 +633,7 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt
 	networks := manager.Query().SubQuery()
 	hostwires := HostwireManager.Query().SubQuery()
 	hosts := HostManager.Query().SubQuery()
-	q := networks.Query()
+	q := networks.Query(networks.Field("id"))
 	q = q.Join(hostwires, sqlchemy.Equals(hostwires.Field("wire_id"), networks.Field("wire_id")))
 	q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), hostwires.Field("host_id")))
 	q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
@@ -645,13 +645,12 @@ func (manager *SNetworkManager) allNetworksQ(providers []string, rangeObj db.ISt
 
 func (manager *SNetworkManager) totalPortCountQ(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) *sqlchemy.SQuery {
 	q := manager.allNetworksQ(providers, rangeObj)
-	networks := manager.Query().SubQuery()
 	if userCred != nil && !db.IsAdminAllowList(userCred, manager) {
 		q = q.Filter(sqlchemy.OR(
-			sqlchemy.Equals(networks.Field("tenant_id"), userCred.GetProjectId()),
-			sqlchemy.IsTrue(networks.Field("is_public"))))
+			sqlchemy.Equals(q.Field("tenant_id"), userCred.GetProjectId()),
+			sqlchemy.IsTrue(q.Field("is_public"))))
 	}
-	return q
+	return manager.Query().In("id", q.Distinct().SubQuery())
 }
 
 type NetworkPortStat struct {
@@ -659,7 +658,11 @@ type NetworkPortStat struct {
 	CountExt int
 }
 
-func (manager *SNetworkManager) TotalPortCount(userCred mcclient.TokenCredential, providers []string, rangeObj db.IStandaloneModel) NetworkPortStat {
+func (manager *SNetworkManager) TotalPortCount(
+	userCred mcclient.TokenCredential,
+	providers []string,
+	rangeObj db.IStandaloneModel,
+) NetworkPortStat {
 	nets := make([]SNetwork, 0)
 	err := manager.totalPortCountQ(userCred, providers, rangeObj).All(&nets)
 	if err != nil {

pkg/compute/tasks/guest_sync_task.go

@@ -47,8 +47,12 @@ func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStand
 }
 
 func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
-	self.SetStage("on_sync_status_complete", nil)
-	guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
+	if self.HasParentTask() {
+		self.OnSyncStatusComplete(ctx, guest, nil)
+	} else {
+		self.SetStage("on_sync_status_complete", nil)
+		guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId())
+	}
 }
 
 func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/usages/handler.go

@@ -509,7 +509,9 @@ func guestHypervisorsUsage(
 	if userCred != nil {
 		projectId = userCred.GetProjectId()
 	}
-	guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, true, pendingDelete, hostTypes, resourceTypes, providers)
+	// temporarily hide system resources
+	// XXX needs more work later
+	guest := models.GuestManager.TotalCount(projectId, rangeObj, status, hypervisors, false, pendingDelete, hostTypes, resourceTypes, providers)
 	count := make(map[string]interface{})
 	count[prefix] = guest.TotalGuestCount
 	count[fmt.Sprintf("%s.cpu", prefix)] = guest.TotalCpuCount