From 13d8ba02a5ebdff0546fb0f757d092d22d44cada Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Fri, 8 Mar 2019 23:22:46 +0800
Subject: [PATCH 1/2] fix: cloudaccount that is in stale syncing status should
 be auto-synced

---
 pkg/compute/models/cloudaccounts.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index 86973715cf..a0ea959b8b 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -998,7 +998,7 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex
 			sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount),
 		),
 	)
-	q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE)
+	// q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE)
 
 	accounts := make([]SCloudaccount, 0)
 	err := db.FetchModelObjects(manager, q, &accounts)
@@ -1008,7 +1008,7 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex
 	}
 
 	for i := range accounts {
-		if accounts[i].needSync() {
+		if accounts[i].needSync() && accounts[i].CanSync() {
 			accounts[i].SubmitSyncAccountTask(ctx, userCred, nil, true)
 		}
 	}

From b485f31e81c5fdffd6006ec85891e84ef936186a Mon Sep 17 00:00:00 2001
From: Qiu Jian <qiujian@yunionyun.com>
Date: Sat, 9 Mar 2019 20:14:16 +0800
Subject: [PATCH 2/2] fix: try to sync disconnected account for every 24 hours

---
 pkg/compute/models/cloudaccounts.go | 37 +++++++++++++++++++++++------
 pkg/compute/options/options.go      |  4 +++-
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go
index a0ea959b8b..d7d7df51b6 100644
--- a/pkg/compute/models/cloudaccounts.go
+++ b/pkg/compute/models/cloudaccounts.go
@@ -635,6 +635,7 @@ func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.
 		}
 	}
 	extra.Add(projects, "projects")
+	extra.Set("sync_interval_seconds", jsonutils.NewInt(int64(self.getSyncIntervalSeconds())))
 	return extra
 }
 
@@ -973,6 +974,26 @@ func (account *SCloudaccount) markAccountConnected(ctx context.Context, userCred
 	return account.SetStatus(userCred, CLOUD_PROVIDER_CONNECTED, "")
 }
 
+func (account *SCloudaccount) shouldProbeStatus() bool {
+	// connected state
+	if account.Status != CLOUD_PROVIDER_DISCONNECTED {
+		return true
+	}
+	// disconencted, but errorCount < threshold
+	if account.ErrorCount < options.Options.MaxCloudAccountErrorCount {
+		return true
+	}
+	// never synced
+	if account.LastSyncEndAt.IsZero() {
+		return true
+	}
+	// last sync is long time ago
+	if time.Now().Sub(account.LastSyncEndAt) > time.Duration(options.Options.DisconnectedCloudAccountRetryProbeIntervalHours)*time.Hour {
+		return true
+	}
+	return false
+}
+
 func (account *SCloudaccount) needSync() bool {
 	if account.LastSyncEndAt.IsZero() {
 		return true
@@ -992,12 +1013,14 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex
 	}
 
 	q := manager.Query()
-	q = q.Filter(
-		sqlchemy.OR(
-			sqlchemy.NotEquals(q.Field("status"), CLOUD_PROVIDER_DISCONNECTED),
-			sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount),
-		),
-	)
+	/*
+		q = q.Filter(
+			sqlchemy.OR(
+				sqlchemy.NotEquals(q.Field("status"), CLOUD_PROVIDER_DISCONNECTED),
+				sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount),
+			),
+		)
+	*/
 	// q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE)
 
 	accounts := make([]SCloudaccount, 0)
@@ -1008,7 +1031,7 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex
 	}
 
 	for i := range accounts {
-		if accounts[i].needSync() && accounts[i].CanSync() {
+		if accounts[i].shouldProbeStatus() && accounts[i].needSync() && accounts[i].CanSync() {
 			accounts[i].SubmitSyncAccountTask(ctx, userCred, nil, true)
 		}
 	}
diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go
index 21ed369e73..6fbf097443 100644
--- a/pkg/compute/options/options.go
+++ b/pkg/compute/options/options.go
@@ -81,9 +81,11 @@ type ComputeOptions struct {
 	CloudSyncWorkerCount         int `help:"how many current synchronization threads" default:"2"`
 	CloudAutoSyncIntervalSeconds int `help:"frequency to check auto sync tasks" default:"30"`
 	DefaultSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"900"`
-	MinimalSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"60"`
+	MinimalSyncIntervalSeconds   int `help:"minimal synchronization interval, default 1 minutes" default:"300"`
 	MaxCloudAccountErrorCount    int `help:"maximal consecutive error count allow for a cloud account" default:"5"`
 
+	DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"24"`
+
 	cloudcommon.CommonOptions
 	cloudcommon.DBOptions
 }