diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 86973715cf..d7d7df51b6 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -635,6 +635,7 @@ func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils. } } extra.Add(projects, "projects") + extra.Set("sync_interval_seconds", jsonutils.NewInt(int64(self.getSyncIntervalSeconds()))) return extra } @@ -973,6 +974,26 @@ func (account *SCloudaccount) markAccountConnected(ctx context.Context, userCred return account.SetStatus(userCred, CLOUD_PROVIDER_CONNECTED, "") } +func (account *SCloudaccount) shouldProbeStatus() bool { + // connected state + if account.Status != CLOUD_PROVIDER_DISCONNECTED { + return true + } + // disconencted, but errorCount < threshold + if account.ErrorCount < options.Options.MaxCloudAccountErrorCount { + return true + } + // never synced + if account.LastSyncEndAt.IsZero() { + return true + } + // last sync is long time ago + if time.Now().Sub(account.LastSyncEndAt) > time.Duration(options.Options.DisconnectedCloudAccountRetryProbeIntervalHours)*time.Hour { + return true + } + return false +} + func (account *SCloudaccount) needSync() bool { if account.LastSyncEndAt.IsZero() { return true @@ -992,13 +1013,15 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex } q := manager.Query() - q = q.Filter( - sqlchemy.OR( - sqlchemy.NotEquals(q.Field("status"), CLOUD_PROVIDER_DISCONNECTED), - sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount), - ), - ) - q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE) + /* + q = q.Filter( + sqlchemy.OR( + sqlchemy.NotEquals(q.Field("status"), CLOUD_PROVIDER_DISCONNECTED), + sqlchemy.LT(q.Field("error_count"), options.Options.MaxCloudAccountErrorCount), + ), + ) + */ + // q = q.Equals("sync_status", CLOUD_PROVIDER_SYNC_STATUS_IDLE) accounts := make([]SCloudaccount, 0) err := db.FetchModelObjects(manager, q, &accounts) @@ -1008,7 +1031,7 @@ func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Contex } for i := range accounts { - if accounts[i].needSync() { + if accounts[i].shouldProbeStatus() && accounts[i].needSync() && accounts[i].CanSync() { accounts[i].SubmitSyncAccountTask(ctx, userCred, nil, true) } } diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 21ed369e73..6fbf097443 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -81,9 +81,11 @@ type ComputeOptions struct { CloudSyncWorkerCount int `help:"how many current synchronization threads" default:"2"` CloudAutoSyncIntervalSeconds int `help:"frequency to check auto sync tasks" default:"30"` DefaultSyncIntervalSeconds int `help:"minimal synchronization interval, default 1 minutes" default:"900"` - MinimalSyncIntervalSeconds int `help:"minimal synchronization interval, default 1 minutes" default:"60"` + MinimalSyncIntervalSeconds int `help:"minimal synchronization interval, default 1 minutes" default:"300"` MaxCloudAccountErrorCount int `help:"maximal consecutive error count allow for a cloud account" default:"5"` + DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"24"` + cloudcommon.CommonOptions cloudcommon.DBOptions }