From bd2a4e28329de00303cfaff162cc2b7048892374 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 19 Jun 2020 01:17:34 +0800 Subject: [PATCH] fix: wrong cloud provider status when sync disconnected cloud account --- pkg/compute/models/cloudaccounts.go | 32 +++++++++++++++----- pkg/compute/options/options.go | 2 +- pkg/compute/tasks/cloud_account_sync_task.go | 8 ++++- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index bd3e0844ce..bc1f0275e3 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -729,9 +729,9 @@ func (self *SCloudaccount) MarkEndSyncWithLock(ctx context.Context, userCred mcc lockman.LockObject(ctx, self) defer lockman.ReleaseObject(ctx, self) - if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE { - return nil - } + // if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_IDLE { + // return nil + // } providers := self.GetCloudproviders() for i := range providers { @@ -1743,6 +1743,18 @@ func (manager *SCloudaccountManager) initAllRecords() { } } +func (self *SCloudaccount) CanSync() bool { + if self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_QUEUED || self.SyncStatus == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING || self.getSyncStatus2() == api.CLOUD_PROVIDER_SYNC_STATUS_SYNCING { + if self.LastSync.IsZero() || time.Now().Sub(self.LastSync) > 1800*time.Second { + return true + } else { + return false + } + } else { + return true + } +} + func (manager *SCloudaccountManager) AutoSyncCloudaccountTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { if isStart && !options.Options.IsSlaveNode { // mark all the records to be idle @@ -1778,7 +1790,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m manager, err := account.getProviderInternal() if err != nil { log.Errorf("account.GetProvider failed: %s", err) - return nil, err + return nil, errors.Wrap(err, "account.getProviderInternal") } balance, status, err := manager.GetBalance() if err != nil { @@ -1796,7 +1808,7 @@ func (account *SCloudaccount) probeAccountStatus(ctx context.Context, userCred m sysInfo, err := manager.GetSysInfo() if err != nil { log.Errorf("manager.GetSysInfo fail %s", err) - return nil, err + return nil, errors.Wrap(err, "manager.GetSysInfo") } factory := manager.GetFactory() diff, err := db.Update(account, func() error { @@ -1849,7 +1861,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc if err != nil { account.markAllProvidersDicconnected(ctx, userCred) account.markAccountDiscconected(ctx, userCred) - return err + return errors.Wrap(err, "account.probeAccountStatus") } account.markAccountConnected(ctx, userCred) providers := account.importAllSubaccounts(ctx, userCred, subaccounts) @@ -1858,7 +1870,7 @@ func (account *SCloudaccount) syncAccountStatus(ctx context.Context, userCred mc _, err := providers[i].prepareCloudproviderRegions(ctx, userCred) if err != nil { log.Errorf("syncCloudproviderRegion fail %s", err) - return err + return errors.Wrap(err, "providers[i].prepareCloudproviderRegions") } } } @@ -1886,6 +1898,10 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre cloudaccountPendingSyncsMutex.Lock() defer cloudaccountPendingSyncsMutex.Unlock() if _, ok := cloudaccountPendingSyncs[account.Id]; ok { + if waitChan != nil { + // an active cloudaccount sync task is running, return with conflict error + waitChan <- errors.Wrap(httperrors.ErrConflict, "cloudaccountPendingSyncs") + } return } cloudaccountPendingSyncs[account.Id] = struct{}{} @@ -1903,7 +1919,7 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre if err != nil { account.markEndSync(userCred) } - waitChan <- err + waitChan <- errors.Wrap(err, "account.syncAccountStatus") } else { syncCnt := 0 if err == nil && autoSync && account.GetEnabled() && account.EnableAutoSync { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 56187c6da8..c49c731b5c 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -125,7 +125,7 @@ type ComputeOptions struct { SyncPurgeRemovedResources []string `help:"resources that shoud be purged immediately if found removed" default:"server"` - DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"24"` + DisconnectedCloudAccountRetryProbeIntervalHours int `help:"interval to wait to probe status of a disconnected cloud account" default:"2"` BaremetalServerReuseHostIp bool `help:"baremetal server reuse host IP address, default true" default:"true"` diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index 4f48439adc..4309269f8c 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -18,10 +18,13 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -43,7 +46,10 @@ func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStanda err := cloudaccount.SyncCallSyncAccountTask(ctx, self.UserCred) if err != nil { - cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred) + if errors.Cause(err) != httperrors.ErrConflict { + log.Debugf("no other sync task, mark end sync for all cloudproviders") + cloudaccount.MarkEndSyncWithLock(ctx, self.UserCred) + } db.OpsLog.LogEvent(cloudaccount, db.ACT_SYNC_HOST_FAILED, err, self.UserCred) self.SetStageFailed(ctx, err.Error()) logclient.AddActionLogWithStartable(self, cloudaccount, logclient.ACT_CLOUD_SYNC, err, self.UserCred, false)