fix(region): avoid panic

This commit is contained in:
ioito
2022-04-22 11:01:07 +08:00
parent 6cd3549b0c
commit af5455813d
14 changed files with 116 additions and 89 deletions
+4 -1
View File
@@ -247,7 +247,10 @@ func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, us
snapshot := snapObj.(*models.SSnapshot)
snapshotStorage := models.StorageManager.FetchStorageById(snapshot.StorageId)
if snapshotStorage.StorageType == api.STORAGE_LOCAL {
snapshotHost := snapshotStorage.GetMasterHost()
snapshotHost, err := snapshotStorage.GetMasterHost()
if err != nil {
return errors.Wrapf(err, "GetMasterHost")
}
if options.Options.SnapshotCreateDiskProtocol == "url" {
input.SnapshotUrl = fmt.Sprintf("%s/download/snapshots/%s/%s/%s", snapshotHost.ManagerUri, snapshotStorage.Id, snapshot.DiskId, snapshot.Id)
input.SnapshotOutOfChain = snapshot.OutOfChain
+16 -13
View File
@@ -402,7 +402,7 @@ func (self *SDisk) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok
return input, httperrors.NewNotFoundError("failed to find storage for disk %s", self.Name)
}
host := storage.GetMasterHost()
host, _ := storage.GetMasterHost()
if host == nil {
return input, httperrors.NewNotFoundError("failed to find host for storage %s with disk %s", storage.Name, self.Name)
}
@@ -476,7 +476,7 @@ func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mc
return input, httperrors.NewResourceNotReadyError("cloudprovider %s not available", provider.Name)
}
host := storage.GetMasterHost()
host, _ := storage.GetMasterHost()
if host == nil {
return input, httperrors.NewResourceNotFoundError("storage %s(%s) need online and attach host for create disk", storage.Name, storage.Id)
}
@@ -555,7 +555,7 @@ func (manager *SDiskManager) validateDiskOnStorage(diskConfig *api.DiskConfig, s
if storage.StorageType != diskConfig.Backend {
return httperrors.NewInputParameterError("Storage type[%s] not match backend %s", storage.StorageType, diskConfig.Backend)
}
if host := storage.GetMasterHost(); host != nil {
if host, _ := storage.GetMasterHost(); host != nil {
//公有云磁盘大小检查。
if err := host.GetHostDriver().ValidateDiskSize(storage, diskConfig.SizeMb>>10); err != nil {
return httperrors.NewInputParameterError("%v", err)
@@ -835,7 +835,7 @@ func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.Token
return nil, httperrors.NewNotFoundError("failed to find storage for disk %s", self.Name)
}
host := storage.GetMasterHost()
host, _ := storage.GetMasterHost()
if host == nil {
return nil, httperrors.NewNotFoundError("failed to find host for storage %s with disk %s", storage.Name, self.Name)
}
@@ -904,7 +904,7 @@ func (disk *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCre
func (disk *SDisk) getHypervisor() string {
storage, _ := disk.GetStorage()
if storage != nil {
host := storage.GetMasterHost()
host, _ := storage.GetMasterHost()
if host != nil {
return host.GetHostDriver().GetHypervisor()
}
@@ -950,7 +950,7 @@ func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredenti
if storage == nil {
return httperrors.NewInternalServerError("disk has no valid storage")
}
if host := storage.GetMasterHost(); host != nil {
if host, _ := storage.GetMasterHost(); host != nil {
if err := host.GetHostDriver().ValidateDiskSize(storage, sizeMb>>10); err != nil {
return httperrors.NewInputParameterError("%v", err)
}
@@ -1158,7 +1158,7 @@ func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) er
// storage is empty, a dirty data, allow delete
return nil
}
host := storage.GetMasterHost()
host, _ := storage.GetMasterHost()
if host == nil {
return httperrors.NewBadRequestError("storage of disk %s no valid host", self.Id)
}
@@ -1302,13 +1302,16 @@ func (self *SDisk) GetMasterHost() (*SHost, error) {
return &host, nil
}
func (self *SDisk) GetFetchUrl() string {
storage, _ := self.GetStorage()
if storage == nil {
return ""
func (self *SDisk) GetFetchUrl() (string, error) {
storage, err := self.GetStorage()
if err != nil {
return "", errors.Wrapf(err, "self.GetStorage")
}
host := storage.GetMasterHost()
return fmt.Sprintf("%s/disks/%s", host.GetFetchUrl(true), self.Id)
host, err := storage.GetMasterHost()
if err != nil {
return "", errors.Wrapf(err, "storage.GetMasterHost")
}
return fmt.Sprintf("%s/disks/%s", host.GetFetchUrl(true), self.Id), nil
}
func (self *SDisk) GetFsFormat() string {
-13
View File
@@ -1473,19 +1473,6 @@ func (self *SHost) getAttachedWires() []SWire {
return ret
}
func (self *SHost) GetMasterHostwire() *SHostwire {
hw := SHostwire{}
hw.SetModelManager(HostwireManager, &hw)
q := self.GetWiresQuery().IsTrue("is_master")
err := q.First(&hw)
if err != nil {
log.Errorf("GetMasterHostwire %s", err)
return nil
}
return &hw
}
func (self *SHostManager) GetEnabledKvmHost() (*SHost, error) {
hostq := HostManager.Query().IsTrue("enabled").Equals("host_status", api.HOST_ONLINE).In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_KVM})
host := SHost{}
+2 -3
View File
@@ -456,11 +456,10 @@ func (self *SSnapshot) GetDisk() (*SDisk, error) {
return disk, nil
}
func (self *SSnapshot) GetHost() *SHost {
func (self *SSnapshot) GetHost() (*SHost, error) {
iStorage, err := StorageManager.FetchById(self.StorageId)
if err != nil {
log.Errorln(err)
return nil
return nil, errors.Wrapf(err, "StorageManager.FetchById(%s)", self.StorageId)
}
storage := iStorage.(*SStorage)
return storage.GetMasterHost()
+14 -13
View File
@@ -488,7 +488,7 @@ func (self *SStorage) GetOvercommitBound() float32 {
}
}
func (self *SStorage) GetMasterHost() *SHost {
func (self *SStorage) GetMasterHost() (*SHost, error) {
hosts := HostManager.Query().SubQuery()
hoststorages := HoststorageManager.Query().SubQuery()
@@ -500,19 +500,16 @@ func (self *SStorage) GetMasterHost() *SHost {
host.SetModelManager(HostManager, &host)
err := q.First(&host)
if err != nil {
if errors.Cause(err) != sql.ErrNoRows {
log.Errorf("GetMasterHost fail %s", err)
}
return nil
return nil, errors.Wrapf(err, "q.First")
}
return &host
return &host, nil
}
func (self *SStorage) GetZoneId() string {
if len(self.ZoneId) > 0 {
return self.ZoneId
}
host := self.GetMasterHost()
host, _ := self.GetMasterHost()
if host != nil {
_, err := db.Update(self, func() error {
self.ZoneId = host.ZoneId
@@ -1616,20 +1613,24 @@ func (self *SStorage) GetSchedtagJointManager() ISchedtagJointManager {
}
func (manager *SStorageManager) StorageSnapshotsRecycle(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
storages := make([]SStorage, 0)
err := manager.Query().Equals("enabled", true).
storages := []SStorage{}
q := manager.Query().Equals("enabled", true).
In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).
In("storage_type", api.SHARED_FILE_STORAGE).All(&storages)
In("storage_type", api.SHARED_FILE_STORAGE)
err := db.FetchModelObjects(manager, q, &storages)
if err != nil {
log.Errorf("Get shared file storage failed %s", err)
return
}
for i := 0; i < len(storages); i++ {
storages[i].SetModelManager(manager, &storages[i])
host := storages[i].GetMasterHost()
host, err := storages[i].GetMasterHost()
if err != nil {
log.Errorf("get master host for storage %s(%s) failed: %v", storages[i].Name, storages[i].Id, err)
continue
}
url := fmt.Sprintf("%s/storages/%s/snapshots-recycle", host.ManagerUri, storages[i].Id)
headers := mcclient.GetTokenHeaders(userCred)
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, headers, nil, false)
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, headers, nil, false)
if err != nil {
log.Errorf("Storage request snapshots recycle failed %s", err)
}
+21 -12
View File
@@ -1118,9 +1118,9 @@ func (self *SKVMRegionDriver) RequestResetToInstanceSnapshot(ctx context.Context
}
func (self *SKVMRegionDriver) ValidateCreateSnapshotData(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, storage *models.SStorage, input *api.SnapshotCreateInput) error {
host := storage.GetMasterHost()
if host == nil {
return fmt.Errorf("failed to get master host, maybe the host is offline")
_, err := storage.GetMasterHost()
if err != nil {
return errors.Wrapf(err, "storage.GetMasterHost")
}
return models.GetStorageDriver(storage.StorageType).ValidateCreateSnapshotData(ctx, userCred, disk, input)
}
@@ -1291,8 +1291,14 @@ func (self *SKVMRegionDriver) OnSnapshotDelete(ctx context.Context, snapshot *mo
func (self *SKVMRegionDriver) RequestSyncDiskStatus(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, task taskman.ITask) error {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
storage, _ := disk.GetStorage()
host := storage.GetMasterHost()
storage, err := disk.GetStorage()
if err != nil {
return nil, errors.Wrapf(err, "disk.GetStorage")
}
host, err := storage.GetMasterHost()
if err != nil {
return nil, errors.Wrapf(err, "storage.GetMasterHost")
}
header := task.GetTaskRequestHeader()
url := fmt.Sprintf("%s/disks/%s/%s/status", host.ManagerUri, storage.Id, disk.Id)
_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false)
@@ -1362,11 +1368,11 @@ func (self *SKVMRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib
if err != nil {
return errors.Wrap(err, "unable to get backups")
}
storage, _ := backups[0].GetStorage()
var host *models.SHost
if storage != nil {
host = storage.GetMasterHost()
storage, err := backups[0].GetStorage()
if err != nil {
return errors.Wrapf(err, "GetStorage")
}
host, _ := storage.GetMasterHost()
if host == nil {
host, err = models.HostManager.GetEnabledKvmHost()
if err != nil {
@@ -1493,7 +1499,7 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u
storage, _ := backup.GetStorage()
var host *models.SHost
if storage != nil {
host = storage.GetMasterHost()
host, _ = storage.GetMasterHost()
}
if host == nil {
host, err = models.HostManager.GetEnabledKvmHost()
@@ -1527,7 +1533,10 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u
func (self *SKVMRegionDriver) RequestSyncSnapshotStatus(ctx context.Context, userCred mcclient.TokenCredential, snapshot *models.SSnapshot, task taskman.ITask) error {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
storage := snapshot.GetStorage()
host := storage.GetMasterHost()
host, err := storage.GetMasterHost()
if err != nil {
return nil, errors.Wrapf(err, "storage.GetMasterHost")
}
header := task.GetTaskRequestHeader()
url := fmt.Sprintf("%s/snapshots/%s/%s/%s/status", host.ManagerUri, storage.Id, snapshot.DiskId, snapshot.Id)
_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false)
@@ -1717,7 +1726,7 @@ func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *m
storage, _ := backup.GetStorage()
var host *models.SHost
if storage != nil {
host = storage.GetMasterHost()
host, _ = storage.GetMasterHost()
}
if host == nil {
host, err = models.HostManager.GetEnabledKvmHost()
+8 -5
View File
@@ -184,7 +184,10 @@ func (self *SRbdStorageDriver) RequestCreateSnapshot(ctx context.Context, snapsh
return errors.Wrap(err, "snapshot get disk")
}
storage := snapshot.GetStorage()
host := storage.GetMasterHost()
host, err := storage.GetMasterHost()
if err != nil {
return errors.Wrapf(err, "storage.GetMasterHost")
}
url := fmt.Sprintf("%s/disks/%s/snapshot/%s", host.ManagerUri, storage.Id, disk.Id)
header := task.GetTaskRequestHeader()
params := jsonutils.NewDict()
@@ -198,15 +201,15 @@ func (self *SRbdStorageDriver) RequestCreateSnapshot(ctx context.Context, snapsh
func (self *SRbdStorageDriver) RequestDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot, task taskman.ITask) error {
storage := snapshot.GetStorage()
host := storage.GetMasterHost()
if host == nil {
return errors.Errorf("storage %s can't get master host", storage.Id)
host, err := storage.GetMasterHost()
if err != nil {
return errors.Wrapf(err, "storage.GetMasterHost")
}
url := fmt.Sprintf("%s/disks/%s/delete-snapshot/%s", host.ManagerUri, storage.Id, snapshot.DiskId)
header := task.GetTaskRequestHeader()
params := jsonutils.NewDict()
params.Set("snapshot_id", jsonutils.NewString(snapshot.Id))
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, params, false)
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, params, false)
if err != nil {
return errors.Wrap(err, "request delete snapshot")
}
+5 -1
View File
@@ -65,7 +65,11 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis
if rebuild {
db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(ctx), self.GetUserCred())
}
storage, _ := disk.GetStorage()
storage, err := disk.GetStorage()
if err != nil {
self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(errors.Wrapf(err, "disk.GetStorage").Error()))
return
}
host, err := disk.GetMasterHost()
if err != nil {
self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(errors.Wrapf(err, "GetMasterHost").Error()))
+27 -15
View File
@@ -16,6 +16,7 @@ package tasks
import (
"context"
"database/sql"
"fmt"
"yunion.io/x/jsonutils"
@@ -104,18 +105,22 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
self.OnGuestDiskDeleteComplete(ctx, disk, nil)
return
}
var (
storage *models.SStorage
host *models.SHost
)
storage, _ = disk.GetStorage()
if storage == nil { // dirty data
self.OnGuestDiskDeleteComplete(ctx, disk, nil)
storage, err := disk.GetStorage()
if err != nil {
if errors.Cause(err) == sql.ErrNoRows { // dirty data
self.OnGuestDiskDeleteComplete(ctx, disk, nil)
return
}
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("disk.GetStorage"))
return
}
host = storage.GetMasterHost()
host, err := storage.GetMasterHost()
if err != nil {
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("storage.GetMasterHost"))
return
}
isPurge := false
if (host == nil || !host.GetEnabled()) && jsonutils.QueryBoolean(self.Params, "purge", false) {
@@ -139,7 +144,7 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("fail to find master host"))
return
}
err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self)
err = host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self)
if err != nil {
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
return
@@ -149,10 +154,13 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
func (self *DiskDeleteTask) OnMasterStorageDeleteDiskComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
self.SetStage("OnGuestDiskDeleteComplete", nil)
storage := models.StorageManager.FetchStorageById(disk.BackupStorageId)
host := storage.GetMasterHost()
if host == nil {
host, err := storage.GetMasterHost()
if err != nil {
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(fmt.Sprintf("backup storage %s fail to find master host", disk.BackupStorageId)))
} else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self); err != nil {
return
}
err = host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self)
if err != nil {
self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
}
}
@@ -230,10 +238,14 @@ func (self *StorageDeleteRbdDiskTask) DeleteDisk(ctx context.Context, storage *m
header := self.GetTaskRequestHeader()
url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disksId[0])
body := jsonutils.NewDict()
host := storage.GetMasterHost()
_, err := host.Request(ctx, self.GetUserCred(), "POST", url, header, body)
host, err := storage.GetMasterHost()
if err != nil {
log.Errorln(err)
self.OnDeleteDiskFailed(ctx, storage, jsonutils.NewString("storage.GetMasterHost"))
return
}
_, err = host.Request(ctx, self.GetUserCred(), "POST", url, header, body)
if err != nil {
params.Set("err", jsonutils.NewString(err.Error()))
self.OnDeleteDiskFailed(ctx, storage, params)
}
}
+6 -6
View File
@@ -102,14 +102,14 @@ func (self *DiskResetTask) OnStartGuest(ctx context.Context, disk *models.SDisk,
func (self *DiskResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
disk := obj.(*models.SDisk)
storage, _ := disk.GetStorage()
if storage == nil {
self.TaskFailed(ctx, disk, fmt.Errorf("Disk storage not found"))
storage, err := disk.GetStorage()
if err != nil {
self.TaskFailed(ctx, disk, errors.Wrapf(err, "disk.GetStorage"))
return
}
host := storage.GetMasterHost()
if host == nil {
self.TaskFailed(ctx, disk, fmt.Errorf("Storage master host not found"))
host, err := storage.GetMasterHost()
if err != nil {
self.TaskFailed(ctx, disk, errors.Wrapf(err, "storage.GetMasterHost"))
return
}
self.RequestResetDisk(ctx, disk, host)
+1 -1
View File
@@ -54,7 +54,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel,
if guest != nil {
host, _ = guest.GetHost()
} else {
host = storage.GetMasterHost()
host, _ = storage.GetMasterHost()
}
reason := "Cannot find host for disk"
+2 -1
View File
@@ -41,7 +41,8 @@ func (self *DiskSaveTask) GetMasterHost(disk *models.SDisk) *models.SHost {
if guests := disk.GetGuests(); len(guests) == 1 {
if host, _ := guests[0].GetHost(); host == nil {
if storage, _ := disk.GetStorage(); storage != nil {
return storage.GetMasterHost()
host, _ = storage.GetMasterHost()
return host
}
} else {
return host
+6 -1
View File
@@ -19,6 +19,7 @@ import (
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -62,7 +63,11 @@ func (self *HADiskCreateTask) OnDiskReady(
rebuild, _ := self.GetParams().Bool("rebuild")
snapshot, _ := self.GetParams().GetString("snapshot")
storage := models.StorageManager.FetchStorageById(disk.BackupStorageId)
host := storage.GetMasterHost()
host, err := storage.GetMasterHost()
if err != nil {
self.OnBackupAllocateFailed(ctx, disk, jsonutils.NewString(errors.Wrapf(err, "storage.GetMasterHost").Error()))
return
}
db.OpsLog.LogEvent(disk, db.ACT_BACKUP_ALLOCATING, disk.GetShortDesc(ctx), self.GetUserCred())
disk.SetStatus(self.UserCred, api.DISK_BACKUP_STARTALLOC,
fmt.Sprintf("Backup disk start alloc use host %s(%s)", host.Name, host.Id),
+4 -4
View File
@@ -182,13 +182,13 @@ func (self *BatchSnapshotsDeleteTask) OnInit(ctx context.Context, obj db.IStanda
}
func (self *BatchSnapshotsDeleteTask) StartStorageDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot) {
host := snapshot.GetHost()
if host == nil {
self.SetStageFailed(ctx, jsonutils.NewString("Cannot found snapshot host"))
host, err := snapshot.GetHost()
if err != nil {
self.SetStageFailed(ctx, jsonutils.NewString(errors.Wrapf(err, "snapshot.GetHost").Error()))
return
}
self.SetStage("OnStorageDeleteSnapshot", nil)
err := host.GetHostDriver().RequestDeleteSnapshotsWithStorage(ctx, host, snapshot, self)
err = host.GetHostDriver().RequestDeleteSnapshotsWithStorage(ctx, host, snapshot, self)
if err != nil {
self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}