diff --git a/cmd/climc/shell/compute/backup.go b/cmd/climc/shell/compute/backup.go index 7a5261fd57..29d677c76c 100644 --- a/cmd/climc/shell/compute/backup.go +++ b/cmd/climc/shell/compute/backup.go @@ -26,6 +26,7 @@ func init() { bsCmd.List(&compute.BackupStorageListOptions{}) bsCmd.Show(&compute.BackupStorageIdOptions{}) bsCmd.Create(&compute.BackupStorageCreateOptions{}) + bsCmd.Update(&compute.BackupStorageUpdateOptions{}) bsCmd.Delete(&compute.BackupStorageIdOptions{}) bsCmd.Perform("public", &options.BasePublicOptions{}) bsCmd.Perform("private", &options.BaseIdOptions{}) diff --git a/pkg/apis/compute/backup.go b/pkg/apis/compute/backup.go index f154917f67..cee934e0ef 100644 --- a/pkg/apis/compute/backup.go +++ b/pkg/apis/compute/backup.go @@ -59,26 +59,18 @@ type BackupStorageCreateInput struct { // enum: nfs StorageType string `json:"storage_type"` - // description: host of nfs, storage_type 为 nfs 时, 此参数必传 - // example: 192.168.222.2 - NfsHost string `json:"nfs_host"` - - // description: shared dir of nfs, storage_type 为 nfs 时, 此参数必传 - // example: /nfs_root/ - NfsSharedDir string `json:"nfs_shared_dir"` - - // description: access url of object storage bucket - // example: https://qxxxxxo.tos-cn-beijing.volces.com - ObjectBucketUrl string `json:"object_bucket_url"` - // description: access key of object storage - ObjectAccessKey string `json:"object_access_key"` - // description: secret of object storage - ObjectSecret string `json:"object_secret"` + SBackupStorageAccessInfo // description: Capacity size in MB CapacityMb int `json:"capacity_mb"` } +type BackupStorageUpdateInput struct { + apis.EnabledStatusInfrasResourceBaseUpdateInput + + SBackupStorageAccessInfo +} + /*type BackupStorageAccessInfo struct { AccessUrl string }*/ @@ -175,12 +167,21 @@ type InstanceBackupManagerSyncstatusInput struct { } type SBackupStorageAccessInfo struct { - NfsHost string `json:"nfs_host"` + // description: host of nfs, storage_type 为 nfs 时, 此参数必传 + // example: 192.168.222.2 + NfsHost string `json:"nfs_host"` + + // description: shared dir of nfs, storage_type 为 nfs 时, 此参数必传 + // example: /nfs_root/ NfsSharedDir string `json:"nfs_shared_dir"` + // description: access url of object storage bucket + // example: https://qxxxxxo.tos-cn-beijing.volces.com ObjectBucketUrl string `json:"object_bucket_url"` + // description: access key of object storage ObjectAccessKey string `json:"object_access_key"` - ObjectSecret string `json:"object_secret"` + // description: secret of object storage + ObjectSecret string `json:"object_secret"` } func (ba *SBackupStorageAccessInfo) String() string { diff --git a/pkg/apis/compute/dnstrafficpolicy.go b/pkg/apis/compute/dnstrafficpolicy.go deleted file mode 100644 index 6293929547..0000000000 --- a/pkg/apis/compute/dnstrafficpolicy.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import ( - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/apis" -) - -type DnsTrafficPolicyCreateInput struct { - apis.EnabledStatusInfrasResourceBaseCreateInput - - Provider string `json:"provider"` - PolicyType string `json:"policy_type"` - PolicyValue string `json:"policy_value"` - // 额外参数 - Options *jsonutils.JSONDict `json:"options"` -} - -type DnsTrafficPolicyDetails struct { - apis.EnabledStatusInfrasResourceBaseDetails -} - -type DnsTrafficPolicyListInput struct { - apis.EnabledStatusInfrasResourceBaseListInput - - Provider []string `json:"provider"` - PolicyType string `json:"policy_type"` -} diff --git a/pkg/apis/compute/dnszonecache.go b/pkg/apis/compute/dnszonecache.go deleted file mode 100644 index fef2864042..0000000000 --- a/pkg/apis/compute/dnszonecache.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package compute - -import "yunion.io/x/onecloud/pkg/apis" - -const ( - DNS_ZONE_CACHE_STATUS_DELETING = "deleting" - DNS_ZONE_CACHE_STATUS_DELETE_FAILED = "delete_failed" - DNS_ZONE_CACHE_STATUS_CREATING = "creating" - DNS_ZONE_CACHE_STATUS_CREATE_FAILED = "create_failed" - DNS_ZONE_CACHE_STATUS_AVAILABLE = "available" - DNS_ZONE_CACHE_STATUS_UNKNOWN = "unknown" -) - -type DnsZoneCacheCreateInput struct { -} - -type DnsZoneCacheDetails struct { - apis.StatusStandaloneResourceDetails - - Account string - Brand string - Provider string -} - -type DnsZoneCacheListInput struct { - apis.StatusStandaloneResourceListInput - - DnsZoneFilterListBase - - CloudaccountId string `json:"cloudaccount_id"` -} diff --git a/pkg/compute/models/backup_storage.go b/pkg/compute/models/backup_storage.go index 21a1e3c174..abf51ce259 100644 --- a/pkg/compute/models/backup_storage.go +++ b/pkg/compute/models/backup_storage.go @@ -132,11 +132,31 @@ func (bs *SBackupStorage) ValidateDeleteCondition(ctx context.Context, info json func (bs *SBackupStorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { bs.SEnabledStatusInfrasResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + bs.SetStatus(userCred, api.BACKUPSTORAGE_STATUS_OFFLINE, "") + if bs.StorageType == api.BACKUPSTORAGE_TYPE_OBJECT_STORAGE { + err := bs.saveObjectSecret(bs.AccessInfo.ObjectSecret) + if err != nil { + log.Errorf("convert object secret fail %s", err) + } + } err := StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "") if err != nil { log.Errorf("unable to sync backup storage status") } - bs.SetStatus(userCred, api.BACKUPSTORAGE_STATUS_OFFLINE, "") +} + +func (bs *SBackupStorage) saveObjectSecret(secret string) error { + sec, err := utils.EncryptAESBase64(bs.Id, secret) + if err != nil { + return errors.Wrap(err, "EncryptAESBase64") + } + accessInfo := *bs.AccessInfo + accessInfo.ObjectSecret = sec + _, err = db.Update(bs, func() error { + bs.AccessInfo = &accessInfo + return nil + }) + return errors.Wrap(err, "Update") } func (bs *SBackupStorage) getMoreDetails(ctx context.Context, out api.BackupStorageDetails) api.BackupStorageDetails { @@ -173,9 +193,9 @@ func (bm *SBackupStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy return q, nil } -func (self *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) { +func (bs *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) { var openTask = true - count, err := taskman.TaskManager.QueryTasksOfObject(self, time.Now().Add(-3*time.Minute), &openTask).CountWithError() + count, err := taskman.TaskManager.QueryTasksOfObject(bs, time.Now().Add(-3*time.Minute), &openTask).CountWithError() if err != nil { return nil, err } @@ -183,5 +203,94 @@ func (self *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mccl return nil, httperrors.NewBadRequestError("Backup has %d task active, can't sync status", count) } - return nil, StartResourceSyncStatusTask(ctx, userCred, self, "BackupStorageSyncstatusTask", "") + return nil, StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "") +} + +func (bs *SBackupStorage) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.BackupStorageUpdateInput, +) (api.BackupStorageUpdateInput, error) { + var err error + if len(input.Name) > 0 { + err := isValidBucketName(input.Name) + if err != nil { + return input, httperrors.NewInputParameterError("invalid bucket name(%s): %s", input.Name, err) + } + } + input.EnabledStatusInfrasResourceBaseUpdateInput, err = bs.SEnabledStatusInfrasResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusInfrasResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData") + } + return input, nil +} + +func (bs *SBackupStorage) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + bs.SEnabledStatusInfrasResourceBase.PostUpdate(ctx, userCred, query, data) + input := api.BackupStorageUpdateInput{} + err := data.Unmarshal(&input) + if err != nil { + log.Errorf("SBackupStorage.PostUpdate Unmarshal data %s fail %s", data, err) + return + } + // update accessinfo + accessInfoChanged := false + accessInfo := *bs.AccessInfo + switch bs.StorageType { + case api.BACKUPSTORAGE_TYPE_NFS: + if len(input.NfsHost) > 0 { + accessInfo.NfsHost = input.NfsHost + accessInfoChanged = true + } + if len(input.NfsSharedDir) > 0 { + accessInfo.NfsSharedDir = input.NfsSharedDir + accessInfoChanged = true + } + case api.BACKUPSTORAGE_TYPE_OBJECT_STORAGE: + if len(input.ObjectBucketUrl) > 0 { + accessInfo.ObjectBucketUrl = input.ObjectBucketUrl + accessInfoChanged = true + } + if len(input.ObjectAccessKey) > 0 { + accessInfo.ObjectAccessKey = input.ObjectAccessKey + accessInfoChanged = true + } + if len(input.ObjectSecret) > 0 { + sec, err := utils.EncryptAESBase64(bs.Id, input.ObjectSecret) + if err != nil { + log.Errorf("EncryptAESBase64 fail %s", err) + return + } + accessInfo.ObjectSecret = sec + accessInfoChanged = true + } + } + if accessInfoChanged { + _, err = db.Update(bs, func() error { + bs.AccessInfo = &accessInfo + return nil + }) + if err != nil { + log.Errorf("update fail %s", err) + } else { + err := StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "") + if err != nil { + log.Errorf("unable to sync backup storage status") + } + } + } +} + +func (bs *SBackupStorage) GetAccessInfo() (*api.SBackupStorageAccessInfo, error) { + accessInfo := *bs.AccessInfo + switch bs.StorageType { + case api.BACKUPSTORAGE_TYPE_OBJECT_STORAGE: + secret, err := utils.DescryptAESBase64(bs.Id, accessInfo.ObjectSecret) + if err != nil { + return nil, errors.Wrap(err, "DescryptAESBase64") + } + accessInfo.ObjectSecret = secret + } + return &accessInfo, nil } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 688f17e367..42d6824cdd 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -808,10 +808,14 @@ func (self *SDisk) getDiskAllocateFromBackupInput(ctx context.Context, backupId if err != nil { return nil, errors.Wrapf(err, "unable to get backupstorage of backup %s", backupId) } + accessInfo, err := bs.GetAccessInfo() + if err != nil { + return nil, errors.Wrap(err, "backupStorage.GetAccessInfo") + } return &api.DiskAllocateFromBackupInput{ BackupId: backupId, BackupStorageId: bs.GetId(), - BackupStorageAccessInfo: jsonutils.Marshal(bs.AccessInfo).(*jsonutils.JSONDict), + BackupStorageAccessInfo: jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict), }, nil } diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index 2d6e018115..dc030ad07e 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -1005,7 +1005,11 @@ func (self *SKVMRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib body := jsonutils.NewDict() body.Set("package_name", jsonutils.NewString(packageName)) body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(backupStorage.AccessInfo)) + accessInfo, err := backupStorage.GetAccessInfo() + if err != nil { + return errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) body.Set("backup_ids", jsonutils.Marshal(backupIds)) body.Set("metadata", jsonutils.Marshal(metadata)) header := task.GetTaskRequestHeader() @@ -1031,7 +1035,11 @@ func (self *SKVMRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, i body := jsonutils.NewDict() body.Set("package_name", jsonutils.NewString(packageName)) body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(backupStorage.AccessInfo)) + accessInfo, err := backupStorage.GetAccessInfo() + if err != nil { + return errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) if metadataOnly { body.Set("metadata_only", jsonutils.JSONTrue) } @@ -1052,7 +1060,11 @@ func (self *SKVMRegionDriver) RequestSyncBackupStorageStatus(ctx context.Context url := fmt.Sprintf("%s/storages/sync-backup-storage", host.ManagerUri) body := jsonutils.NewDict() body.Set("backup_storage_id", jsonutils.NewString(bs.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(bs.AccessInfo)) + accessInfo, err := bs.GetAccessInfo() + if err != nil { + return nil, errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) header := task.GetTaskRequestHeader() _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) if err != nil { @@ -1109,7 +1121,7 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u if utils.IsInStringArray(originStatus, []string{api.BACKUP_STATUS_CREATING, api.BACKUP_STATUS_SNAPSHOT, api.BACKUP_STATUS_SAVING, api.BACKUP_STATUS_CLEANUP_SNAPSHOT, api.BACKUP_STATUS_DELETING}) { return nil, backup.SetStatus(userCred, originStatus, "sync status") } - backupStroage, err := backup.GetBackupStorage() + backupStorage, err := backup.GetBackupStorage() if err != nil { return nil, errors.Wrap(err, "unable to get backupStorage") } @@ -1128,8 +1140,12 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u url := fmt.Sprintf("%s/storages/sync-backup", host.ManagerUri) body := jsonutils.NewDict() body.Set("backup_id", jsonutils.NewString(backup.GetId())) - body.Set("backup_storage_id", jsonutils.NewString(backupStroage.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(backupStroage.AccessInfo)) + body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) + accessInfo, err := backupStorage.GetAccessInfo() + if err != nil { + return nil, errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) header := task.GetTaskRequestHeader() _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) if err != nil { @@ -1332,7 +1348,7 @@ func (self *SKVMRegionDriver) GetMaxElasticcacheSecurityGroupCount() int { } func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *models.SDiskBackup, task taskman.ITask) error { - backupStroage, err := backup.GetBackupStorage() + backupStorage, err := backup.GetBackupStorage() if err != nil { return errors.Wrap(err, "unable to get backupStorage") } @@ -1350,8 +1366,12 @@ func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *m url := fmt.Sprintf("%s/storages/delete-backup", host.ManagerUri) body := jsonutils.NewDict() body.Set("backup_id", jsonutils.NewString(backup.GetId())) - body.Set("backup_storage_id", jsonutils.NewString(backupStroage.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(backupStroage.AccessInfo)) + body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) + accessInfo, err := backupStorage.GetAccessInfo() + if err != nil { + return errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) header := task.GetTaskRequestHeader() _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) if err != nil { @@ -1361,7 +1381,7 @@ func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *m } func (self *SKVMRegionDriver) RequestCreateBackup(ctx context.Context, backup *models.SDiskBackup, snapshotId string, task taskman.ITask) error { - backupStroage, err := backup.GetBackupStorage() + backupStorage, err := backup.GetBackupStorage() if err != nil { return errors.Wrap(err, "unable to get backupStorage") } @@ -1382,8 +1402,12 @@ func (self *SKVMRegionDriver) RequestCreateBackup(ctx context.Context, backup *m body := jsonutils.NewDict() body.Set("snapshot_id", jsonutils.NewString(snapshotId)) body.Set("backup_id", jsonutils.NewString(backup.GetId())) - body.Set("backup_storage_id", jsonutils.NewString(backupStroage.GetId())) - body.Set("backup_storage_access_info", jsonutils.Marshal(backupStroage.AccessInfo)) + body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) + accessInfo, err := backupStorage.GetAccessInfo() + if err != nil { + return errors.Wrap(err, "GetAccessInfo") + } + body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) if len(backup.EncryptKeyId) > 0 { body.Set("encrypt_key_id", jsonutils.NewString(backup.EncryptKeyId)) } diff --git a/pkg/compute/tasks/backup_storage_syncstatus_task.go b/pkg/compute/tasks/backup_storage_syncstatus_task.go index c2fe124fc1..341a467a1f 100644 --- a/pkg/compute/tasks/backup_storage_syncstatus_task.go +++ b/pkg/compute/tasks/backup_storage_syncstatus_task.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -35,6 +36,7 @@ func init() { func (self *BackupStorageSyncstatusTask) taskFailed(ctx context.Context, bs *models.SBackupStorage, err jsonutils.JSONObject) { logclient.AddActionLogWithContext(ctx, bs, logclient.ACT_SYNC_STATUS, err, self.UserCred, false) + bs.SetStatus(self.UserCred, api.BACKUPSTORAGE_STATUS_OFFLINE, err.String()) self.SetStageFailed(ctx, err) } diff --git a/pkg/mcclient/options/compute/backup.go b/pkg/mcclient/options/compute/backup.go index 62f43ab03d..b5def5b986 100644 --- a/pkg/mcclient/options/compute/backup.go +++ b/pkg/mcclient/options/compute/backup.go @@ -128,6 +128,21 @@ func (opts *BackupStorageCreateOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(opts), nil } +type BackupStorageUpdateOptions struct { + options.BaseUpdateOptions + + NfsHost string `help:"nfs host, required when storage_type is nfs"` + NfsSharedDir string `help:"nfs shared dir, required when storage_type is nfs" ` + + ObjectBucketUrl string `help:"object bucket url, required when storage_type is object"` + ObjectAccessKey string `help:"object storage access key, required when storage_type is object"` + ObjectSecret string `help:"object storage secret, required when storage_type is object"` +} + +func (opts *BackupStorageUpdateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + type InstanceBackupListOptions struct { options.BaseListOptions