diff --git a/pkg/apis/compute/backup.go b/pkg/apis/compute/backup.go index ea37392986..ade6533b7c 100644 --- a/pkg/apis/compute/backup.go +++ b/pkg/apis/compute/backup.go @@ -83,6 +83,11 @@ type BackupStorageDetails struct { type BackupStorageListInput struct { apis.EnabledStatusInfrasResourceBaseListInput + + // filter by server_id + ServerId string `json:"server_id"` + // filter by disk_id + DiskId string `json:"disk_id"` } type DiskBackupListInput struct { diff --git a/pkg/apis/compute/backup_storages.go b/pkg/apis/compute/backup_storages.go new file mode 100644 index 0000000000..4820c9ca7b --- /dev/null +++ b/pkg/apis/compute/backup_storages.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +type BackupstorageResourceInput struct { + // 备份存储（ID或Name） + BackupstorageId string `json:"backupstorage_id"` + // swagger:ignore + // Deprecated + // filter by backupstorage_id + Backupstorage string `json:"backupstorage" yunion-deprecated-by:"backupstorage_id"` +} + +type BackupstorageFilterListInputBase struct { + BackupstorageResourceInput + + // 以备份存储名称排序 + // pattern:asc|desc + OrderByBackupstorage string `json:"order_by_backupstorage"` +} + +type BackupstorageResourceInfo struct { + // 备份存储名称 + Backupstorage string `json:"backupstorage"` + + // 备份存储类型 + BackupstorageType TBackupStorageType `json:"backupstorage_type"` + + // 备份存储状态 + BackupstorageStatus string `json:"backupstorage_status"` +} + +type BackupstorageFilterListInput struct { + BackupstorageFilterListInputBase +} diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index daa592f361..d0500f745d 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -71,6 +71,7 @@ type HostListInput struct { StorageFilterListInput UsableResourceListInput + BackupstorageFilterListInput // filter by ResourceType ResourceType string `json:"resource_type"` diff --git a/pkg/apis/compute/hostbackupstorage.go b/pkg/apis/compute/hostbackupstorage.go new file mode 100644 index 0000000000..36b6eed9a2 --- /dev/null +++ b/pkg/apis/compute/hostbackupstorage.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type HostBackupstorageDetails struct { + HostJointResourceDetails + + SHostBackupstorage + + // 存储名称 + Backupstorage string `json:"backupstorage"` + // 存储大小 + CapacityMb int64 `json:"capacity_mb"` + // 存储类型 + // example: local + StorageType TBackupStorageType `json:"storage_type"` + // 是否启用 + Enabled bool `json:"enabled"` +} + +type HostBackupstorageListInput struct { + HostJointsListInput + + BackupstorageFilterListInput +} + +type HostBackupstorageCreateInput struct { + apis.JoinResourceBaseCreateInput + BackupstorageId string `json:"backupstorage_id"` + HostId string `json:"host_id"` +} diff --git a/pkg/apis/compute/hostjoin.go b/pkg/apis/compute/hostjoin.go index 6b31e5ad59..0ac517d723 100644 --- a/pkg/apis/compute/hostjoin.go +++ b/pkg/apis/compute/hostjoin.go @@ -27,7 +27,8 @@ type HostJointResourceDetailsBase struct { type HostJointResourceDetails struct { apis.JointResourceBaseDetails - HostJointResourceDetailsBase + // HostJointResourceDetailsBase + HostResourceInfo } type HostJointsListInput struct { diff --git a/pkg/apis/compute/zz_generated.model.go b/pkg/apis/compute/zz_generated.model.go index 2cf20df9d3..fea1a681c0 100644 --- a/pkg/apis/compute/zz_generated.model.go +++ b/pkg/apis/compute/zz_generated.model.go @@ -1296,6 +1296,15 @@ type SHost struct { UefiInfo jsonutils.JSONObject `json:"uefi_info"` } +// SHostBackupstorage is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SHostBackupstorage. +type SHostBackupstorage struct { + SHostJointsBase + // 宿主机Id + HostId string `json:"host_id"` + // 存储Id + BackupstorageId string `json:"backupstorage_id"` +} + // SHostIsolatedDeviceModel is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SHostIsolatedDeviceModel. type SHostIsolatedDeviceModel struct { SHostJointsBase diff --git a/pkg/compute/models/backup_storage.go b/pkg/compute/models/backup_storage.go index e670a57739..f8778415eb 100644 --- a/pkg/compute/models/backup_storage.go +++ b/pkg/compute/models/backup_storage.go @@ -140,12 +140,16 @@ func (bs *SBackupStorage) PostCreate(ctx context.Context, userCred mcclient.Toke log.Errorf("convert object secret fail %s", err) } } - err := StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "") + err := bs.startSyncStatusTask(ctx, userCred, "") if err != nil { log.Errorf("unable to sync backup storage status") } } +func (bs *SBackupStorage) startSyncStatusTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + return StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", parentTaskId) +} + func (bs *SBackupStorage) saveObjectSecret(secret string) error { sec, err := utils.EncryptAESBase64(bs.Id, secret) if err != nil { @@ -189,11 +193,57 @@ func (bm *SBackupStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy var err error q, err = bm.SEnabledStatusInfrasResourceBaseManager.ListItemFilter(ctx, q, userCred, input.EnabledStatusInfrasResourceBaseListInput) if err != nil { - return nil, err + return nil, errors.Wrap(err, "SEnabledStatusInfrasResourceBaseManager.ListItemFilter") + } + if len(input.ServerId) > 0 { + serverObj, err := GuestManager.FetchByIdOrName(ctx, userCred, input.ServerId) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return nil, httperrors.NewResourceNotFoundError2(GuestManager.Keyword(), input.ServerId) + } else { + return nil, errors.Wrap(err, "GuestManager.FetchByIdOrName") + } + } + server := serverObj.(*SGuest) + input.ServerId = server.Id + hostIds, err := server.getDisksCandidateHostIds() + if err != nil { + return nil, errors.Wrap(err, "getDisksCandidateHostIds") + } + q = bm.filterByCandidateHostIds(q, hostIds) + } + if len(input.DiskId) > 0 { + diskObj, err := DiskManager.FetchByIdOrName(ctx, userCred, input.DiskId) + if err != nil { + if errors.Cause(err) == errors.ErrNotFound { + return nil, httperrors.NewResourceNotFoundError2(DiskManager.Keyword(), input.DiskId) + } else { + return nil, errors.Wrap(err, "DiskManager.FetchByIdOrName") + } + } + disk := diskObj.(*SDisk) + input.DiskId = disk.Id + hostIds, err := disk.getCandidateHostIds() + if err != nil { + return nil, errors.Wrap(err, "getDisksCandidateHostIds") + } + q = bm.filterByCandidateHostIds(q, hostIds) } return q, nil } +func (bm *SBackupStorageManager) filterByCandidateHostIds(q *sqlchemy.SQuery, candidateIds []string) *sqlchemy.SQuery { + hbsSubQ := HostBackupstorageManager.Query().SubQuery() + + q = q.LeftJoin(hbsSubQ, sqlchemy.Equals(q.Field("id"), hbsSubQ.Field("backupstorage_id"))) + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(hbsSubQ.Field("host_id")), + sqlchemy.In(hbsSubQ.Field("host_id"), candidateIds), + )) + + return q +} + func (bs *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) { var openTask = true count, err := taskman.TaskManager.QueryTasksOfObject(bs, time.Now().Add(-3*time.Minute), &openTask).CountWithError() @@ -204,7 +254,7 @@ func (bs *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mcclie return nil, httperrors.NewBadRequestError("Backup has %d task active, can't sync status", count) } - return nil, StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "") + return nil, bs.startSyncStatusTask(ctx, userCred, "") } func (bs *SBackupStorage) ValidateUpdateData( @@ -295,3 +345,23 @@ func (bs *SBackupStorage) GetAccessInfo() (*api.SBackupStorageAccessInfo, error) } return &accessInfo, nil } + +func (bs *SBackupStorage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + log.Infof("Host delete do nothing") + // cleanup hostbackupstorage + hbs, err := HostBackupstorageManager.GetBackupStoragesByBackup(bs.Id) + if err != nil { + return errors.Wrap(err, "GetBackupStoragesByBackup") + } + var errs []error + for i := range hbs { + err := hbs[i].Detach(ctx, userCred) + if err != nil { + errs = append(errs, errors.Wrapf(err, "Detach %s %s", hbs[i].HostId, hbs[i].BackupstorageId)) + } + } + if len(errs) > 0 { + return errors.NewAggregate(errs) + } + return bs.SEnabledStatusInfrasResourceBase.Delete(ctx, userCred) +} diff --git a/pkg/compute/models/backupstorageresource.go b/pkg/compute/models/backupstorageresource.go new file mode 100644 index 0000000000..18dda48845 --- /dev/null +++ b/pkg/compute/models/backupstorageresource.go @@ -0,0 +1,204 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "database/sql" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +type SBackupstorageResourceBase struct { + BackupstorageId string `width:"36" charset:"ascii" nullable:"true" list:"user" index:"true" create:"optional"` +} + +type SBackupstorageResourceBaseManager struct { +} + +func ValidateBackupstorageResourceInput(ctx context.Context, userCred mcclient.TokenCredential, query api.BackupstorageResourceInput) (*SBackupStorage, api.BackupstorageResourceInput, error) { + storageObj, err := BackupStorageManager.FetchByIdOrName(ctx, userCred, query.BackupstorageId) + if err != nil { + if err == sql.ErrNoRows { + return nil, query, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", BackupStorageManager.Keyword(), query.BackupstorageId) + } else { + return nil, query, errors.Wrap(err, "BackupStorageManager.FetchByIdOrName") + } + } + query.BackupstorageId = storageObj.GetId() + return storageObj.(*SBackupStorage), query, nil +} + +func (self *SBackupstorageResourceBase) GetBackupstorage() *SBackupStorage { + obj, err := BackupStorageManager.FetchById(self.BackupstorageId) + if err != nil { + log.Errorf("fail to fetch storage by id: %s: %s", self.BackupstorageId, err) + return nil + } + return obj.(*SBackupStorage) +} + +func (manager *SBackupstorageResourceBaseManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.BackupstorageResourceInfo { + rows := make([]api.BackupstorageResourceInfo, len(objs)) + storageIds := make([]string, len(objs)) + for i := range objs { + var base *SBackupstorageResourceBase + reflectutils.FindAnonymouStructPointer(objs[i], &base) + if base != nil { + storageIds[i] = base.BackupstorageId + } + } + + storages := make(map[string]SBackupStorage) + err := db.FetchStandaloneObjectsByIds(BackupStorageManager, storageIds, &storages) + if err != nil { + log.Errorf("FetchStandaloneObjectsByIds fail %s", err) + return nil + } + + for i := range rows { + rows[i] = api.BackupstorageResourceInfo{} + if _, ok := storages[storageIds[i]]; ok { + storage := storages[storageIds[i]] + rows[i].Backupstorage = storage.Name + rows[i].BackupstorageStatus = storage.Status + rows[i].BackupstorageType = storage.StorageType + } + } + + return rows +} + +func (manager *SBackupstorageResourceBaseManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.BackupstorageFilterListInput, +) (*sqlchemy.SQuery, error) { + if len(query.BackupstorageId) > 0 { + storageObj, _, err := ValidateBackupstorageResourceInput(ctx, userCred, query.BackupstorageResourceInput) + if err != nil { + return nil, errors.Wrap(err, "ValidateBackupstorageResourceInput") + } + q = q.Equals("backupstorage_id", storageObj.GetId()) + } + return q, nil +} + +func (manager *SBackupstorageResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + switch field { + case "backupstorage": + storages := BackupStorageManager.Query("name", "id").Distinct().SubQuery() + q.AppendField(storages.Field("name", field)) + q = q.Join(storages, sqlchemy.Equals(q.Field("backupstorage_id"), storages.Field("id"))) + q.GroupBy(storages.Field("name")) + return q, nil + case "storage_type": + storages := BackupStorageManager.Query(field, "id").Distinct().SubQuery() + q.AppendField(storages.Field(field)) + q = q.Join(storages, sqlchemy.Equals(q.Field("backupstorage_id"), storages.Field("id"))) + q.GroupBy(storages.Field(field)) + return q, nil + } + return q, httperrors.ErrNotFound +} + +func (manager *SBackupstorageResourceBaseManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.BackupstorageFilterListInput, +) (*sqlchemy.SQuery, error) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, nil + } + orderQ := BackupStorageManager.Query("id") + orderSubQ := orderQ.SubQuery() + orderQ, orders, fields := manager.GetOrderBySubQuery(orderQ, orderSubQ, orderQ.Field("id"), userCred, query, nil, nil) + q = q.LeftJoin(orderSubQ, sqlchemy.Equals(q.Field("backupstorage_id"), orderSubQ.Field("id"))) + q = db.OrderByFields(q, orders, fields) + return q, nil +} + +func (manager *SBackupstorageResourceBaseManager) GetOrderBySubQuery( + q *sqlchemy.SQuery, + subq *sqlchemy.SSubQuery, + joinField sqlchemy.IQueryField, + userCred mcclient.TokenCredential, + query api.BackupstorageFilterListInput, + orders []string, + fields []sqlchemy.IQueryField, +) (*sqlchemy.SQuery, []string, []sqlchemy.IQueryField) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, orders, fields + } + backupStorageQ := BackupStorageManager.Query().SubQuery() + q = q.LeftJoin(backupStorageQ, sqlchemy.Equals(joinField, backupStorageQ.Field("id"))) + q = q.AppendField(backupStorageQ.Field("name").Label("backupstorage")) + orders = append(orders, query.OrderByBackupstorage) + fields = append(fields, subq.Field("backupstorage")) + return q, orders, fields +} + +func (manager *SBackupstorageResourceBaseManager) GetOrderByFields(query api.BackupstorageFilterListInput) []string { + fields := make([]string, 0) + fields = append(fields, query.OrderByBackupstorage) + return fields +} + +func (manager *SBackupstorageResourceBaseManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + if keys.ContainsAny(manager.GetExportKeys()...) { + subq := BackupStorageManager.Query("id", "name").SubQuery() + q = q.LeftJoin(subq, sqlchemy.Equals(q.Field("backupstorage_id"), subq.Field("id"))) + if keys.Contains("backupstorage") { + q = q.AppendField(subq.Field("name", "backupstorage")) + } + } + return q, nil +} + +func (manager *SBackupstorageResourceBaseManager) GetExportKeys() []string { + keys := []string{"backupstorage"} + return keys +} + +func (model *SBackupstorageResourceBase) GetChangeOwnerCandidateDomainIds() []string { + storage := model.GetBackupstorage() + if storage != nil { + return storage.GetChangeOwnerCandidateDomainIds() + } + return nil +} diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 608c937fe4..8cd4efde0a 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -1396,6 +1396,18 @@ func (self *SDisk) GetPathAtHost(host *SHost) string { return "" } +func (disk *SDisk) getCandidateHostIds() ([]string, error) { + hss, err := HoststorageManager.GetHostStoragesByStorageId(disk.StorageId) + if err != nil { + return nil, errors.Wrap(err, "GetHostStoragesByStorageId") + } + candidates := make([]string, 0) + for i := range hss { + candidates = append(candidates, hss[i].HostId) + } + return candidates, nil +} + func (self *SDisk) GetMasterHost(storage *SStorage) (*SHost, error) { if storage.MasterHost != "" { return storage.GetMasterHost() diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index cba76ca62d..2883cf3af1 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -6815,3 +6815,24 @@ func (guest *SGuest) IsSriov() bool { } return false } + +func (guest *SGuest) getDisksCandidateHostIds() ([]string, error) { + disks, err := guest.GetDisks() + if err != nil { + return nil, errors.Wrap(err, "guest.GetDisks") + } + ret := stringutils2.NewSortedStrings(nil) + for i := range disks { + candidates, err := disks[i].getCandidateHostIds() + if err != nil { + return nil, errors.Wrap(err, "getCandidateHostIds") + } + sorted := stringutils2.NewSortedStrings(candidates) + if i > 0 { + ret = stringutils2.Intersect(ret, sorted) + } else { + ret = sorted + } + } + return ret, nil +} diff --git a/pkg/compute/models/hostbackupstorages.go b/pkg/compute/models/hostbackupstorages.go new file mode 100644 index 0000000000..e59ee11cec --- /dev/null +++ b/pkg/compute/models/hostbackupstorages.go @@ -0,0 +1,276 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +type SHostBackupstorageManager struct { + SHostJointsManager + SBackupstorageResourceBaseManager +} + +var HostBackupstorageManager *SHostBackupstorageManager + +func init() { + db.InitManager(func() { + HostBackupstorageManager = &SHostBackupstorageManager{ + SHostJointsManager: NewHostJointsManager( + "host_id", + SHostBackupstorage{}, + "hostbackupstorages_tbl", + "hostbackupstorage", + "hostbackupstorages", + BackupStorageManager, + ), + } + HostBackupstorageManager.SetVirtualObject(HostBackupstorageManager) + HostBackupstorageManager.TableSpec().AddIndex(false, "host_id", "backupstorage_id") + }) +} + +type SHostBackupstorage struct { + SHostJointsBase + + // 宿主机Id + HostId string `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"required" json:"host_id"` + // 存储Id + BackupstorageId string `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"required" json:"backupstorage_id" index:"true"` +} + +func (manager *SHostBackupstorageManager) GetMasterFieldName() string { + return "host_id" +} + +func (manager *SHostBackupstorageManager) GetSlaveFieldName() string { + return "backupstorage_id" +} + +func (manager *SHostBackupstorageManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.HostBackupstorageDetails { + rows := make([]api.HostBackupstorageDetails, len(objs)) + + hostRows := manager.SHostJointsManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + backupStorageIds := make([]string, len(rows)) + + for i := range rows { + rows[i] = api.HostBackupstorageDetails{ + HostJointResourceDetails: hostRows[i], + } + backupStorageIds[i] = objs[i].(*SHostBackupstorage).BackupstorageId + } + + backupStorages := make(map[string]SBackupStorage) + err := db.FetchStandaloneObjectsByIds(BackupStorageManager, backupStorageIds, &backupStorages) + if err != nil { + log.Errorf("db.FetchStandaloneObjectsByIds fail %s", err) + return rows + } + + for i := range rows { + if bs, ok := backupStorages[backupStorageIds[i]]; ok { + rows[i] = objs[i].(*SHostBackupstorage).getExtraDetails(bs, rows[i]) + } + } + + return rows +} + +func (hb *SHostBackupstorage) GetHost() *SHost { + host, _ := HostManager.FetchById(hb.HostId) + if host != nil { + return host.(*SHost) + } + return nil +} + +func (hb *SHostBackupstorage) GetBackupStorage() *SBackupStorage { + bs, err := BackupStorageManager.FetchById(hb.BackupstorageId) + if err != nil { + log.Errorf("Hoststorage fetch storage %q error: %v", hb.BackupstorageId, err) + } + if bs != nil { + return bs.(*SBackupStorage) + } + return nil +} + +func (manager *SHostBackupstorageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.HostBackupstorageCreateInput) (api.HostBackupstorageCreateInput, error) { + { + backStorageObj, err := validators.ValidateModel(ctx, userCred, BackupStorageManager, &input.BackupstorageId) + if err != nil { + return input, err + } + backupStorage := backStorageObj.(*SBackupStorage) + input.BackupstorageId = backupStorage.Id + } + { + hostObj, err := validators.ValidateModel(ctx, userCred, HostManager, &input.HostId) + if err != nil { + return input, err + } + host := hostObj.(*SHost) + input.HostId = host.Id + } + { + hs, err := manager.GetBackupStoragesByHost(input.HostId, api.BACKUPSTORAGE_TYPE_NFS) + if err != nil { + return input, errors.Wrap(err, "GetBackupStoragesByHost") + } + if len(hs) >= 1 { + return input, errors.Wrapf(httperrors.ErrResourceBusy, "host %s has been attached to a NFS backupstorage", input.HostId) + } + } + var err error + input.JoinResourceBaseCreateInput, err = manager.SJointResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.JoinResourceBaseCreateInput) + if err != nil { + return input, err + } + return input, nil +} + +func (hb *SHostBackupstorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + hb.SHostJointsBase.PostCreate(ctx, userCred, ownerId, query, data) + + bs := hb.GetBackupStorage() + if bs != nil { + bs.startSyncStatusTask(ctx, userCred, "") + } +} + +func (hb *SHostBackupstorage) PostDelete(ctx context.Context, userCred mcclient.TokenCredential) { + hb.SHostJointsBase.PostDelete(ctx, userCred) + + bs := hb.GetBackupStorage() + if bs != nil { + bs.startSyncStatusTask(ctx, userCred, "") + } +} + +func (hb *SHostBackupstorage) getExtraDetails(storage SBackupStorage, out api.HostBackupstorageDetails) api.HostBackupstorageDetails { + out.Backupstorage = storage.Name + out.CapacityMb = int64(storage.CapacityMb) + out.StorageType = storage.StorageType + out.Enabled = storage.Enabled.Bool() + return out +} + +func (hb *SHostBackupstorage) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + return hb.SHostJointsBase.ValidateDeleteCondition(ctx, nil) +} + +func (hb *SHostBackupstorage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, hb) +} + +func (hb *SHostBackupstorage) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DetachJoint(ctx, userCred, hb) +} + +func (manager *SHostBackupstorageManager) GetBackupStoragesByBackup(backupStorageId string) ([]SHostBackupstorage, error) { + hoststorages := make([]SHostBackupstorage, 0) + q := HostBackupstorageManager.Query().Equals("backupstorage_id", backupStorageId) + err := db.FetchModelObjects(manager, q, &hoststorages) + if err != nil { + return nil, err + } + return hoststorages, nil +} + +func (manager *SHostBackupstorageManager) GetBackupStoragesByHost(hostId string, backupType api.TBackupStorageType) ([]SHostBackupstorage, error) { + hoststorages := make([]SHostBackupstorage, 0) + backups := BackupStorageManager.Query().Equals("storage_type", backupType).SubQuery() + q := HostBackupstorageManager.Query().Equals("host_id", hostId) + q = q.Join(backups, sqlchemy.Equals(q.Field("backupstorage_id"), backups.Field("id"))) + err := db.FetchModelObjects(manager, q, &hoststorages) + if err != nil { + return nil, err + } + return hoststorages, nil +} + +func (manager *SHostBackupstorageManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.HostBackupstorageListInput, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SHostJointsManager.ListItemFilter(ctx, q, userCred, query.HostJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SHostResourceBaseManager.ListItemFilter") + } + q, err = manager.SBackupstorageResourceBaseManager.ListItemFilter(ctx, q, userCred, query.BackupstorageFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "SBackupstorageResourceBaseManager.ListItemFilter") + } + + return q, nil +} + +func (manager *SHostBackupstorageManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.HostBackupstorageListInput, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SHostJointsManager.OrderByExtraFields(ctx, q, userCred, query.HostJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SHostResourceBaseManager.OrderByExtraFields") + } + q, err = manager.SBackupstorageResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.BackupstorageFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "SBackupstorageResourceBaseManager.OrderByExtraFields") + } + + return q, nil +} + +func (manager *SHostBackupstorageManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SHostJointsManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SHostJointsManager.ListItemExportKeys") + } + + return q, nil +} diff --git a/pkg/compute/models/hostjoints.go b/pkg/compute/models/hostjoints.go index 12d0ed3d9b..81c69d33ed 100644 --- a/pkg/compute/models/hostjoints.go +++ b/pkg/compute/models/hostjoints.go @@ -63,10 +63,12 @@ func (manager *SHostJointsManager) FetchCustomizeColumns( rows := make([]api.HostJointResourceDetails, len(objs)) jointRows := manager.SJointResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + hostRows := manager.SHostResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) for i := range rows { rows[i] = api.HostJointResourceDetails{ JointResourceBaseDetails: jointRows[i], + HostResourceInfo: hostRows[i], } } diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a75b4449c5..f44d1238de 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -75,6 +75,8 @@ type SHostManager struct { SZoneResourceBaseManager SManagedResourceBaseManager SHostnameResourceBaseManager + + SBackupstorageResourceBaseManager } var HostManager *SHostManager @@ -334,6 +336,22 @@ func (manager *SHostManager) ListItemFilter( } } + if len(query.BackupstorageId) > 0 { + hbsQ := HostBackupstorageManager.Query("host_id", "backupstorage_id") + hbsQ, err = manager.SBackupstorageResourceBaseManager.ListItemFilter(ctx, hbsQ, userCred, query.BackupstorageFilterListInput) + if err != nil { + return q, errors.Wrap(err, "SBackupStorageResouceBaseManager.ListItemFiled") + } + hbsSubQ := hbsQ.SubQuery() + q = q.LeftJoin(hbsSubQ, sqlchemy.Equals(q.Field("id"), hbsSubQ.Field("host_id"))) + notAttached := (query.StorageNotAttached != nil && *query.StorageNotAttached) + if !notAttached { + q = q.Filter(sqlchemy.IsNotNull(hbsSubQ.Field("backupstorage_id"))) + } else { + q = q.Filter(sqlchemy.IsNull(hbsSubQ.Field("backupstorage_id"))) + } + } + hostStorageType := query.HostStorageType if len(hostStorageType) > 0 { hoststorages := HoststorageManager.Query() @@ -1530,8 +1548,69 @@ func (hh *SHost) getAttachedWires() []SWire { return ret } -func (hh *SHostManager) GetEnabledKvmHost() (*SHost, error) { - hostq := HostManager.Query().IsTrue("enabled").Equals("host_status", api.HOST_ONLINE).In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_KVM}) +func (hh *SHostManager) GetEnabledKvmHostForBackupStorage(bs *SBackupStorage) (*SHost, error) { + hbs, err := HostBackupstorageManager.GetBackupStoragesByBackup(bs.Id) + if err != nil { + return nil, errors.Wrap(err, "GetBackupStoragesByBackup") + } + candidates := make([]string, 0) + for i := range hbs { + candidates = append(candidates, hbs[i].HostId) + } + host, err := HostManager.GetEnabledKvmHost(candidates) + if err != nil { + return nil, errors.Wrap(err, "GetEnabledKvmHost") + } + return host, nil +} + +func (hh *SHostManager) GetEnabledKvmHostForDiskBackup(backup *SDiskBackup) (*SHost, error) { + bs, err := backup.GetBackupStorage() + if err != nil { + return nil, errors.Wrap(err, "unable to get backupStorage") + } + storage, err := backup.GetStorage() + if err != nil { + return nil, errors.Wrap(err, "unable to get storage of diskbackup") + } + + hbs, err := HostBackupstorageManager.GetBackupStoragesByBackup(bs.Id) + if err != nil { + return nil, errors.Wrap(err, "HostBackupstorageManager.GetBackupStoragesByBackup") + } + hbsCandidates := stringutils2.NewSortedStrings(nil) + for i := range hbs { + hbsCandidates = hbsCandidates.Append(hbs[i].HostId) + } + hss, err := HoststorageManager.GetHostStoragesByStorageId(storage.Id) + if err != nil { + return nil, errors.Wrap(err, "HoststorageManager.GetStorages") + } + hssCandidates := stringutils2.NewSortedStrings(nil) + for i := range hss { + hssCandidates = hssCandidates.Append(hss[i].HostId) + } + var candidates []string + if len(hbsCandidates) == 0 { + candidates = []string(hssCandidates) + } else { + candidates = []string(stringutils2.Intersect(hbsCandidates, hssCandidates)) + } + + host, err := HostManager.GetEnabledKvmHost(candidates) + if err != nil { + return nil, errors.Wrap(err, "GetEnabledKvmHost") + } + return host, nil +} + +func (hh *SHostManager) GetEnabledKvmHost(candidates []string) (*SHost, error) { + hostq := HostManager.Query().IsTrue("enabled") + hostq = hostq.Equals("host_status", api.HOST_ONLINE) + hostq = hostq.In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_KVM}) + if len(candidates) > 0 { + hostq = hostq.In("id", candidates) + } host := SHost{} err := hostq.First(&host) if err != nil { diff --git a/pkg/compute/models/hoststorages.go b/pkg/compute/models/hoststorages.go index 2d743997e0..280358a9d5 100644 --- a/pkg/compute/models/hoststorages.go +++ b/pkg/compute/models/hoststorages.go @@ -312,12 +312,22 @@ func (self *SHoststorage) Detach(ctx context.Context, userCred mcclient.TokenCre return db.DetachJoint(ctx, userCred, self) } -func (manager *SHoststorageManager) GetStorages(hostId string) ([]SHoststorage, error) { +func (manager *SHoststorageManager) GetHostStoragesByHostId(hostId string) ([]SHoststorage, error) { hoststorage := make([]SHoststorage, 0) - hoststorages := HoststorageManager.Query().SubQuery() - err := hoststorages.Query().Equals("host_id", hostId).All(&hoststorage) + hoststoragesQ := HoststorageManager.Query().Equals("host_id", hostId) + err := db.FetchModelObjects(manager, hoststoragesQ, &hoststorage) if err != nil { - return nil, err + return nil, errors.Wrap(err, "FetchModelObjects") + } + return hoststorage, nil +} + +func (manager *SHoststorageManager) GetHostStoragesByStorageId(storageId string) ([]SHoststorage, error) { + hoststorage := make([]SHoststorage, 0) + hoststoragesQ := HoststorageManager.Query().Equals("storage_id", storageId) + err := db.FetchModelObjects(manager, hoststoragesQ, &hoststorage) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") } return hoststorage, nil } diff --git a/pkg/compute/models/purge.go b/pkg/compute/models/purge.go index 72f678014d..024792b3aa 100644 --- a/pkg/compute/models/purge.go +++ b/pkg/compute/models/purge.go @@ -674,6 +674,7 @@ func (self *SZone) purgeHosts(ctx context.Context, managerId string) error { hosts := HostManager.Query("id").Equals("manager_id", managerId).Equals("zone_id", self.Id) isolateds := IsolatedDeviceManager.Query("id").In("host_id", hosts.SubQuery()) hoststorages := HoststorageManager.Query("row_id").In("host_id", hosts.SubQuery()) + hostbackupStorages := HostBackupstorageManager.Query("row_id").In("host_id", hosts.SubQuery()) hostwires := HostwireManagerDeprecated.Query("row_id").In("host_id", hosts.SubQuery()) guests := GuestManager.Query("id").In("host_id", hosts.SubQuery()) guestdisks := GuestdiskManager.Query("row_id").In("guest_id", guests.SubQuery()) @@ -722,6 +723,7 @@ func (self *SZone) purgeHosts(ctx context.Context, managerId string) error { {manager: InstanceBackupManager, key: "id", q: instancebackups}, {manager: GuestManager, key: "id", q: guests}, {manager: HoststorageManager, key: "row_id", q: hoststorages}, + {manager: HostBackupstorageManager, key: "row_id", q: hostbackupStorages}, {manager: HostwireManagerDeprecated, key: "row_id", q: hostwires}, {manager: IsolatedDeviceManager, key: "id", q: isolateds}, {manager: HostManager, key: "id", q: hosts}, @@ -738,6 +740,7 @@ func (self *SZone) purgeHosts(ctx context.Context, managerId string) error { func (self *SHost) purge(ctx context.Context, userCred mcclient.TokenCredential) error { isolateds := IsolatedDeviceManager.Query("id").Equals("host_id", self.Id) hoststorages := HoststorageManager.Query("row_id").Equals("host_id", self.Id) + hostbackupStorages := HostBackupstorageManager.Query("row_id").Equals("host_id", self.Id) hostwires := HostwireManagerDeprecated.Query("row_id").Equals("host_id", self.Id) guests := GuestManager.Query("id").Equals("host_id", self.Id) guestdisks := GuestdiskManager.Query("row_id").In("guest_id", guests.SubQuery()) @@ -778,6 +781,7 @@ func (self *SHost) purge(ctx context.Context, userCred mcclient.TokenCredential) {manager: InstanceBackupManager, key: "id", q: instancebackups}, {manager: GuestManager, key: "id", q: guests}, {manager: HoststorageManager, key: "row_id", q: hoststorages}, + {manager: HostBackupstorageManager, key: "row_id", q: hostbackupStorages}, {manager: HostwireManagerDeprecated, key: "row_id", q: hostwires}, {manager: IsolatedDeviceManager, key: "id", q: isolateds}, } diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index 197f38cece..f95c6ac29c 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -940,17 +940,11 @@ func (self *SKVMRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib if err != nil { return errors.Wrap(err, "unable to get backups") } - storage, err := backups[0].GetStorage() + host, err := models.HostManager.GetEnabledKvmHostForDiskBackup(&backups[0]) if err != nil { - return errors.Wrapf(err, "GetStorage") - } - host, _ := storage.GetMasterHost() - if host == nil { - host, err = models.HostManager.GetEnabledKvmHost() - if err != nil { - return errors.Wrap(err, "unable to GetEnabledKvmHost") - } + return errors.Wrap(err, "GetEnabledKvmHostForDiskBackup") } + backupIds := make([]string, len(backups)) for i := range backupIds { backupIds[i] = backups[i].GetId() @@ -984,7 +978,7 @@ func (self *SKVMRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, i if err != nil { return errors.Wrap(err, "unable to get backupStorage") } - host, err := models.HostManager.GetEnabledKvmHost() + host, err := models.HostManager.GetEnabledKvmHostForBackupStorage(backupStorage) if err != nil { return errors.Wrap(err, "unable to GetEnabledKvmHost") } @@ -1011,9 +1005,9 @@ func (self *SKVMRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, i func (self *SKVMRegionDriver) RequestSyncBackupStorageStatus(ctx context.Context, userCred mcclient.TokenCredential, bs *models.SBackupStorage, task taskman.ITask) error { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - host, err := models.HostManager.GetEnabledKvmHost() + host, err := models.HostManager.GetEnabledKvmHostForBackupStorage(bs) if err != nil { - return nil, errors.Wrap(err, "unable to GetEnabledKvmHost") + return nil, errors.Wrap(err, "GetEnabledKvmHostForBackupStorage") } url := fmt.Sprintf("%s/storages/sync-backup-storage", host.ManagerUri) body := jsonutils.NewDict() @@ -1083,7 +1077,7 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u if err != nil { return nil, errors.Wrap(err, "unable to get backupStorage") } - storage, _ := backup.GetStorage() + /*storage, _ := backup.GetStorage() var host *models.SHost if storage != nil { host, _ = storage.GetMasterHost() @@ -1093,6 +1087,10 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u if err != nil { return nil, errors.Wrap(err, "unable to GetEnabledKvmHost") } + }*/ + host, err := models.HostManager.GetEnabledKvmHostForDiskBackup(backup) + if err != nil { + return nil, errors.Wrap(err, "GetEnabledKvmHostForDiskBackup") } log.Infof("host: %s, ManagerUri: %s", host.GetId(), host.ManagerUri) url := fmt.Sprintf("%s/storages/sync-backup", host.ManagerUri) @@ -1300,17 +1298,11 @@ func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *m if err != nil { return errors.Wrap(err, "unable to get backupStorage") } - storage, _ := backup.GetStorage() - var host *models.SHost - if storage != nil { - host, _ = storage.GetMasterHost() - } - if host == nil { - host, err = models.HostManager.GetEnabledKvmHost() - if err != nil { - return errors.Wrap(err, "unable to GetEnabledKvmHost") - } + host, err := models.HostManager.GetEnabledKvmHostForDiskBackup(backup) + if err != nil { + return errors.Wrap(err, "GetEnabledKvmHostForDiskBackup") } + url := fmt.Sprintf("%s/storages/delete-backup", host.ManagerUri) body := jsonutils.NewDict() body.Set("backup_id", jsonutils.NewString(backup.GetId())) diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index 9b9d060eca..860fb66cb1 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -252,6 +252,7 @@ func InitHandlers(app *appsrv.Application) { models.HostwireManagerDeprecated, models.HostnetworkManager, models.HoststorageManager, + models.HostBackupstorageManager, models.HostschedtagManager, models.HostIsolatedDeviceModelManager, models.StorageschedtagManager, diff --git a/pkg/mcclient/modules/compute/mod_hostbackupstorages.go b/pkg/mcclient/modules/compute/mod_hostbackupstorages.go new file mode 100644 index 0000000000..7b5de7e5cd --- /dev/null +++ b/pkg/mcclient/modules/compute/mod_hostbackupstorages.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + HostBackupstorages modulebase.JointResourceManager +) + +func init() { + HostBackupstorages = modules.NewJointComputeManager("hostbackupstorage", "hostbackupstorages", + []string{"Host_ID", "Host", "Backupstorage_ID", + "Backupstorage", "Capacity_Mb", "Host_Status", "Backupstorage_Status"}, + []string{}, + &Hosts, + &BackupStorages) + modules.RegisterCompute(&HostBackupstorages) +}