diff --git a/cmd/climc/shell/imageguest.go b/cmd/climc/shell/imageguest.go new file mode 100644 index 0000000000..f2fa0d2df5 --- /dev/null +++ b/cmd/climc/shell/imageguest.go @@ -0,0 +1,139 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + + type GuestImageCreateOptions struct { + NAME string `help:"Name of guest image"` + ImageNumber int `help:"common image number of guest image"` + Protected bool `help:"if guest image is protected"` + } + + R(&GuestImageCreateOptions{}, "guest-image-create", "Create guest image's metadata", func(s *mcclient.ClientSession, + args *GuestImageCreateOptions) error { + + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(args.NAME), "name") + if args.ImageNumber > 0 { + params.Add(jsonutils.NewInt(int64(args.ImageNumber)), "image_number") + } + if args.Protected { + params.Add(jsonutils.JSONTrue, "protected") + } + ret, err := modules.GuestImages.Create(s, params) + if err != nil { + return err + } + printObject(ret) + return nil + }, + ) + + type GuestImageListOptions struct { + options.BaseListOptions + + Name string `help:"Name filter"` + } + + R(&GuestImageListOptions{}, "guest-image-list", "List guest images", func(s *mcclient.ClientSession, + args *GuestImageListOptions) error { + + params, err := args.Params() + if err != nil { + return err + } + + if len(args.Name) > 0 { + params.Add(jsonutils.NewString(args.Name), "name") + } + params.Add(jsonutils.JSONTrue, "details") + rets, err := modules.GuestImages.List(s, params) + if err != nil { + return err + } + printList(rets, modules.GuestImages.GetColumns(s)) + return nil + }, + ) + + type GuestImageDeleteOptions struct { + ID []string `help:"Image ID or name"` + OverridePendingDelete *bool `help:"Delete image directly instead of pending delete" short-token:"f"` + } + R(&GuestImageDeleteOptions{}, "guest-image-delete", "Delete a image", func(s *mcclient.ClientSession, + args *GuestImageDeleteOptions) error { + params, err := options.StructToParams(args) + if err != nil { + return err + } + ret := modules.GuestImages.BatchDeleteWithParam(s, args.ID, params, nil) + printBatchResults(ret, modules.GuestImages.GetColumns(s)) + return nil + }) + + type GuestImageCancelDeleteOptions struct { + ID string `help:"Guest Image id or name"` + } + R(&GuestImageCancelDeleteOptions{}, "guest-image-cancel-delete", "Cancel pending delete images", + func(s *mcclient.ClientSession, + args *GuestImageCancelDeleteOptions) error { + if image, e := modules.GuestImages.PerformAction(s, args.ID, "cancel-delete", nil); e != nil { + return e + } else { + printObject(image) + } + return nil + }) + + type GuestImageOptions struct { + ID string `help:"Guest Image id or name"` + } + R(&GuestImageOptions{}, "guest-image-mark-protected", "Mark image protected", func(s *mcclient.ClientSession, + args *GuestImageOptions) error { + + params := jsonutils.NewDict() + params.Add(jsonutils.JSONTrue, "protected") + result, err := modules.GuestImages.Update(s, args.ID, params) + if err != nil { + return err + } + printObject(result) + return nil + }, + ) + R(&GuestImageOptions{}, "guest-image-mark-unprotected", "Mark image protected", func(s *mcclient.ClientSession, + args *GuestImageOptions) error { + + params := jsonutils.NewDict() + params.Add(jsonutils.JSONFalse, "protected") + result, err := modules.GuestImages.Update(s, args.ID, params) + if err != nil { + return err + } + printObject(result) + return nil + }, + ) + +} diff --git a/cmd/climc/shell/images.go b/cmd/climc/shell/images.go index a55d62419b..933988d4ed 100644 --- a/cmd/climc/shell/images.go +++ b/cmd/climc/shell/images.go @@ -492,5 +492,4 @@ func init() { printObject(srv) return nil }) - } diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 88266faa55..f1271c45e0 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -555,6 +555,23 @@ func init() { return nil }) + R(&options.ServerSaveImageOptions{}, "server-save-guest-image", + "save root disk and data disks to new images and upload to glance.", func(s *mcclient.ClientSession, + opts *options.ServerSaveImageOptions) error { + + params, err := options.StructToParams(opts) + if err != nil { + return err + } + srv, err := modules.Servers.PerformAction(s, opts.ID, "save-guest-image", params) + if err != nil { + return err + } + printObject(srv) + return nil + }, + ) + type ServerChangeOwnerOptions struct { ID string `help:"Server to change owner" json:"-"` PROJECT string `help:"Project ID or change" json:"tenant"` diff --git a/docs/image/guest-image.yaml b/docs/image/guest-image.yaml new file mode 100644 index 0000000000..9d0d0ee200 --- /dev/null +++ b/docs/image/guest-image.yaml @@ -0,0 +1,22 @@ +get: + summary: 获取指定主机镜像的详细信息 + parameters: + - $ref: "../parameters/image.yaml#/imageId" + responses: + 200: + description: 主机镜像详细信息 + schema: + $ref: "../schemas/guest_image.yaml#/GuestImageResponse" + tags: + - guestimage + +delete: + summary: 删除指定主机镜像 + parameters: + - $ref: "../parameters/image.yaml#/imageId" + responses: + 200: + schema: + $ref: "../schemas/guest_image.yaml#/GuestImageResponse" + tags: + - guestimage diff --git a/docs/image/guest-images.yaml b/docs/image/guest-images.yaml new file mode 100644 index 0000000000..5470bef79c --- /dev/null +++ b/docs/image/guest-images.yaml @@ -0,0 +1,13 @@ +get: + summary: 按指定条件列出主机镜像 + parameters: + - $ref: "../parameters/common.yaml#/offset" + - $ref: "../parameters/common.yaml#/limit" + + responses: + 200: + description: 主机镜像列表信息 + schema: + $ref: "../schemas/guest_image.yaml#/GuestImageListResponse" + tags: + - guestimage diff --git a/docs/image/image.yaml b/docs/image/image.yaml index f72792cb43..fb6a247f3c 100644 --- a/docs/image/image.yaml +++ b/docs/image/image.yaml @@ -87,6 +87,12 @@ head: type: string #example: 2019-05-30T09:26:41.000000Z description: 镜像最近一次更新时间 + x-image-meta-is_data: + type: boolean + description: 是否是数据盘镜像 + x-image-meta-is_guest_image: + type: boolean + description: 镜像是否是主机镜像的一部分 tags: - images diff --git a/docs/index.yaml b/docs/index.yaml index dd0f1be34b..1847894673 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -212,6 +212,8 @@ paths: $ref: "./instance/revoke-secgroup.yaml" /servers/{instanceId}/save-image: $ref: "./instance/save-image.yaml" + /servers/{instanceId}/save-guest-image: + $ref: "./instance/save-guest-image.yaml" /servers/{instanceId}/send-keys: $ref: "./instance/send-keys.yaml" /servers/{instanceId}/set-extra-option: @@ -248,6 +250,12 @@ paths: /images/{imageId}/mark-standard: $ref: "./image/image-standard.yaml" + /guestimages: + $ref: "./image/guest-images.yaml" + /guestimages/{imageId}: + $ref: "./image/guest-image.yaml" + + /natgateways: $ref: "./natgateway/natgateways.yaml" /natgateways/{natgatewayId}: diff --git a/docs/instance/save-guest-image.yaml b/docs/instance/save-guest-image.yaml new file mode 100644 index 0000000000..bef918f099 --- /dev/null +++ b/docs/instance/save-guest-image.yaml @@ -0,0 +1,16 @@ +post: + summary: 保存主机镜像 + parameters: + - $ref: '../parameters/instance.yaml#/instanceId' + - in: body + name: server + required: true + schema: + $ref: '../schemas/instance.yaml#/InstanceSaveImage' + responses: + 200: + description: 实例信息 + schema: + $ref: '../schemas/instance.yaml#/InstanceResponse' + tags: + - instance diff --git a/docs/schemas/guest_image.yaml b/docs/schemas/guest_image.yaml new file mode 100644 index 0000000000..3d72b61ca8 --- /dev/null +++ b/docs/schemas/guest_image.yaml @@ -0,0 +1,107 @@ +GuestImageListResponse: + type: object + properties: + limit: + type: interger + example: 20 + guestimages: + type: array + items: + $ref: '#/GuestImage' + total: + type: integer + example: 124 + +GuestImageResponse: + type: object + properties: + guestimage: + type: object + $ref: '#/GuestImage' + +GuestImage: + type: object + description: 主机镜像 + properties: + id: + type: string + example: ea27c0ce-5870-49f3-8d57-f53e63f40361 + description: 镜像ID + readOnly: true + can_delete: + type: boolean + example: true + description: 镜像是否可以删除 + can_update: + type: boolean + example: true + description: 镜像是否可以更新 + created_at: + type: string + example: 2019-05-30T09:23:16.000000Z + description: 镜像创建时间 + name: + type: string + example: ubuntu-16.04.5-server-20181117.qcow2 + description: 镜像名称 + properties: + type: object + $ref: './image.yaml#/ImageProperties' + protected: + type: boolean + example: true + description: 是否受保护 + size: + type: integer + example: 792592384 + description: 主机镜像实际大小(系统盘镜像和数据盘镜像的大小之和),单位Byte + min_ram_size: + type: integer + example: 2048 + description: 创建vm时的最低内存限制 + root_image: + type: object + $ref: '#/SubImageInfo' + description: 系统盘镜像 + data_images: + type: array + items: + $ref: '#/SubImageInfo' + description: 数据盘镜像列表 + status: + type: string + example: active + enum: [queued,saving,active,converting] + description: 镜像状态, queued(队列中), saving(保存中), active(可用) + tenant: + type: string + example: system + description: 所属项目名称 + tenant_id: + type: string + example: a7f2e2a81a1e4850a41eae5f140ceb14 + description: 所属项目ID + updated_at: + type: string + example: 2019-05-30T09:26:41.000000Z + description: 最近一次更新时间 + +SubImageInfo: + type: object + description: 子镜像信息 + properties: + id: + type: string + example: fasd2344-5470-32f3-4d47-f53323f40361 + description: 子镜像的ID + name: + type: string + example: root + description: 子镜像的Name + min_disk_mb: + type: integer + example: 40960 + description: 磁盘最低大小限制(MB) + disk_format: + type: string + description: 磁盘格式 diff --git a/docs/schemas/image.yaml b/docs/schemas/image.yaml index dec3b8815f..8e811d0a09 100644 --- a/docs/schemas/image.yaml +++ b/docs/schemas/image.yaml @@ -100,6 +100,14 @@ Image: type: string example: 2019-05-30T09:26:41.000000Z description: 最近一次更新时间 + is_guest_image: + type: boolean + example: true + description: 是否是主机镜像的一部分 + is_data: + type: boolean + example: true + description: 是否是数据盘的镜像 ImageProperties: diff --git a/docs/schemas/instance.yaml b/docs/schemas/instance.yaml index 77c4f48f0e..38c49e81b3 100644 --- a/docs/schemas/instance.yaml +++ b/docs/schemas/instance.yaml @@ -657,6 +657,9 @@ InstanceCreate: type: array items: $ref: '#/InstanceDisk' + guest_image_id: + type: string + example: 如果使用主机镜像创建主机，需要额外指定主机镜像的 ID，来检验 disks 的 Image ID 是否正确 InstanceDisk: @@ -1053,6 +1056,11 @@ Instance: items: $ref: '#/InstanceDiskInfo' description: 实例盘列表 + disk_count: + type: integer + example: 4 + description: 磁盘数量 + InstanceListResponse: type: object diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 0f88619295..c421a670d7 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -202,6 +202,9 @@ type ServerCreateInput struct { ParentTaskId string `json:"__parent_task_id,omitempty"` // default stroage type if host is given DefaultStorageType string `json:"default_storage_type,omitempty"` + + // Guest Image ID + GuestImageID string } type ServerCloneInput struct { diff --git a/pkg/apis/compute/guest_image.go b/pkg/apis/compute/guest_image.go new file mode 100644 index 0000000000..0c8a016e2d --- /dev/null +++ b/pkg/apis/compute/guest_image.go @@ -0,0 +1,27 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +type sPair struct { + ID string + Name string + MinDiskMB int32 + DiskFormat string +} + +type SImagesInGuest struct { + RootImage sPair + DataImages []sPair +} diff --git a/pkg/cloudcommon/db/opslog.go b/pkg/cloudcommon/db/opslog.go index 3dc4acfd81..7c4a9e7681 100644 --- a/pkg/cloudcommon/db/opslog.go +++ b/pkg/cloudcommon/db/opslog.go @@ -80,11 +80,12 @@ const ( ACT_SPLIT = "net_split" ACT_MERGE = "net_merge" - ACT_SAVING = "saving" - ACT_SAVE = "save" - ACT_SAVE_FAIL = "save_fail" - ACT_PROBE = "probe" - ACT_PROBE_FAIL = "probe_fail" + ACT_SAVING = "saving" + ACT_SAVE = "save" + ACT_SAVE_FAIL = "save_fail" + ACT_PROBE = "probe" + ACT_PROBE_FAIL = "probe_fail" + ACT_IMAGE_DELETE_FAIL = "delete_fail" ACT_SWITCHED = "switched" ACT_SWITCH_FAILED = "switch_failed" @@ -216,6 +217,8 @@ const ( ACT_GUEST_ATTACH_ISOLATED_DEVICE_FAIL = "guest_attach_isolated_deivce_fail" ACT_GUEST_DETACH_ISOLATED_DEVICE = "guest_detach_isolated_deivce" ACT_GUEST_DETACH_ISOLATED_DEVICE_FAIL = "guest_detach_isolated_deivce_fail" + ACT_GUEST_SAVE_GUEST_IMAGE = "guest_save_guest_image" + ACT_GUEST_SAVE_GUEST_IMAGE_FAIL = "guest_save_guest_image_fail" ACT_CHANGE_BANDWIDTH = "eip_change_bandwidth" diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index c6b7eb1299..1875d7b032 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -473,6 +473,12 @@ func (self *SBaremetalGuestDriver) StartGuestSaveImage(ctx context.Context, user return httperrors.NewUnsupportOperationError("Cannot save image for baremtal") } +func (self *SBaremetalGuestDriver) StartGuestSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, + guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + + return httperrors.NewUnsupportOperationError("Cannot save image for baremtal") +} + func (self *SBaremetalGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { task, err := taskman.TaskManager.NewTask(ctx, "BaremetalServerResetTask", guest, userCred, nil, "", parentTaskId, nil) if err != nil { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index d1ca091c66..59e6e380ed 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -297,3 +297,15 @@ func (self *SVirtualizedGuestDriver) StartGuestSaveImage(ctx context.Context, us } return nil } + +func (self *SVirtualizedGuestDriver) StartGuestSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, + guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + + if task, err := taskman.TaskManager.NewTask(ctx, "GuestSaveGuestImageTask", guest, userCred, params, parentTaskId, + "", nil); err != nil { + return err + } else { + task.ScheduleRun(nil) + } + return nil +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 6eb075235c..4c72c0d2db 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -24,10 +24,9 @@ import ( "strings" "time" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/regutils" @@ -187,6 +186,79 @@ func (self *SGuest) StartGuestSaveImage(ctx context.Context, userCred mcclient.T return self.GetDriver().StartGuestSaveImage(ctx, userCred, self, data, parentTaskId) } +func (self *SGuest) AllowPerformSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + + return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "save-guest-image") +} + +func (self *SGuest) PerformSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + + if !utils.IsInStringArray(self.Status, []string{api.VM_READY}) { + return nil, httperrors.NewBadRequestError("Cannot save image in status %s", self.Status) + } + if !data.Contains("name") { + return nil, httperrors.NewMissingParameterError("Image name is required") + } + if self.Hypervisor != api.HYPERVISOR_KVM { + return nil, httperrors.NewBadRequestError("Support only by KVM Hypervisor") + } + disks := self.CategorizeDisks() + + if disks.Root == nil { + return nil, httperrors.NewInternalServerError("No root image") + } + + // build images + images := jsonutils.NewArray() + diskList := append(disks.Data, disks.Root) + for _, disk := range diskList { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(disk.DiskFormat), "disk_format") + params.Add(jsonutils.NewInt(int64(disk.DiskSize)), "virtual_size") + images.Add(params) + } + + // build parameters + kwargs := data.(*jsonutils.JSONDict) + + kwargs.Add(jsonutils.NewInt(int64(len(disks.Data)+1)), "image_number") + properties := jsonutils.NewDict() + if notes, err := kwargs.GetString("notes"); err != nil && len(notes) > 0 { + properties.Add(jsonutils.NewString(notes), "notes") + } + osType := self.OsType + if len(osType) == 0 { + osType = "Linux" + } + properties.Add(jsonutils.NewString(osType), "os_type") + kwargs.Add(properties, "properties") + + kwargs.Add(images, "images") + + s := auth.GetAdminSession(ctx, options.Options.Region, "") + ret, err := modules.GuestImages.Create(s, kwargs) + if err != nil { + return nil, err + } + imageIds, err := ret.Get("image_ids") + if err != nil { + return nil, fmt.Errorf("something wrong in glance") + } + tmp := imageIds.(*jsonutils.JSONArray) + if tmp.Length() != len(disks.Data)+1 { + return nil, fmt.Errorf("create subimage of guest image error") + } + taskParams := jsonutils.NewDict() + taskParams.Add(imageIds, "image_ids") + return nil, self.StartGuestSaveGuestImage(ctx, userCred, taskParams, "") +} + +func (self *SGuest) StartGuestSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error { + return self.GetDriver().StartGuestSaveGuestImage(ctx, userCred, self, data, parentTaskId) +} + func (self *SGuest) AllowPerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "sync") diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index b453f5246f..780ff86a76 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -112,6 +112,8 @@ type IGuestDriver interface { StartGuestSaveImage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + StartGuestSaveGuestImage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + RequestStopGuestForDelete(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error RequestDetachDisksFromGuestForDelete(ctx context.Context, guest *SGuest, task taskman.ITask) error diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 3aa666c37c..5b1615d5ab 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -54,6 +54,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/netutils2" @@ -927,6 +928,12 @@ func (manager *SGuestManager) validateCreateData( input.InstanceGroupIds = newGroupIds } + // check that all image of disk is the part of guest imgae, if use guest image to create guest + err = manager.checkGuestImage(ctx, input) + if err != nil { + return nil, err + } + var hypervisor string // var rootStorageType string var osProf osprofile.SOSProfile @@ -1529,6 +1536,10 @@ func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict, fields stringutils2 extra.Add(jsonutils.Marshal(metaData), "metadata") } + q := self.GetDisksQuery() + count, _ := q.CountWithError() + extra.Add(jsonutils.NewInt(int64(count)), "disk_count") + return extra } @@ -4755,3 +4766,51 @@ func (self *SGuest) GetDiskSnapshotsNotInInstanceSnapshots() ([]SSnapshot, error } return snapshots, nil } + +func (self *SGuestManager) checkGuestImage(ctx context.Context, input *api.ServerCreateInput) error { + // that data disks has image id show that these image is part of guest image. + for _, config := range input.Disks[1:] { + if len(config.ImageId) != 0 && len(input.GuestImageID) == 0 { + return httperrors.NewMissingParameterError("guest_image_id") + } + } + + if len(input.GuestImageID) == 0 { + return nil + } + + guestImageId := input.GuestImageID + params := jsonutils.NewDict() + params.Add(jsonutils.JSONTrue, "details") + + s := auth.GetAdminSession(ctx, options.Options.Region, "") + ret, err := modules.GuestImages.Get(s, guestImageId, params) + if err != nil { + return errors.Wrap(err, "get guest image from glance error") + } + + images := &api.SImagesInGuest{} + err = ret.Unmarshal(images) + if err != nil { + return errors.Wrap(err, "get guest image from glance error") + } + imageIdMap := make(map[string]struct{}) + for _, pair := range images.DataImages { + imageIdMap[pair.ID] = struct{}{} + } + imageIdMap[images.RootImage.ID] = struct{}{} + + // check + for _, diskConfig := range input.Disks { + if len(diskConfig.ImageId) != 0 { + if _, ok := imageIdMap[diskConfig.ImageId]; !ok { + return httperrors.NewBadRequestError("image %s do not belong to guest image %s", diskConfig.ImageId, guestImageId) + } + delete(imageIdMap, diskConfig.ImageId) + } + } + if len(imageIdMap) != 0 { + return httperrors.NewBadRequestError("miss some subimage of guest image") + } + return nil +} diff --git a/pkg/compute/tasks/guest_save_instance_image_task.go b/pkg/compute/tasks/guest_save_instance_image_task.go new file mode 100644 index 0000000000..6fc090e16d --- /dev/null +++ b/pkg/compute/tasks/guest_save_instance_image_task.go @@ -0,0 +1,102 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type GuestSaveGuestImageTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestSaveGuestImageTask{}) +} + +func (self *GuestSaveGuestImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + // prepare save image + guest := obj.(*models.SGuest) + + self.SetStage("OnSaveRootImageComplete", nil) + disks := guest.CategorizeDisks() + imageIds, _ := self.Params.GetArray("image_ids") + self.Params.Remove("image_ids") + + // data disk + for index, dataDisk := range disks.Data { + params := jsonutils.DeepCopy(self.Params).(*jsonutils.JSONDict) + params.Add(imageIds[index], "image_id") + if err := dataDisk.StartDiskSaveTask(ctx, self.UserCred, params, self.GetTaskId()); err != nil { + self.taskFailed(ctx, guest, err.Error()) + } + } + + self.Params.Add(imageIds[len(imageIds)-1], "image_id") + if err := disks.Root.StartDiskSaveTask(ctx, self.UserCred, self.Params, self.GetTaskId()); err != nil { + self.taskFailed(ctx, guest, err.Error()) + } +} + +func (self *GuestSaveGuestImageTask) OnSaveRootImageComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + subTasks := taskman.SubTaskManager.GetTotalSubtasks(self.Id, "on_save_root_image_complete", taskman.SUBTASK_FAIL) + + if len(subTasks) > 0 { + self.taskFailed(ctx, guest, "subtask failed") + } + + if restart, _ := self.GetParams().Bool("restart"); restart { + self.SetStage("on_start_server_complete", nil) + guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetTaskId()) + } else { + self.taskSuc(ctx, guest) + } +} + +func (self *GuestSaveGuestImageTask) OnSaveRootImageCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + log.Errorf("Guest save image failed: %s", data.PrettyString()) + self.taskFailed(ctx, guest, "") +} + +func (self *GuestSaveGuestImageTask) OnStartServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.taskSuc(ctx, guest) +} + +func (self *GuestSaveGuestImageTask) OnStartServerCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + // even if start server failed, the task that save guest image is successful + self.taskSuc(ctx, guest) +} + +func (self *GuestSaveGuestImageTask) taskSuc(ctx context.Context, guest *models.SGuest) { + self.SetStageComplete(ctx, nil) +} + +func (self *GuestSaveGuestImageTask) taskFailed(ctx context.Context, guest *models.SGuest, reason string) { + + guest.SetStatus(self.UserCred, api.VM_SAVE_DISK_FAILED, reason) + db.OpsLog.LogEvent(guest, db.ACT_GUEST_SAVE_GUEST_IMAGE_FAIL, reason, self.UserCred) + logclient.AddActionLogWithStartable(self, guest, logclient.ACT_IMAGE_SAVE, reason, self.UserCred, false) + + self.SetStageFailed(ctx, reason) +} diff --git a/pkg/image/models/image_guest.go b/pkg/image/models/image_guest.go new file mode 100644 index 0000000000..0626e3c5ea --- /dev/null +++ b/pkg/image/models/image_guest.go @@ -0,0 +1,410 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "fmt" + "sort" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/image/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +type SGuestImageManager struct { + db.SSharableVirtualResourceBaseManager +} + +var GuestImageManager *SGuestImageManager + +func init() { + GuestImageManager = &SGuestImageManager{ + db.NewSharableVirtualResourceBaseManager( + SGuestImage{}, + "guestimages_tbl", + "guestimage", + "guestimages", + ), + } + GuestImageManager.SetVirtualObject(GuestImageManager) +} + +type SGuestImage struct { + db.SSharableVirtualResourceBase + + Protected tristate.TriState `nullable:"false" default:"true" list:"user" get:"user" create:"optional" update:"user"` +} + +func (manager *SGuestImageManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + + if !data.Contains("image_number") { + return nil, httperrors.NewMissingParameterError("image_number") + } + imageNum, _ := data.Int("image_number") + + pendingUsage := SQuota{Image: int(imageNum)} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, nil, + &pendingUsage); err != nil { + + return nil, httperrors.NewOutOfQuotaError("%s", err) + } + return data, nil +} + +func (gi *SGuestImage) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + + err := gi.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) + if err != nil { + return err + } + gi.Status = api.IMAGE_STATUS_QUEUED + return nil +} + +func (gi *SGuestImage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + + kwargs := data.(*jsonutils.JSONDict) + // deal public params + kwargs.Remove("size") + kwargs.Remove("image_number") + kwargs.Remove("name") + if !kwargs.Contains("images") { + return + } + images, _ := kwargs.GetArray("images") + kwargs.Remove("images") + kwargs.Add(jsonutils.NewString(gi.Id), "guest_image_id") + + imageIds := jsonutils.NewArray() + suc := true + + // HACK + appParams := appsrv.AppContextGetParams(ctx) + appParams.Request.ContentLength = 0 + kwargs.Add(jsonutils.NewString("此镜像为主机镜像子镜像，不可操作"), "description") + + for i := 0; i < len(images); i++ { + params := jsonutils.DeepCopy(kwargs).(*jsonutils.JSONDict) + image := images[i].(*jsonutils.JSONDict) + for _, key := range image.SortedKeys() { + tmp, _ := image.Get(key) + params.Add(tmp, key) + } + if i == len(images)-1 { + params.Add(jsonutils.NewString(fmt.Sprintf("%s-%s", gi.Name, "root")), "generate_name") + } else { + params.Add(jsonutils.NewString(fmt.Sprintf("%s-%s-%d", gi.Name, "data", i)), "generate_name") + params.Add(jsonutils.JSONTrue, "is_data") + } + model, err := db.DoCreate(ImageManager, ctx, userCred, query, params, ownerId) + if err != nil { + imageIds.Add(jsonutils.NewString("")) + suc = false + break + } else { + func() { + lockman.LockObject(ctx, model) + defer lockman.ReleaseObject(ctx, model) + + model.PostCreate(ctx, userCred, ownerId, query, data) + }() + imageIds.Add(jsonutils.NewString(model.GetId())) + } + } + + imageNumber, _ := data.Int("image_number") + pendingUsage := SQuota{Image: int(imageNumber)} + QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage, &pendingUsage) + + if !suc { + gi.SetStatus(userCred, api.IMAGE_STATUS_KILLED, "create subimage failed") + } + + // HACK + tmp := query.(*jsonutils.JSONDict) + tmp.Add(imageIds, "image_ids") +} + +func (gi *SGuestImage) ValidateDeleteCondition(ctx context.Context) error { + if gi.Protected.IsTrue() { + return httperrors.NewForbiddenError("image is protected") + } + return nil +} + +func (gi *SGuestImage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + log.Infof("image delete to nothing") + return nil +} + +func (gi *SGuestImage) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + // delete joint + guestJoints, err := GuestImageJointManager.GetByGuestImageId(gi.Id) + if err != nil { + return errors.Wrap(err, "get guest image joint failed") + } + for i := range guestJoints { + guestJoints[i].Delete(ctx, userCred) + } + return gi.SVirtualResourceBase.Delete(ctx, userCred) +} + +func (gi *SGuestImage) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject) error { + + images, err := GuestImageJointManager.GetImagesByGuestImageId(gi.Id) + if err != nil { + return errors.Wrap(err, "get images of guest images failed") + } + if len(images) == 0 { + return gi.RealDelete(ctx, userCred) + } + overridePendingDelete := false + purge := false + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) + } + if gi.Status == api.IMAGE_STATUS_QUEUED { + gi.checkStatus(ctx, userCred) + } + if utils.IsInStringArray(gi.Status, []string{api.IMAGE_STATUS_QUEUED, api.IMAGE_STATUS_KILLED}) { + overridePendingDelete = true + } + return gi.startDeleteTask(ctx, userCred, "", purge, overridePendingDelete) +} + +func (gi *SGuestImage) startDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, + isPurge bool, overridePendingDelete bool) error { + + params := jsonutils.NewDict() + if isPurge { + params.Add(jsonutils.JSONTrue, "purge") + } + if overridePendingDelete { + params.Add(jsonutils.JSONTrue, "override_pending_delete") + } + params.Add(jsonutils.NewString(gi.Status), "image_status") + gi.SetStatus(userCred, api.IMAGE_STATUS_DEACTIVATED, "") + if task, err := taskman.TaskManager.NewTask(ctx, "GuestImageDeleteTask", gi, userCred, params, parentTaskId, "", + nil); err != nil { + + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + +func (gi *SGuestImage) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + + return db.IsAdminAllowPerform(userCred, gi, "cancel-delete") +} + +func (gi *SGuestImage) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + + if gi.PendingDeleted { + err := gi.DoCancelPendingDelete(ctx, userCred) + return nil, err + } + return nil, nil +} + +func (gi *SGuestImage) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + subImages, err := GuestImageJointManager.GetImagesByGuestImageId(gi.Id) + for i := range subImages { + err = subImages[i].DoCancelPendingDelete(ctx, userCred) + if err != nil { + return errors.Wrapf(err, "subimage %s cancel delete error", subImages[i].GetId()) + } + } + err = gi.SVirtualResourceBase.DoCancelPendingDelete(ctx, userCred) + if err != nil { + return err + } + _, err = db.Update(gi, func() error { + gi.Status = api.IMAGE_STATUS_ACTIVE + return nil + }) + return errors.Wrap(err, "guest image cancel delete error") +} + +type sPair struct { + ID string + Name string + MinDiskMB int32 + DiskFormat string +} + +func (self *SGuestImage) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + extra *jsonutils.JSONDict) *jsonutils.JSONDict { + + if self.Status != api.IMAGE_STATUS_ACTIVE { + self.checkStatus(ctx, userCred) + extra.Add(jsonutils.NewString(self.Status), "status") + } + images, err := GuestImageJointManager.GetImagesByGuestImageId(self.Id) + if err != nil { + return extra + } + var size int64 = 0 + if len(images) == 0 { + extra.Add(jsonutils.NewInt(size), "size") + return extra + } + dataImages := make([]sPair, 0, len(images)-1) + var rootImage sPair + for i := range images { + image := images[i] + size += image.Size + if !image.IsData.IsTrue() { + rootImage = sPair{image.Id, images[i].Name, image.MinDiskMB, image.DiskFormat} + extra.Add(jsonutils.NewInt(int64(image.MinRamMB)), "min_ram_mb") + continue + } + dataImages = append(dataImages, sPair{image.Id, image.Name, image.MinDiskMB, image.DiskFormat}) + } + // make sure that the sort of dataimage is fixed + sort.Slice(dataImages, func(i, j int) bool { + return dataImages[i].Name < dataImages[j].Name + }) + extra.Add(jsonutils.NewInt(size), "size") + extra.Add(jsonutils.Marshal(rootImage), "root_image") + extra.Add(jsonutils.Marshal(dataImages), "data_images") + // properties of root image + properties, _ := ImagePropertyManager.GetProperties(rootImage.ID) + if err != nil { + return extra + } + propJson := jsonutils.NewDict() + for k, v := range properties { + propJson.Add(jsonutils.NewString(v), k) + } + extra.Add(propJson, "properties") + return extra +} + +func (self *SGuestImage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject) *jsonutils.JSONDict { + + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(ctx, userCred, query, extra) +} + +func (self *SGuestImage) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, + query jsonutils.JSONObject) (*jsonutils.JSONDict, error) { + + extra, err := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + if err != nil { + return nil, err + } + if query.Contains("image_ids") { + imageIds, _ := query.Get("image_ids") + extra.Add(imageIds, "image_ids") + return extra, nil + } + + return self.getMoreDetails(ctx, userCred, query, extra), nil +} + +var checkStatus = map[string]int{ + api.IMAGE_STATUS_ACTIVE: 1, + api.IMAGE_STATUS_QUEUED: 2, + api.IMAGE_STATUS_SAVING: 3, + api.IMAGE_STATUS_DEACTIVATED: 4, + api.IMAGE_STATUS_KILLED: 5, +} + +func (self *SGuestImage) checkStatus(ctx context.Context, userCred mcclient.TokenCredential) error { + images, err := GuestImageJointManager.GetImagesByGuestImageId(self.Id) + if err != nil { + return err + } + if len(images) == 0 { + return nil + } + + status := api.IMAGE_STATUS_ACTIVE + + for i := range images { + if checkStatus[images[i].Status] > checkStatus[status] { + status = images[i].Status + } + } + if self.Status != status { + self.SetStatus(userCred, status, "") + self.Status = status + } + return nil +} + +func (self *SGuestImage) getSize(ctx context.Context, userCred mcclient.TokenCredential) (int64, error) { + images, err := GuestImageJointManager.GetImagesByGuestImageId(self.Id) + if err != nil { + return 0, err + } + var size int64 = 0 + for i := range images { + size += images[i].Size + } + return size, nil +} + +func (self *SGuestImageManager) getExpiredPendingDeleteImages() []SGuestImage { + deadline := time.Now().Add(time.Duration(-options.Options.PendingDeleteExpireSeconds) * time.Second) + + // there are so many common images of one guest image, so that batch shrink three times + q := self.Query().IsTrue("pending_deleted").LT("pending_deleted_at", + deadline).Limit(options.Options.PendingDeleteMaxCleanBatchSize / 3) + + images := make([]SGuestImage, 0) + err := db.FetchModelObjects(self, q, &images) + if err != nil { + log.Errorf("fetch guest images error %s", err) + return nil + } + return images +} + +func (self *SGuestImageManager) CleanPendingDeleteImages(ctx context.Context, userCred mcclient.TokenCredential, + isStart bool) { + + images := self.getExpiredPendingDeleteImages() + if images == nil { + return + } + for i := range images { + images[i].startDeleteTask(ctx, userCred, "", false, true) + } +} diff --git a/pkg/image/models/image_guest_joint.go b/pkg/image/models/image_guest_joint.go new file mode 100644 index 0000000000..7335bba1de --- /dev/null +++ b/pkg/image/models/image_guest_joint.go @@ -0,0 +1,122 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/errors" +) + +type SGuestImageJointManager struct { + db.SJointResourceBaseManager +} + +type SGuestImageJoint struct { + db.SJointResourceBase + + GuestImageId string `width:"128" charset:"ascii" create:"required"` + ImageId string `width:"128" charset:"ascii" create:"required"` +} + +var GuestImageJointManager *SGuestImageJointManager + +func init() { + GuestImageJointManager = &SGuestImageJointManager{ + db.NewJointResourceBaseManager( + SGuestImageJoint{}, + "guest_image_tbl", + "guestimagejoint", + "guestimagejoints", + GuestImageManager, + ImageManager, + ), + } + GuestImageJointManager.SetVirtualObject(GuestImageJointManager) +} + +func (gm *SGuestImageJointManager) GetByGuestImageId(guestImageId string) ([]SGuestImageJoint, error) { + q := gm.Query().Equals("guest_image_id", guestImageId) + ret := make([]SGuestImageJoint, 0, 1) + err := db.FetchModelObjects(gm, q, &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (gm *SGuestImageJointManager) GetByImageId(imageId string) (*SGuestImageJoint, error) { + q := gm.Query().Equals("image_id", imageId) + ret := SGuestImageJoint{} + err := q.First(&ret) + if err != nil { + return nil, err + } + return &ret, nil +} + +func (gm *SGuestImageJointManager) GetGuestImageByImageId(imageId string) (*SGuestImage, error) { + gits, err := gm.GetByImageId(imageId) + if err != nil { + return nil, err + } + model, err := GuestImageManager.FetchById(gits.GuestImageId) + if err != nil { + return nil, err + } + return model.(*SGuestImage), nil +} + +func (gm *SGuestImageJointManager) GetImagesByGuestImageId(guestImageId string) ([]SImage, error) { + giJoints, err := gm.GetByGuestImageId(guestImageId) + if err != nil { + return nil, errors.Wrap(err, "get joints of guest and image failed") + } + if len(giJoints) == 0 { + return []SImage{}, nil + } + imageIds := make([]string, len(giJoints)) + for i := range giJoints { + imageIds[i] = giJoints[i].ImageId + } + q := ImageManager.Query().In("id", imageIds) + images := make([]SImage, 0, len(imageIds)) + err = db.FetchModelObjects(ImageManager, q, &images) + if err != nil { + return nil, errors.Wrap(err, "fetch images failed") + } + return images, nil +} + +func (gt *SGuestImageJoint) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, gt) +} + +func (gt *SGuestImageJointManager) CreateGuestImageJoint(ctx context.Context, guestImageId, + imageId string) (*SGuestImageJoint, error) { + + gi := SGuestImageJoint{} + gi.GuestImageId = guestImageId + gi.ImageId = imageId + + // + if err := gt.TableSpec().Insert(&gi); err != nil { + return nil, errors.Wrapf(err, "insert guestimage joint error") + } + gi.SetVirtualObject(gt) + return &gi, nil +} diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 3b03277a65..7613b61a78 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -117,8 +117,10 @@ type SImage struct { MinDiskMB int32 `name:"min_disk" nullable:"false" default:"0" list:"user" create:"optional" update:"user"` MinRamMB int32 `name:"min_ram" nullable:"false" default:"0" list:"user" create:"optional" update:"user"` - Protected tristate.TriState `nullable:"false" default:"true" list:"user" get:"user" create:"optional" update:"user"` - IsStandard tristate.TriState `nullable:"false" default:"false" list:"user" get:"user" create:"admin_optional"` + Protected tristate.TriState `nullable:"false" default:"true" list:"user" get:"user" create:"optional" update:"user"` + IsStandard tristate.TriState `nullable:"false" default:"false" list:"user" get:"user" create:"admin_optional"` + IsGuestImage tristate.TriState `nullable:"false" default:"false" create:"optional" list:"user"` + IsData tristate.TriState `nullable:"false" default:"false" create:"optional" list:"user"` // image copy from url, save origin checksum before probe OssChecksum string `width:"32" charset:"ascii" nullable:"true" get:"user" list:"user"` @@ -187,20 +189,22 @@ func (self *SImage) CustomizedGetDetailsBody(ctx context.Context, userCred mccli filePath := self.getLocalLocation() status := self.Status - formatStr := jsonutils.GetAnyString(query, []string{"format", "disk_format"}) - if len(formatStr) > 0 { - subimg := ImageSubformatManager.FetchSubImage(self.Id, formatStr) - if subimg != nil { - isTorrent := jsonutils.QueryBoolean(query, "torrent", false) - if !isTorrent { - filePath = subimg.getLocalLocation() - status = subimg.Status + if self.IsGuestImage.IsFalse() { + formatStr := jsonutils.GetAnyString(query, []string{"format", "disk_format"}) + if len(formatStr) > 0 { + subimg := ImageSubformatManager.FetchSubImage(self.Id, formatStr) + if subimg != nil { + isTorrent := jsonutils.QueryBoolean(query, "torrent", false) + if !isTorrent { + filePath = subimg.getLocalLocation() + status = subimg.Status + } else { + filePath = subimg.getLocalTorrentLocation() + status = subimg.TorrentStatus + } } else { - filePath = subimg.getLocalTorrentLocation() - status = subimg.TorrentStatus + return nil, httperrors.NewNotFoundError("format %s not found", formatStr) } - } else { - return nil, httperrors.NewNotFoundError("format %s not found", formatStr) } } @@ -356,9 +360,14 @@ func (manager *SImageManager) ValidateCreateData(ctx context.Context, userCred m return nil, err } - pendingUsage := SQuota{Image: 1} - if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage); err != nil { - return nil, httperrors.NewOutOfQuotaError("%s", err) + // If this image is the part of guest image (contains "guest_image_id"), + // we do not need to check and set pending quota + // because that pending quota has been checked and set in SGuestImage.ValidateCreateData + if !data.Contains("guest_image_id") { + pendingUsage := SQuota{Image: 1} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage); err != nil { + return nil, httperrors.NewOutOfQuotaError("%s", err) + } } return data, nil @@ -371,6 +380,10 @@ func (self *SImage) CustomizeCreate(ctx context.Context, userCred mcclient.Token } self.Status = api.IMAGE_STATUS_QUEUED self.Owner = self.ProjectId + // if belong to a guest image, + if data.Contains("guest_image_id") { + self.IsGuestImage = tristate.True + } return nil } @@ -382,6 +395,11 @@ func (self *SImage) GetPath(format string) string { return path } +func (self *SImage) OnJointFailed(ctx context.Context, userCred mcclient.TokenCredential) { + log.Errorf("create joint of image and guest image failed") + self.SetStatus(userCred, api.IMAGE_STATUS_KILLED, "") +} + func (self *SImage) OnSaveFailed(ctx context.Context, userCred mcclient.TokenCredential, msg string) { self.saveFailed(userCred, msg) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_IMAGE_SAVE, nil, userCred, false) @@ -472,8 +490,11 @@ func (self *SImage) SaveImageFromStream(reader io.Reader, calChecksum bool) erro func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) - pendingUsage := SQuota{Image: 1} - QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage, &pendingUsage) + // if SImage belong to a guest image, pending quota will not be set. + if self.IsGuestImage.IsFalse() { + pendingUsage := SQuota{Image: 1} + QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, userCred, nil, &pendingUsage, &pendingUsage) + } if data.Contains("properties") { // update properties @@ -503,6 +524,16 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede self.startImageCopyFromUrlTask(ctx, userCred, copyFrom, "") } } + + // This image is belong to some guestImage. + // Code below must be run without upload. + if data.Contains("guest_image_id") { + guestImageId, _ := data.GetString("guest_image_id") + _, err := GuestImageJointManager.CreateGuestImageJoint(ctx, guestImageId, self.Id) + if err != nil { + self.OnJointFailed(ctx, userCred) + } + } } // After image probe and customization, image size and checksum changed @@ -523,6 +554,9 @@ func (self *SImage) ImageProbeAndCustomization( func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { if self.Status != api.IMAGE_STATUS_QUEUED { + if self.IsGuestImage.IsTrue() { + return nil, httperrors.NewForbiddenError("image is the part of guest imgae") + } appParams := appsrv.AppContextGetParams(ctx) if appParams != nil && appParams.Request.ContentLength > 0 { return nil, httperrors.NewInvalidStatusError("cannot upload in status %s", self.Status) @@ -530,16 +564,32 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To } else { appParams := appsrv.AppContextGetParams(ctx) if appParams != nil { + isProbe := true + if self.IsData.IsTrue() { + isProbe = false + } if appParams.Request.ContentLength > 0 { self.SetStatus(userCred, api.IMAGE_STATUS_SAVING, "update start upload") - err := self.SaveImageFromStream(appParams.Request.Body, false) + // If isProbe is true calculating checksum is not necessary wheng saving from stream, + // otherwise, it is needed. + err := self.SaveImageFromStream(appParams.Request.Body, !isProbe) if err != nil { self.OnSaveFailed(ctx, userCred, fmt.Sprintf("update upload failed %s", err)) return nil, httperrors.NewGeneralError(err) } self.OnSaveSuccess(ctx, userCred, "update upload success") - data.Remove("status") - self.ImageProbeAndCustomization(ctx, userCred, true) + if !isProbe { + // no probe + self.SetStatus(userCred, api.IMAGE_STATUS_ACTIVE, "data disk image upload success") + } else { + data.Remove("status") + // For guest image, DoConvertAfterProbe is not necessary. + if self.IsGuestImage.IsTrue() { + self.ImageProbeAndCustomization(ctx, userCred, false) + } else { + self.ImageProbeAndCustomization(ctx, userCred, true) + } + } } else { copyFrom := appParams.Request.Header.Get(modules.IMAGE_META_COPY_FROM) if len(copyFrom) > 0 { @@ -595,6 +645,9 @@ func (self *SImage) ValidateDeleteCondition(ctx context.Context) error { if self.IsPublic { return httperrors.NewInvalidStatusError("image is shared") } + if self.IsGuestImage.IsTrue() { + return httperrors.NewForbiddenError("image is the part of guest image") + } return self.SVirtualResourceBase.ValidateDeleteCondition(ctx) } @@ -687,7 +740,7 @@ func (self *SImage) StartImageConvertTask(ctx context.Context, userCred mcclient } func (self *SImage) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { - return db.IsAdminAllowPerform(userCred, self, "cancel-delete") + return db.IsAdminAllowPerform(userCred, self, "cancel-delete") && self.IsGuestImage.IsFalse() } func (self *SImage) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { @@ -702,7 +755,9 @@ func (manager *SImageManager) getExpiredPendingDeleteDisks() []SImage { deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds*-1) * time.Second) q := manager.Query() - q = q.IsTrue("pending_deleted").LT("pending_deleted_at", deadline).Limit(options.Options.PendingDeleteMaxCleanBatchSize) + // those images part of guest image will be clean in GuestImageManager.CleanPendingDeleteImages + q = q.IsTrue("pending_deleted").LT("pending_deleted_at", + deadline).Limit(options.Options.PendingDeleteMaxCleanBatchSize).IsFalse("belong_guest_image") disks := make([]SImage, 0) err := db.FetchModelObjects(ImageManager, q, &disks) @@ -720,7 +775,8 @@ func (manager *SImageManager) CleanPendingDeleteImages(ctx context.Context, user return } for i := 0; i < len(disks); i += 1 { - disks[i].startDeleteImageTask(ctx, userCred, "", false, false) + // clean pendingdelete so that overridePendingDelete is true + disks[i].startDeleteImageTask(ctx, userCred, "", false, true) } } @@ -1135,7 +1191,8 @@ func (self *SImage) DoCheckStatus(ctx context.Context, userCred mcclient.TokenCr } needConvert := false subimgs := ImageSubformatManager.GetAllSubImages(self.Id) - if len(subimgs) == 0 { + // for image the part of a guest image, convert is not necessary. + if len(subimgs) == 0 && self.IsGuestImage.IsFalse() { needConvert = true } for i := 0; i < len(subimgs); i += 1 { @@ -1160,7 +1217,7 @@ func (self *SImage) AllowPerformMarkStandard( query jsonutils.JSONObject, data jsonutils.JSONObject, ) bool { - return db.IsAdminAllowPerform(userCred, self, "mark-standard") + return db.IsAdminAllowPerform(userCred, self, "mark-standard") && self.IsGuestImage.IsFalse() } func (self *SImage) PerformMarkStandard( @@ -1188,7 +1245,7 @@ func (self *SImage) PerformMarkStandard( } func (self *SImage) AllowPerformUpdateTorrentStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { - return true + return self.IsGuestImage.IsFalse() } func (self *SImage) PerformUpdateTorrentStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { diff --git a/pkg/image/service/handlers.go b/pkg/image/service/handlers.go index 4559b7d557..fabfec9666 100644 --- a/pkg/image/service/handlers.go +++ b/pkg/image/service/handlers.go @@ -54,6 +54,8 @@ func initHandlers(app *appsrv.Application) { models.ImagePropertyManager, models.ImageSubformatManager, + models.GuestImageJointManager, + models.QuotaManager, models.QuotaUsageManager, } { @@ -64,6 +66,8 @@ func initHandlers(app *appsrv.Application) { db.OpsLog, db.Metadata, models.ImageManager, + + models.GuestImageManager, } { db.RegisterModelManager(manager) handler := db.NewModelHandler(manager) diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go index 6b294e4439..ab7db5749d 100644 --- a/pkg/image/service/service.go +++ b/pkg/image/service/service.go @@ -103,6 +103,8 @@ func StartService() { cron := cronman.InitCronJobManager(true, options.Options.CronJobWorkerCount) cron.AddJobAtIntervals("CleanPendingDeleteImages", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.ImageManager.CleanPendingDeleteImages) cron.AddJobAtIntervals("CalculateQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.QuotaManager.CalculateQuotaUsages) + cron.AddJobAtIntervals("CleanPendingDeleteGuestImages", + time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestImageManager.CleanPendingDeleteImages) cron.Start() } diff --git a/pkg/image/tasks/guest_image_delete_task.go b/pkg/image/tasks/guest_image_delete_task.go new file mode 100644 index 0000000000..55d0a55446 --- /dev/null +++ b/pkg/image/tasks/guest_image_delete_task.go @@ -0,0 +1,113 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/image/models" + "yunion.io/x/onecloud/pkg/image/options" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type GuestImageDeleteTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(GuestImageDeleteTask{}) +} + +func (self *GuestImageDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guestImage := obj.(*models.SGuestImage) + isPurge := jsonutils.QueryBoolean(self.Params, "purge", false) + isOverridePendingDelete := jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) + + if options.Options.EnablePendingDelete && !isPurge && !isOverridePendingDelete { + if guestImage.PendingDeleted { + self.SetStageComplete(ctx, nil) + } + self.startPendingDelete(ctx, guestImage) + } else { + self.startDelete(ctx, guestImage) + } +} + +func (self *GuestImageDeleteTask) startPendingDelete(ctx context.Context, guestImage *models.SGuestImage) { + images, err := models.GuestImageJointManager.GetImagesByGuestImageId(guestImage.GetId()) + if err != nil { + self.taskFailed(ctx, guestImage, err.Error()) + } + for i := range images { + images[i].StopTorrents() + err := images[i].DoPendingDelete(ctx, self.UserCred) + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("image %s pending delete failed", images[i].GetId())) + return + } + } + err = guestImage.DoPendingDelete(ctx, self.UserCred) + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("guest image %s pending delete failed", guestImage.GetId())) + } + self.SetStageComplete(ctx, nil) +} + +func (self *GuestImageDeleteTask) startDelete(ctx context.Context, guestImage *models.SGuestImage) { + images, err := models.GuestImageJointManager.GetImagesByGuestImageId(guestImage.GetId()) + if err != nil { + self.taskFailed(ctx, guestImage, err.Error()) + } + for i := range images { + err := images[i].RemoveFiles() + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("fail to remove %s: %s", images[i].GetPath(""), err)) + return + } + err = images[i].SetStatus(self.UserCred, api.IMAGE_STATUS_DELETED, "delete") + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("fail to set image %s status ", images[i].GetId())) + return + } + err = images[i].RealDelete(ctx, self.UserCred) + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("fail to real delete image %s", images[i].GetId())) + return + } + } + err = guestImage.SetStatus(self.UserCred, api.IMAGE_STATUS_DELETED, "delete") + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("fail to set guest image status %s", guestImage.GetId())) + } + err = guestImage.RealDelete(ctx, self.UserCred) + if err != nil { + self.taskFailed(ctx, guestImage, fmt.Sprintf("fail to real delete guest image %s", guestImage.GetId())) + } + self.SetStageComplete(ctx, nil) +} + +func (self *GuestImageDeleteTask) taskFailed(ctx context.Context, guestImage *models.SGuestImage, reason string) { + log.Errorf("Guest Image %s delete failed: %s", guestImage.Id, reason) + db.OpsLog.LogEvent(guestImage, db.ACT_IMAGE_DELETE_FAIL, reason, self.UserCred) + logclient.AddActionLogWithContext(ctx, guestImage, logclient.ACT_DELETE, reason, self.UserCred, false) + self.SetStageFailed(ctx, reason) +} diff --git a/pkg/image/tasks/image_copy_from_url_task.go b/pkg/image/tasks/image_copy_from_url_task.go index 65dfb7a9bf..b1633af290 100644 --- a/pkg/image/tasks/image_copy_from_url_task.go +++ b/pkg/image/tasks/image_copy_from_url_task.go @@ -69,7 +69,11 @@ func (self *ImageCopyFromUrlTask) OnInit(ctx context.Context, obj db.IStandalone func (self *ImageCopyFromUrlTask) OnImageImportComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { image := obj.(*models.SImage) image.OnSaveTaskSuccess(self, self.UserCred, "create upload success") - image.ImageProbeAndCustomization(ctx, self.UserCred, true) + if image.IsGuestImage.IsTrue() { + image.ImageProbeAndCustomization(ctx, self.UserCred, false) + } else { + image.ImageProbeAndCustomization(ctx, self.UserCred, true) + } self.SetStageComplete(ctx, nil) } diff --git a/pkg/image/tasks/image_probe_task.go b/pkg/image/tasks/image_probe_task.go index 49e4880740..6892f95db6 100644 --- a/pkg/image/tasks/image_probe_task.go +++ b/pkg/image/tasks/image_probe_task.go @@ -165,6 +165,7 @@ func (self *ImageProbeTask) OnProbeFailed(ctx context.Context, image *models.SIm self.SetStage("OnConvertComplete", nil) image.StartImageConvertTask(ctx, self.UserCred, self.GetId()) } else { + image.SetStatus(self.UserCred, api.IMAGE_STATUS_ACTIVE, "") self.SetStageFailed(ctx, reason) } } @@ -179,6 +180,7 @@ func (self *ImageProbeTask) OnProbeSuccess(ctx context.Context, image *models.SI self.SetStage("OnConvertComplete", nil) image.StartImageConvertTask(ctx, self.UserCred, self.GetId()) } else { + image.SetStatus(self.UserCred, api.IMAGE_STATUS_ACTIVE, "") self.SetStageComplete(ctx, nil) } } diff --git a/pkg/mcclient/modules/mod_images_guest.go b/pkg/mcclient/modules/mod_images_guest.go new file mode 100644 index 0000000000..8db10ac5ae --- /dev/null +++ b/pkg/mcclient/modules/mod_images_guest.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import "yunion.io/x/onecloud/pkg/mcclient/modulebase" + +var GuestImages modulebase.ResourceManager + +func init() { + GuestImages = NewImageManager("guestimage", "guestimages", + []string{"ID", "Name", "Status", "Size"}, + []string{}) + register(&GuestImages) +} diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index de698297fd..b5a9a9a03f 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -283,6 +283,8 @@ type ServerCreateOptions struct { EipBw int `help:"allocate EIP with bandwidth in MB when server is created" json:"eip_bw,omitzero"` EipChargeType string `help:"newly allocated EIP charge type" choices:"traffic|bandwidth" json:"eip_charge_type,omitempty"` Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` + + GuestImageID string `help:"create from guest image, need to specify the guest image id"` } func (o *ServerCreateOptions) ToScheduleInput() (*schedapi.ScheduleInput, error) { @@ -362,6 +364,7 @@ func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error) Eip: opts.Eip, EnableCloudInit: opts.EnableCloudInit, OsType: opts.OsType, + GuestImageID: opts.GuestImageID, } if opts.GenerateName {