diff --git a/pkg/apis/image/consts.go b/pkg/apis/image/consts.go index f0a8155127..15fc035ffe 100644 --- a/pkg/apis/image/consts.go +++ b/pkg/apis/image/consts.go @@ -19,6 +19,17 @@ const ( ImageTypeISO = TImageType("iso") LocalFilePrefix = "file://" + + // image properties + IMAGE_OS_ARCH = "os_arch" + IMAGE_OS_DISTRO = "os_distribution" + IMAGE_OS_TYPE = "os_type" + IMAGE_OS_VERSION = "os_version" + IMAGE_UEFI_SUPPORT = "uefi_support" + IMAGE_IS_LVM_PARTITION = "is_lvm_partition" + IMAGE_IS_READONLY = "is_readonly" + IMAGE_PARTITION_TYPE = "partition_type" + IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init" ) var ( diff --git a/pkg/cloudcommon/db/opslog.go b/pkg/cloudcommon/db/opslog.go index 6a34de0f5d..147e8d2c9b 100644 --- a/pkg/cloudcommon/db/opslog.go +++ b/pkg/cloudcommon/db/opslog.go @@ -74,9 +74,11 @@ const ( ACT_SPLIT = "net_split" ACT_MERGE = "net_merge" - ACT_SAVING = "saving" - ACT_SAVE = "save" - ACT_SAVE_FAIL = "save_fail" + ACT_SAVING = "saving" + ACT_SAVE = "save" + ACT_SAVE_FAIL = "save_fail" + ACT_PROBE = "probe" + ACT_PROBE_FAIL = "probe_fail" ACT_SWITCHED = "switched" ACT_SWITCH_FAILED = "switch_failed" diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 0466e850a6..ee8ffbf2ce 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -38,6 +38,7 @@ import ( billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" + imageapi "yunion.io/x/onecloud/pkg/apis/image" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/cmdline" "yunion.io/x/onecloud/pkg/cloudcommon/consts" @@ -884,6 +885,12 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m input.Disks[0] = diskConfig imgProperties := diskConfig.ImageProperties + imgSupportUEFI := imgProperties[imageapi.IMAGE_UEFI_SUPPORT] == "true" + imgIsWindows := imgProperties[imageapi.IMAGE_OS_TYPE] == "Windows" + if imgSupportUEFI && imgIsWindows && len(input.IsolatedDevices) > 0 { + input.Bios = "UEFI" // windows gpu passthrough + } + if input.Cdrom != "" { cdromStr := input.Cdrom image, err := parseIsoInfo(ctx, userCred, cdromStr) diff --git a/pkg/hostman/downloader/downloadhandler.go b/pkg/hostman/downloader/downloadhandler.go index f584e6e156..7cbab665da 100644 --- a/pkg/hostman/downloader/downloadhandler.go +++ b/pkg/hostman/downloader/downloadhandler.go @@ -65,7 +65,7 @@ func AddDownloadHandler(prefix string, app *appsrv.Application) { func customizeHandlerInfo(info *appsrv.SHandlerInfo) { switch info.GetName(nil) { case "disk_download", "download", "snapshot_download": - info.SetProcessTimeout(time.Minute * 30).SetWorkerManager(streamingWorkerMan) + info.SetProcessTimeout(time.Minute * 60).SetWorkerManager(streamingWorkerMan) } } diff --git a/pkg/hostman/guestfs/fsdriver/base.go b/pkg/hostman/guestfs/fsdriver/base.go index a0f64d04c9..8b536c96f6 100644 --- a/pkg/hostman/guestfs/fsdriver/base.go +++ b/pkg/hostman/guestfs/fsdriver/base.go @@ -109,11 +109,19 @@ func (d *sGuestRootFsDriver) CommitChanges(rootfs IDiskPartition) error { return nil } +func (d *sGuestRootFsDriver) DetectIsUEFISupport(IDiskPartition) bool { + return false +} + +func (l *sGuestRootFsDriver) IsCloudinitInstall() bool { + return false +} + type SReleaseInfo struct { - Distro string - Version string - Arch string - Language string + Distro string `json:"os_distribution,omitempty"` + Version string `json:"os_version,omitempty"` + Arch string `json:"os_arch,omitempty"` + Language string `json:"os_language,omitempty"` } func newReleaseInfo(distro, version, arch string) *SReleaseInfo { diff --git a/pkg/hostman/guestfs/fsdriver/interface.go b/pkg/hostman/guestfs/fsdriver/interface.go index b40b975c60..9e5b6a4cc9 100644 --- a/pkg/hostman/guestfs/fsdriver/interface.go +++ b/pkg/hostman/guestfs/fsdriver/interface.go @@ -41,8 +41,11 @@ type IDiskPartition interface { SupportSerialPorts() bool Mount() bool + MountPartReadOnly() bool Umount() bool GetMountPath() string + IsReadonly() bool + GetPhysicalPartitionType() string } type IRootFsDriver interface { @@ -69,6 +72,8 @@ type IRootFsDriver interface { DisableSerialConsole(IDiskPartition) error CommitChanges(IDiskPartition) error DeployFiles(deploys []jsonutils.JSONObject) error + DetectIsUEFISupport(IDiskPartition) bool + IsCloudinitInstall() bool PrepareFsForTemplate(IDiskPartition) error } diff --git a/pkg/hostman/guestfs/fsdriver/linux.go b/pkg/hostman/guestfs/fsdriver/linux.go index a8c30c77ec..a9cba09deb 100644 --- a/pkg/hostman/guestfs/fsdriver/linux.go +++ b/pkg/hostman/guestfs/fsdriver/linux.go @@ -431,6 +431,14 @@ func (l *sLinuxRootFs) disableSerialConsoleSystemd(rootFs IDiskPartition) { } } +func (l *sLinuxRootFs) DetectIsUEFISupport(part IDiskPartition) bool { + return part.Exists("/efi", false) +} + +func (l *sLinuxRootFs) IsCloudinitInstall() bool { + return l.GetPartition().Exists("/usr/bin/cloud-init", false) +} + type sDebianLikeRootFs struct { *sLinuxRootFs } diff --git a/pkg/hostman/guestfs/fsdriver/windows.go b/pkg/hostman/guestfs/fsdriver/windows.go index 544032cea5..9b4b714e75 100644 --- a/pkg/hostman/guestfs/fsdriver/windows.go +++ b/pkg/hostman/guestfs/fsdriver/windows.go @@ -476,3 +476,22 @@ func (w *SWindowsRootFs) DeployFstabScripts(rootFs IDiskPartition, disks []jsonu w.putGuestScriptContents("/windows/mountdisk.js", WinScriptMountDisk) return nil } + +func (w *SWindowsRootFs) DetectIsUEFISupport(part IDiskPartition) bool { + content, err := w.rootFs.FileGetContents("/windows/panther/setupact.log", true) + if err != nil { + log.Errorln(err) + return false + } + contentStr := string(content) + sep := "Detected boot environment: " + idx := strings.Index(contentStr, sep) + if idx < 0 { + return false + } + if strings.HasPrefix(contentStr[idx+len(sep):], "EFI") || + strings.HasPrefix(contentStr[idx+len(sep):], "UEFI") { + return true + } + return false +} diff --git a/pkg/hostman/guestfs/kvmpart.go b/pkg/hostman/guestfs/kvmpart.go index f6da5723d8..4812fb3a20 100644 --- a/pkg/hostman/guestfs/kvmpart.go +++ b/pkg/hostman/guestfs/kvmpart.go @@ -32,19 +32,49 @@ type SKVMGuestDiskPartition struct { partDev string fs string - readonly bool + readonly bool + sourceDev string + IsLVMPart bool } -func NewKVMGuestDiskPartition(devPath string) *SKVMGuestDiskPartition { +func NewKVMGuestDiskPartition(devPath, sourceDev string, isLVM bool) *SKVMGuestDiskPartition { var res = new(SKVMGuestDiskPartition) res.partDev = devPath res.fs = res.getFsFormat() + res.sourceDev = sourceDev + res.IsLVMPart = isLVM fileutils2.CleanFailedMountpoints() mountPath := fmt.Sprintf("/tmp/%s", strings.Replace(devPath, "/", "_", -1)) res.SLocalGuestFS = NewLocalGuestFS(mountPath) return res } +func (p *SKVMGuestDiskPartition) GetPhysicalPartitionType() string { + dev := p.partDev + if p.IsLVMPart { + // for lvm part, sourceDev like /dev/nbdxpx + idx := strings.LastIndexByte(p.sourceDev, 'p') + if idx > 0 { + dev = p.sourceDev[:idx] + } else { + dev = p.sourceDev + } + } + cmd := fmt.Sprintf(`fdisk -l %s | grep "Disk label type:"`, dev) + output, err := procutils.NewCommand("sh", "-c", cmd).Run() + if err != nil { + log.Errorf("get disk label type error %s", output) + return "" + } + idx := strings.Index(string(output), "Disk label type:") + tp := strings.TrimSpace(string(output)[idx+len("Disk label type:"):]) + if tp == "dos" { + return "mbr" + } else { + return tp + } +} + func (p *SKVMGuestDiskPartition) GetPartDev() string { return p.partDev } @@ -57,6 +87,22 @@ func (p *SKVMGuestDiskPartition) getFsFormat() string { return fileutils2.GetFsFormat(p.partDev) } +func (p *SKVMGuestDiskPartition) MountPartReadOnly() bool { + if len(p.fs) == 0 || utils.IsInStringArray(p.fs, []string{"swap", "btrfs"}) { + return false + } + + // no fsck, becasue read only + + err := p.mount(true) + if err != nil { + log.Errorf("SKVMGuestDiskPartition mount as readonly error: %s", err) + return false + } + p.readonly = true + return true +} + func (p *SKVMGuestDiskPartition) Mount() bool { if len(p.fs) == 0 || utils.IsInStringArray(p.fs, []string{"swap", "btrfs"}) { return false diff --git a/pkg/hostman/guestfs/sshpart/sshpart.go b/pkg/hostman/guestfs/sshpart/sshpart.go index 35b0a6c180..f0ffe5a810 100644 --- a/pkg/hostman/guestfs/sshpart/sshpart.go +++ b/pkg/hostman/guestfs/sshpart/sshpart.go @@ -113,6 +113,18 @@ func (p *SSHPartition) osPathExists(path string) bool { return true } +func (p *SSHPartition) MountPartReadOnly() bool { + return false // Not implement +} + +func (p *SSHPartition) IsReadonly() bool { + return false // sshpart not implement mount as readonly +} + +func (p *SSHPartition) GetPhysicalPartitionType() string { + return "" // Not implement +} + func (p *SSHPartition) Mount() bool { if err := p.osMkdirP(p.mountPath, 0); err != nil { log.Errorf("SSHPartition mount error: %s", err) diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index 06b2872eaf..4f3ebd6dfb 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -35,6 +35,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/metadata" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" + "yunion.io/x/onecloud/pkg/util/sysutils" ) type SHostService struct { @@ -43,8 +44,12 @@ type SHostService struct { func (host *SHostService) StartService() { common_options.ParseOptions(&options.HostOptions, os.Args, "host.conf", "host") - options.HostOptions.EnableRbac = false // disable rbac + isRoot := sysutils.IsRootPermission() + if !isRoot { + log.Fatalf("host service must running with root permissions") + } + options.HostOptions.EnableRbac = false // disable rbac app := app_common.InitApp(&options.HostOptions.CommonOptions, false) hostInstance := hostinfo.Instance() if err := hostInstance.Init(); err != nil { diff --git a/pkg/hostman/storageman/diskutils.go b/pkg/hostman/storageman/diskutils.go index a289a721ca..461c593bee 100644 --- a/pkg/hostman/storageman/diskutils.go +++ b/pkg/hostman/storageman/diskutils.go @@ -58,6 +58,10 @@ func NewKVMGuestDisk(imagePath string) *SKVMGuestDisk { return ret } +func (d *SKVMGuestDisk) IsLVMPartition() bool { + return len(d.lvms) > 0 +} + func (d *SKVMGuestDisk) Connect() bool { d.nbdDev = nbd.GetNBDManager().AcquireNbddev() if len(d.nbdDev) == 0 { @@ -141,7 +145,7 @@ func (d *SKVMGuestDisk) findPartitions() error { } for i := 0; i < len(files); i++ { if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") { - var part = guestfs.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name())) + var part = guestfs.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name()), "", false) d.partitions = append(d.partitions, part) } } @@ -259,9 +263,35 @@ func (d *SKVMGuestDisk) Disconnect() bool { } } -func (d *SKVMGuestDisk) MountKvmRootfs() fsdriver.IRootFsDriver { +func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool { for i := 0; i < len(d.partitions); i++ { - if d.partitions[i].Mount() { + if d.partitions[i].IsMounted() { + if rootfs.DetectIsUEFISupport(d.partitions[i]) { + return true + } + } else { + if d.partitions[i].Mount() { + support := rootfs.DetectIsUEFISupport(d.partitions[i]) + d.partitions[i].Umount() + if support { + return true + } + } + } + } + return false +} + +func (d *SKVMGuestDisk) MountKvmRootfs() fsdriver.IRootFsDriver { + return d.mountKvmRootfs(false) +} +func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) fsdriver.IRootFsDriver { + for i := 0; i < len(d.partitions); i++ { + mountFunc := d.partitions[i].Mount + if readonly { + mountFunc = d.partitions[i].MountPartReadOnly + } + if mountFunc() { if fs := guestfs.DetectRootFs(d.partitions[i]); fs != nil { log.Infof("Use rootfs %s, partition %s", fs, d.partitions[i].GetPartDev()) @@ -274,6 +304,10 @@ func (d *SKVMGuestDisk) MountKvmRootfs() fsdriver.IRootFsDriver { return nil } +func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() fsdriver.IRootFsDriver { + return d.mountKvmRootfs(true) +} + func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) { if part := fd.GetPartition(); part != nil { part.Umount() diff --git a/pkg/hostman/storageman/lvmutils.go b/pkg/hostman/storageman/lvmutils.go index 6a2cb83df5..814a2bbd35 100644 --- a/pkg/hostman/storageman/lvmutils.go +++ b/pkg/hostman/storageman/lvmutils.go @@ -192,7 +192,7 @@ func (p *SKVMGuestLVMPartition) FindPartitions() []*guestfs.SKVMGuestDiskPartiti parts := []*guestfs.SKVMGuestDiskPartition{} for _, f := range files { partPath := fmt.Sprintf("/dev/%s/%s", p.vgname, f.Name()) - part := guestfs.NewKVMGuestDiskPartition(partPath) + part := guestfs.NewKVMGuestDiskPartition(partPath, p.partDev, true) parts = append(parts, part) } return parts diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 183a90639d..7572c17f7e 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -117,7 +117,7 @@ type SImage struct { func (manager *SImageManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) { switch info.GetName(nil) { case "get_details", "create", "update": - info.SetProcessTimeout(time.Minute * 30).SetWorkerManager(imgStreamingWorkerMan) + info.SetProcessTimeout(time.Minute * 120).SetWorkerManager(imgStreamingWorkerMan) } } @@ -208,7 +208,7 @@ func (self *SImage) CustomizedGetDetailsBody(ctx context.Context, userCred mccli } defer fp.Close() - _, err = streamutils.StreamPipe(fp, appParams.Response) + _, err = streamutils.StreamPipe(fp, appParams.Response, false) if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -374,19 +374,20 @@ func (self *SImage) saveFailed(userCred mcclient.TokenCredential, msg string) { db.OpsLog.LogEvent(self, db.ACT_SAVE_FAIL, msg, userCred) } -func (self *SImage) saveImageFromStream(localPath string, reader io.Reader) (*streamutils.SStreamProperty, error) { +func (self *SImage) saveImageFromStream(localPath string, reader io.Reader, calChecksum bool) (*streamutils.SStreamProperty, error) { fp, err := os.Create(localPath) if err != nil { return nil, err } defer fp.Close() - return streamutils.StreamPipe(reader, fp) + return streamutils.StreamPipe(reader, fp, calChecksum) } -func (self *SImage) SaveImageFromStream(reader io.Reader) error { +//Image always do probe and customize after save from stream +func (self *SImage) SaveImageFromStream(reader io.Reader, calChecksum bool) error { localPath := self.GetPath("") - sp, err := self.saveImageFromStream(localPath, reader) + sp, err := self.saveImageFromStream(localPath, reader, calChecksum) if err != nil { log.Errorf("saveImageFromStream fail %s", err) return err @@ -401,15 +402,20 @@ func (self *SImage) SaveImageFromStream(reader io.Reader) error { format = string(img.Format) virtualSizeBytes = img.SizeBytes - fastChksum, err := fileutils2.FastCheckSum(localPath) - if err != nil { - return err + var fastChksum string + if calChecksum { + fastChksum, err = fileutils2.FastCheckSum(localPath) + if err != nil { + return err + } } db.Update(self, func() error { self.Size = sp.Size - self.Checksum = sp.CheckSum - self.FastHash = fastChksum + if calChecksum { + self.Checksum = sp.CheckSum + self.FastHash = fastChksum + } self.Location = fmt.Sprintf("%s%s", LocalFilePrefix, localPath) if len(format) > 0 { self.DiskFormat = format @@ -443,15 +449,14 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede db.OpsLog.LogEvent(self, db.ACT_SAVING, "create upload", userCred) self.SetStatus(userCred, api.IMAGE_STATUS_SAVING, "create upload") - err := self.SaveImageFromStream(appParams.Request.Body) + err := self.SaveImageFromStream(appParams.Request.Body, false) if err != nil { self.OnSaveFailed(ctx, userCred, fmt.Sprintf("create upload fail %s", err)) return } self.OnSaveSuccess(ctx, userCred, "create upload success") - - self.StartImageConvertTask(ctx, userCred, "") + self.ImageProbeAndCustomization(ctx, userCred, true) } else { copyFrom := appParams.Request.Header.Get(modules.IMAGE_META_COPY_FROM) if len(copyFrom) > 0 { @@ -460,6 +465,22 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede } } +// After image probe and customization, image size and checksum changed +// will recalculate checksum in the end +func (self *SImage) ImageProbeAndCustomization( + ctx context.Context, userCred mcclient.TokenCredential, doConvertAfterProbe bool, +) error { + data := jsonutils.NewDict() + data.Set("do_convert", jsonutils.NewBool(doConvertAfterProbe)) + task, err := taskman.TaskManager.NewTask( + ctx, "ImageProbeTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { if self.Status != api.IMAGE_STATUS_QUEUED { appParams := appsrv.AppContextGetParams(ctx) @@ -471,14 +492,14 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To if appParams != nil { if appParams.Request.ContentLength > 0 { self.SetStatus(userCred, api.IMAGE_STATUS_SAVING, "update start upload") - err := self.SaveImageFromStream(appParams.Request.Body) + err := self.SaveImageFromStream(appParams.Request.Body, false) if err != nil { self.OnSaveFailed(ctx, userCred, fmt.Sprintf("update upload failed %s", err)) return nil, httperrors.NewGeneralError(err) } self.OnSaveSuccess(ctx, userCred, "update upload success") data.Remove("status") - self.StartImageConvertTask(ctx, userCred, "") + self.ImageProbeAndCustomization(ctx, userCred, true) } else { copyFrom := appParams.Request.Header.Get(modules.IMAGE_META_COPY_FROM) if len(copyFrom) > 0 { diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go index 7d7717f74f..f0ca5db0d3 100644 --- a/pkg/image/service/service.go +++ b/pkg/image/service/service.go @@ -33,6 +33,7 @@ import ( _ "yunion.io/x/onecloud/pkg/image/tasks" "yunion.io/x/onecloud/pkg/image/torrent" "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/sysutils" ) const ( @@ -45,6 +46,11 @@ func StartService() { dbOpts := &opts.DBOptions common_options.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE) + isRoot := sysutils.IsRootPermission() + if !isRoot { + log.Fatalf("glance service must running with root permissions") + } + if opts.PortV2 > 0 { log.Infof("Port V2 %d is specified, use v2 port", opts.PortV2) opts.Port = opts.PortV2 diff --git a/pkg/image/tasks/image_copy_from_url_task.go b/pkg/image/tasks/image_copy_from_url_task.go index 59a869ad42..49059ce7a6 100644 --- a/pkg/image/tasks/image_copy_from_url_task.go +++ b/pkg/image/tasks/image_copy_from_url_task.go @@ -50,7 +50,7 @@ func (self *ImageCopyFromUrlTask) OnInit(ctx context.Context, obj db.IStandalone if err != nil { return nil, err } - err = image.SaveImageFromStream(resp.Body) + err = image.SaveImageFromStream(resp.Body, false) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (self *ImageCopyFromUrlTask) OnInit(ctx context.Context, obj db.IStandalone func (self *ImageCopyFromUrlTask) OnImageImportComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { image := obj.(*models.SImage) image.OnSaveTaskSuccess(self, self.UserCred, "create upload success") - image.StartImageConvertTask(ctx, self.UserCred, "") + image.ImageProbeAndCustomization(ctx, self.UserCred, true) self.SetStageComplete(ctx, nil) } diff --git a/pkg/image/tasks/image_probe_task.go b/pkg/image/tasks/image_probe_task.go new file mode 100644 index 0000000000..7b25425e65 --- /dev/null +++ b/pkg/image/tasks/image_probe_task.go @@ -0,0 +1,186 @@ +package tasks + +import ( + "context" + "fmt" + "os" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" + "yunion.io/x/onecloud/pkg/hostman/storageman" + "yunion.io/x/onecloud/pkg/image/models" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type ImageProbeTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(ImageProbeTask{}) +} + +func (self *ImageProbeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + image := obj.(*models.SImage) + self.StartImageProbe(ctx, image) +} + +func (self *ImageProbeTask) StartImageProbe(ctx context.Context, image *models.SImage) { + probeErr := self.doProbe(ctx, image) + + // if failed, set image unavailable, because size and chksum changed + if err := self.updateImageMetadata(ctx, image); err != nil { + image.SetStatus(self.UserCred, api.IMAGE_STATUS_KILLED, + fmt.Sprintf("Image update failed after probe %s", err)) + self.SetStageFailed(ctx, err.Error()) + return + } + + if probeErr == nil { + self.OnProbeSuccess(ctx, image) + } else { + self.OnProbeFailed(ctx, image, probeErr.Error()) + } +} + +func (self *ImageProbeTask) doProbe(ctx context.Context, image *models.SImage) error { + diskPath := image.GetPath("") + kvmDisk := storageman.NewKVMGuestDisk(diskPath) + defer kvmDisk.Disconnect() + if !kvmDisk.Connect() { + return fmt.Errorf("Disk connector failed to connect image") + } + + // Fsck is executed during mount + rootfs := kvmDisk.MountKvmRootfs() + if rootfs == nil { + return fmt.Errorf("Failed mounting rootfs for kvm disk") + } + defer kvmDisk.UmountKvmRootfs(rootfs) + + imageInfo := self.getImageInfo(kvmDisk, rootfs) + self.updateImageInfo(ctx, image, imageInfo) + return nil +} + +func (self *ImageProbeTask) updateImageMetadata( + ctx context.Context, image *models.SImage, +) error { + imagePath := image.GetPath("") + fp, err := os.Open(imagePath) + if err != nil { + return err + } + defer fp.Close() + + stat, err := fp.Stat() + if err != nil { + return err + } + + chksum, err := fileutils2.MD5(imagePath) + if err != nil { + return err + } + + fastchksum, err := fileutils2.FastCheckSum(imagePath) + if err != nil { + return err + } + + _, err = db.Update(image, func() error { + image.Size = stat.Size() + image.Checksum = chksum + image.FastHash = fastchksum + return nil + }) + if err != nil { + return err + } + return nil +} + +func (self *ImageProbeTask) updateImageInfo(ctx context.Context, image *models.SImage, imageInfo *sImageInfo) { + imageProperties := jsonutils.Marshal(imageInfo.osInfo).(*jsonutils.JSONDict) + if len(imageInfo.OsType) > 0 { + imageProperties.Set(api.IMAGE_OS_TYPE, jsonutils.NewString(imageInfo.OsType)) + } + if imageInfo.IsUEFISupport { + imageProperties.Set(api.IMAGE_UEFI_SUPPORT, jsonutils.JSONTrue) + } + if imageInfo.IsLVMPartition { + imageProperties.Set(api.IMAGE_IS_LVM_PARTITION, jsonutils.JSONTrue) + } + if imageInfo.IsReadonly { + imageProperties.Set(api.IMAGE_IS_READONLY, jsonutils.JSONTrue) + } + if imageInfo.IsInstalledCloudInit { + imageProperties.Set(api.IMAGE_INSTALLED_CLOUDINIT, jsonutils.JSONTrue) + + } + if len(imageInfo.PhysicalPartitionType) > 0 { + imageProperties.Set(api.IMAGE_PARTITION_TYPE, jsonutils.NewString(imageInfo.PhysicalPartitionType)) + } + models.ImagePropertyManager.SaveProperties(ctx, self.UserCred, image.Id, imageProperties) +} + +type sImageInfo struct { + osInfo *fsdriver.SReleaseInfo + + OsType string + IsUEFISupport bool + IsLVMPartition bool + IsReadonly bool + PhysicalPartitionType string // mbr or gbt or unknown + IsInstalledCloudInit bool +} + +func (self *ImageProbeTask) getImageInfo(kvmDisk *storageman.SKVMGuestDisk, rootfs fsdriver.IRootFsDriver) *sImageInfo { + partition := rootfs.GetPartition() + return &sImageInfo{ + osInfo: rootfs.GetReleaseInfo(partition), + OsType: rootfs.GetOs(), + IsUEFISupport: kvmDisk.DetectIsUEFISupport(rootfs), + IsLVMPartition: kvmDisk.IsLVMPartition(), + IsReadonly: partition.IsReadonly(), + PhysicalPartitionType: partition.GetPhysicalPartitionType(), + IsInstalledCloudInit: rootfs.IsCloudinitInstall(), + } +} + +func (self *ImageProbeTask) OnProbeFailed(ctx context.Context, image *models.SImage, reason string) { + log.Infof("Image %s Probe Failed ...", image.Name) + db.OpsLog.LogEvent(image, db.ACT_PROBE_FAIL, reason, self.UserCred) + logclient.AddActionLogWithContext(ctx, image, logclient.ACT_IMAGE_PROBE, reason, self.UserCred, false) + + if jsonutils.QueryBoolean(self.Params, "do_convert", false) { + self.SetStage("OnConvertComplete", nil) + image.StartImageConvertTask(ctx, self.UserCred, self.GetId()) + } else { + self.SetStageFailed(ctx, reason) + } +} + +func (self *ImageProbeTask) OnProbeSuccess(ctx context.Context, image *models.SImage) { + log.Infof("Image %s Probe Success ...", image.Name) + db.OpsLog.LogEvent(image, db.ACT_PROBE, "Image Probe Success", self.UserCred) + logclient.AddActionLogWithContext( + ctx, image, logclient.ACT_IMAGE_PROBE, "Image Probe Success", self.UserCred, true) + + if jsonutils.QueryBoolean(self.Params, "do_convert", false) { + self.SetStage("OnConvertComplete", nil) + image.StartImageConvertTask(ctx, self.UserCred, self.GetId()) + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *ImageProbeTask) OnConvertComplete(ctx context.Context, image *models.SImage, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/util/logclient/logclient.go b/pkg/util/logclient/logclient.go index 4a7dfd47c3..fc2b7415b8 100644 --- a/pkg/util/logclient/logclient.go +++ b/pkg/util/logclient/logclient.go @@ -68,6 +68,7 @@ var WhiteList = []string{ ACT_IMAGE_SAVE, ACT_RECYCLE_PREPAID, ACT_UNDO_RECYCLE_PREPAID, ACT_FETCH, ACT_VM_CHANGE_NIC, ACT_HOST_IMPORT_LIBVIRT_SERVERS, ACT_GUEST_CREATE_FROM_IMPORT, ACT_DISK_CREATE_SNAPSHOT, + ACT_IMAGE_PROBE, } const ( @@ -138,7 +139,8 @@ const ( ACT_LB_REMOVE_LISTENER_RULE = "移除负载均衡转发规则" ACT_DELETE_BACKUP = "删除备份机" - ACT_IMAGE_SAVE = "上传镜像" + ACT_IMAGE_SAVE = "上传镜像" + ACT_IMAGE_PROBE = "镜像检测" ACT_RECYCLE_PREPAID = "池化预付费主机" ACT_UNDO_RECYCLE_PREPAID = "取消池化预付费主机" diff --git a/pkg/util/streamutils/streamutils.go b/pkg/util/streamutils/streamutils.go index 54f089ad67..b044f56e73 100644 --- a/pkg/util/streamutils/streamutils.go +++ b/pkg/util/streamutils/streamutils.go @@ -17,6 +17,7 @@ package streamutils import ( "crypto/md5" "fmt" + "hash" "io" ) @@ -25,17 +26,22 @@ type SStreamProperty struct { Size int64 } -func StreamPipe(reader io.Reader, writer io.Writer) (*SStreamProperty, error) { +func StreamPipe(reader io.Reader, writer io.Writer, CalChecksum bool) (*SStreamProperty, error) { sp := SStreamProperty{} - md5sum := md5.New() + var md5sum hash.Hash + if CalChecksum { + md5sum = md5.New() + } buf := make([]byte, 4096) for { n, err := reader.Read(buf) if n > 0 { sp.Size += int64(n) - md5sum.Write(buf[:n]) + if CalChecksum { + md5sum.Write(buf[:n]) + } offset := 0 for offset < n { m, err := writer.Write(buf[offset:n]) @@ -53,6 +59,8 @@ func StreamPipe(reader io.Reader, writer io.Writer) (*SStreamProperty, error) { } } - sp.CheckSum = fmt.Sprintf("%x", md5sum.Sum(nil)) + if CalChecksum { + sp.CheckSum = fmt.Sprintf("%x", md5sum.Sum(nil)) + } return &sp, nil } diff --git a/pkg/util/sysutils/privilege.go b/pkg/util/sysutils/privilege.go new file mode 100644 index 0000000000..ef9c44a066 --- /dev/null +++ b/pkg/util/sysutils/privilege.go @@ -0,0 +1,7 @@ +package sysutils + +import "os" + +func IsRootPermission() bool { + return os.Geteuid() == 0 +}