From e16cf8438cb7a8282a24e50783f59cb7462c229a Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Fri, 27 Sep 2024 00:37:44 +0800 Subject: [PATCH] Automated cherry pick of #21313: Automated cherry pick of #21312: Automated cherry pick of #20996: fix(region,host): lvm snapshot delete and create from backup (#21315) * fix(region,host): lvm snapshot delete and create from backup * fix(host): slvm delete snapshot and save images * fix(host): add option boot vga pci addr * fix(host): slvm active exclusive mode on convert snapshot --- pkg/compute/storagedrivers/base.go | 23 +++++++ pkg/compute/tasks/snapshot_delete_task.go | 16 +++++ .../guestman/guesthandlers/guesthandler.go | 9 +++ pkg/hostman/guestman/guesthelper.go | 2 + pkg/hostman/guestman/guestman.go | 4 +- pkg/hostman/guestman/guesttasks.go | 7 +- pkg/hostman/guestman/qemu-kvm.go | 12 ++-- pkg/hostman/hostinfo/hostinfo.go | 60 +++++++++-------- pkg/hostman/isolated_device/gpu.go | 42 ++++++++++-- pkg/hostman/options/options.go | 7 +- pkg/hostman/storageman/disk_base.go | 8 +-- pkg/hostman/storageman/disk_local.go | 16 +++-- pkg/hostman/storageman/disk_lvm.go | 25 ++------ pkg/hostman/storageman/disk_rbd.go | 6 +- pkg/hostman/storageman/disk_slvm.go | 64 +++++++++++++++++++ pkg/hostman/storageman/storage_lvm.go | 63 ++++++++++++++---- pkg/hostman/storageman/storage_slvm.go | 4 +- .../storagehandler/storagehandler.go | 10 +++ pkg/hostman/storageman/storagehelper.go | 2 + pkg/util/qemuimg/qemuimg.go | 9 ++- 20 files changed, 299 insertions(+), 90 deletions(-) diff --git a/pkg/compute/storagedrivers/base.go b/pkg/compute/storagedrivers/base.go index c48ac995c8..44eeac4820 100644 --- a/pkg/compute/storagedrivers/base.go +++ b/pkg/compute/storagedrivers/base.go @@ -113,6 +113,15 @@ func (self *SBaseStorageDriver) RequestDeleteSnapshot(ctx context.Context, snaps if err != nil && err != sql.ErrNoRows { return errors.Wrap(err, "get disk by snapshot") } + sDisk, _ := disk.(*models.SDisk) + if sDisk.IsEncrypted() { + if encryptInfo, err := sDisk.GetEncryptInfo(ctx, task.GetUserCred()); err != nil { + return errors.Wrap(err, "faild get encryptInfo") + } else { + params.Set("encrypt_info", jsonutils.Marshal(encryptInfo)) + } + } + if !snapshot.OutOfChain { if convertSnapshot != nil { params.Set("convert_snapshot", jsonutils.NewString(convertSnapshot.Id)) @@ -152,6 +161,20 @@ func (self *SBaseStorageDriver) RequestDeleteSnapshot(ctx context.Context, snaps params := jsonutils.NewDict() params.Set("delete_snapshot", jsonutils.NewString(snapshot.Id)) params.Set("disk_id", jsonutils.NewString(snapshot.DiskId)) + + disk, err := models.DiskManager.FetchById(snapshot.DiskId) + if err != nil && err != sql.ErrNoRows { + return errors.Wrap(err, "get disk by snapshot") + } + sDisk, _ := disk.(*models.SDisk) + if sDisk.IsEncrypted() { + if encryptInfo, err := sDisk.GetEncryptInfo(ctx, task.GetUserCred()); err != nil { + return errors.Wrap(err, "faild get encryptInfo") + } else { + params.Set("encrypt_info", jsonutils.Marshal(encryptInfo)) + } + } + if !snapshot.OutOfChain { if convertSnapshot != nil { params.Set("convert_snapshot", jsonutils.NewString(convertSnapshot.Id)) diff --git a/pkg/compute/tasks/snapshot_delete_task.go b/pkg/compute/tasks/snapshot_delete_task.go index 4672662820..b67efef0b3 100644 --- a/pkg/compute/tasks/snapshot_delete_task.go +++ b/pkg/compute/tasks/snapshot_delete_task.go @@ -16,6 +16,7 @@ package tasks import ( "context" + "database/sql" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/jsonutils" @@ -112,6 +113,21 @@ func (self *SnapshotDeleteTask) OnReloadDiskSnapshot(ctx context.Context, snapsh } if snapshot.FakeDeleted { params := jsonutils.NewDict() + disk, err := models.DiskManager.FetchById(snapshot.DiskId) + if err != nil && err != sql.ErrNoRows { + self.TaskFailed(ctx, snapshot, jsonutils.NewString(err.Error())) + return + } + sDisk, _ := disk.(*models.SDisk) + if sDisk.IsEncrypted() { + if encryptInfo, err := sDisk.GetEncryptInfo(ctx, self.GetUserCred()); err != nil { + self.TaskFailed(ctx, snapshot, jsonutils.NewString(err.Error())) + return + } else { + params.Set("encrypt_info", jsonutils.Marshal(encryptInfo)) + } + } + params.Set("delete_snapshot", jsonutils.NewString(snapshot.Id)) params.Set("disk_id", jsonutils.NewString(snapshot.DiskId)) params.Set("auto_deleted", jsonutils.JSONTrue) diff --git a/pkg/hostman/guestman/guesthandlers/guesthandler.go b/pkg/hostman/guestman/guesthandlers/guesthandler.go index af6e67f85b..0504aaa4c6 100644 --- a/pkg/hostman/guestman/guesthandlers/guesthandler.go +++ b/pkg/hostman/guestman/guesthandlers/guesthandler.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis" computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" @@ -721,6 +722,14 @@ func guestDeleteSnapshot(ctx context.Context, userCred mcclient.TokenCredential, Disk: disk, } + if body.Contains("encrypt_info") { + encryptInfo := apis.SEncryptInfo{} + if err = body.Unmarshal(&encryptInfo, "encrypt_info"); err != nil { + return nil, httperrors.NewInputParameterError("unmarshal encrypt_info failed %s", err) + } + params.EncryptInfo = encryptInfo + } + // blockStream indicate snapshot<-disk blockStream := jsonutils.QueryBoolean(body, "block_stream", false) autoDeleted := jsonutils.QueryBoolean(body, "auto_deleted", false) diff --git a/pkg/hostman/guestman/guesthelper.go b/pkg/hostman/guestman/guesthelper.go index e3832f410e..862c87dab0 100644 --- a/pkg/hostman/guestman/guesthelper.go +++ b/pkg/hostman/guestman/guesthelper.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" @@ -143,6 +144,7 @@ type SDeleteDiskSnapshot struct { Disk storageman.IDisk ConvertSnapshot string BlockStream bool + EncryptInfo apis.SEncryptInfo } type SLibvirtServer struct { diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index d8f8bdaf7b..1c2d5950b3 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -1291,11 +1291,11 @@ func (m *SGuestManager) DeleteSnapshot(ctx context.Context, params interface{}) if len(delParams.ConvertSnapshot) > 0 || delParams.BlockStream { guest, _ := m.GetKVMServer(delParams.Sid) return guest.ExecDeleteSnapshotTask(ctx, delParams.Disk, delParams.DeleteSnapshot, - delParams.ConvertSnapshot, delParams.BlockStream) + delParams.ConvertSnapshot, delParams.BlockStream, delParams.EncryptInfo) } else { res := jsonutils.NewDict() res.Set("deleted", jsonutils.JSONTrue) - return res, delParams.Disk.DeleteSnapshot(delParams.DeleteSnapshot, "", false) + return res, delParams.Disk.DeleteSnapshot(delParams.DeleteSnapshot, "", false, delParams.EncryptInfo) } } diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 47e59d6fcc..fc5bc2e774 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/pkg/util/version" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" @@ -2041,19 +2042,21 @@ type SGuestSnapshotDeleteTask struct { deleteSnapshot string convertSnapshot string blockStream bool + encryptInfo apis.SEncryptInfo tmpPath string } func NewGuestSnapshotDeleteTask( ctx context.Context, s *SKVMGuestInstance, disk storageman.IDisk, - deleteSnapshot, convertSnapshot string, blockStream bool, + deleteSnapshot, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo, ) *SGuestSnapshotDeleteTask { return &SGuestSnapshotDeleteTask{ SGuestReloadDiskTask: NewGuestReloadDiskTask(ctx, s, disk), deleteSnapshot: deleteSnapshot, convertSnapshot: convertSnapshot, blockStream: blockStream, + encryptInfo: encryptInfo, } } @@ -2092,7 +2095,7 @@ func (s *SGuestSnapshotDeleteTask) onStreamDiskComplete() { } func (s *SGuestSnapshotDeleteTask) doDiskConvert() error { - return s.disk.ConvertSnapshot(s.convertSnapshot) + return s.disk.ConvertSnapshot(s.convertSnapshot, s.encryptInfo) } func (s *SGuestSnapshotDeleteTask) doReloadDisk(device string) { diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index ef83b70fb8..438d378bec 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -918,7 +918,7 @@ func (s *SKVMGuestInstance) ImportServer(pendingDelete bool) { } else if s.Desc.IsSlave { go s.DirtyServerRequestStart() } else { - s.StartGuest(context.Background(), nil, jsonutils.NewDict()) + s.StartGuest(context.Background(), auth.AdminCredential(), jsonutils.NewDict()) } return } @@ -3182,25 +3182,25 @@ func (s *SKVMGuestInstance) StaticSaveSnapshot( func (s *SKVMGuestInstance) ExecDeleteSnapshotTask( ctx context.Context, disk storageman.IDisk, - deleteSnapshot string, convertSnapshot string, blockStream bool, + deleteSnapshot string, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo, ) (jsonutils.JSONObject, error) { if s.IsRunning() { if s.isLiveSnapshotEnabled() { - task := NewGuestSnapshotDeleteTask(ctx, s, disk, deleteSnapshot, convertSnapshot, blockStream) + task := NewGuestSnapshotDeleteTask(ctx, s, disk, deleteSnapshot, convertSnapshot, blockStream, encryptInfo) task.Start() return nil, nil } else { return nil, fmt.Errorf("Guest dosen't support live snapshot delete") } } else { - return s.deleteStaticSnapshotFile(ctx, disk, deleteSnapshot, convertSnapshot, blockStream) + return s.deleteStaticSnapshotFile(ctx, disk, deleteSnapshot, convertSnapshot, blockStream, encryptInfo) } } func (s *SKVMGuestInstance) deleteStaticSnapshotFile( - ctx context.Context, disk storageman.IDisk, deleteSnapshot, convertSnapshot string, blockStream bool, + ctx context.Context, disk storageman.IDisk, deleteSnapshot, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo, ) (jsonutils.JSONObject, error) { - if err := disk.DeleteSnapshot(deleteSnapshot, convertSnapshot, blockStream); err != nil { + if err := disk.DeleteSnapshot(deleteSnapshot, convertSnapshot, blockStream, encryptInfo); err != nil { log.Errorln(err) return nil, err } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 4cb282e0ab..29a1ddfb10 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -26,10 +26,12 @@ import ( "regexp" "strconv" "strings" + "sync" "syscall" "time" "github.com/vishvananda/netlink" + "golang.org/x/sync/errgroup" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -2175,31 +2177,11 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs, options.HostOptions.AMDVgpuPFs, options.HostOptions.NVIDIAVgpuPFs, enableDevWhitelist) - h.IsolatedDeviceMan.BatchCustomProbe() - // sync each isolated device found - updateDevs := jsonutils.NewArray() - for _, dev := range h.IsolatedDeviceMan.GetDevices() { - dev.SetHostId(h.HostId) - data := isolated_device.GetApiResourceData(dev) - updateDevs.Add(data) + objs, err := h.getRemoteIsolatedDevices() + if err != nil { + return nil, errors.Wrap(err, "getRemoteIsolatedDevices") } - params := jsonutils.NewDict() - params.Set("isolated_devices", updateDevs) - ret, err := modules.Hosts.PerformAction(h.GetSession(), h.HostId, "sync-isolated-devices", params) - if err != nil { - return nil, errors.Wrap(err, "sync isolated devices") - } - devRet, err := ret.Get("isolated_devices") - if err != nil { - return nil, errors.Wrap(err, "sync isolated devices faild get dev rets") - } - devRets, _ := devRet.(*jsonutils.JSONArray) - if devRets.Length() != len(h.IsolatedDeviceMan.GetDevices()) { - return nil, errors.Wrap(err, "sync devices not match") - } - - objs, _ := devRets.GetArray() for _, obj := range objs { info := isolated_device.CloudDeviceInfo{} if err := obj.Unmarshal(&info); err != nil { @@ -2209,11 +2191,39 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { if dev != nil { dev.SetDeviceInfo(info) } else { - return nil, errors.Wrapf(err, "unknown device %s", obj) + // detach device + h.IsolatedDeviceMan.AppendDetachedDevice(&info) } } - return devRets, nil + h.IsolatedDeviceMan.StartDetachTask() + h.IsolatedDeviceMan.BatchCustomProbe() + + // sync each isolated device found + eg := errgroup.Group{} + mtx := sync.Mutex{} + updateDevs := jsonutils.NewArray() + + devs := h.IsolatedDeviceMan.GetDevices() + for i := range devs { + dev := devs[i] + eg.Go(func() error { + if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil { + log.Errorf("Sync deviceInfo %s error: %v", dev.String(), err) + return errors.Wrapf(err, "Sync device %s", dev.String()) + } else { + mtx.Lock() + updateDevs.Add(obj) + mtx.Unlock() + return nil + } + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + return updateDevs, nil } func (h *SHostInfo) deployAdminAuthorizedKeys() { diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index 4ed6714e29..47ff6b83dc 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -92,6 +92,18 @@ func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistM if utils.IsInStringArray(dev.Addr, filteredAddrs) { continue } + if o.HostOptions.BootVgaPciAddr != "" { + if dev.Addr == o.HostOptions.BootVgaPciAddr && !o.HostOptions.UseBootVga { + continue + } + } else { + if ok, err := dev.IsBootVGA(); err != nil { + return nil, err, nil + } else if ok && !o.HostOptions.UseBootVga { + continue + } + } + if !utils.IsInArray(dev.ClassCode, GpuClassCodes) { continue } @@ -119,7 +131,7 @@ func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistM continue } - if err := dev.forceBindVFIOPCIDriver(); err != nil { + if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga, o.HostOptions.BootVgaPciAddr); err != nil { warns = append(warns, errors.Wrapf(err, "force bind vfio-pci driver %s", dev.Addr)) continue } @@ -217,7 +229,7 @@ func NewPCIDevice(addr string, executors ...IExecutor) (*PCIDevice, error) { if err := dev.checkSameIOMMUGroupDevice(); err != nil { return nil, err } - if err := dev.forceBindVFIOPCIDriver(); err != nil { + if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga, o.HostOptions.BootVgaPciAddr); err != nil { return nil, fmt.Errorf("Force bind vfio-pci driver: %v", err) } return dev, nil @@ -398,10 +410,32 @@ func (d *PCIDevice) IsBootVGA() (bool, error) { return false, nil } -func (d *PCIDevice) forceBindVFIOPCIDriver() error { - if !utils.IsInArray(d.ClassCode, GpuClassCodes) { +func (d *PCIDevice) forceBindVFIOPCIDriver(useBootVGA bool, bootVgaPciAddr string) error { + if !utils.IsInStringArray(d.ClassCode, []string{CLASS_CODE_VGA, CLASS_CODE_3D}) { return nil } + + if !useBootVGA && bootVgaPciAddr != "" { + if d.Addr == bootVgaPciAddr { + log.Infof("device %#v is specific boot vga addr, skip it", d) + return nil + } + } else { + isBootVGA, err := d.IsBootVGA() + if err != nil { + return err + } + if !useBootVGA && isBootVGA { + log.Infof("%#v is boot vga card, skip it", d) + return nil + } + } + + if d.IsVFIOPCIDriverUsed() { + log.Infof("%s already use vfio-pci driver", d) + return nil + } + devs := []*PCIDevice{} devs = append(devs, d.RestIOMMUGroupDevs...) devs = append(devs, d) diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index f213911728..81002786ed 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -132,9 +132,10 @@ type SHostOptions struct { SetVncPassword bool `default:"true" help:"Auto set vnc password after monitor connected"` UseBootVga bool `default:"false" help:"Use boot VGA GPU for guest"` - EnableHostAgentNumaAllocate bool `default:"false" help:"Enable host agent numa allocate"` - EnableCpuBinding bool `default:"true" help:"Enable cpu binding and rebalance"` - EnableOpenflowController bool `default:"false"` + EnableHostAgentNumaAllocate bool `default:"false" help:"Enable host agent numa allocate"` + EnableCpuBinding bool `default:"true" help:"Enable cpu binding and rebalance"` + EnableOpenflowController bool `default:"false"` + BootVgaPciAddr string `help:"Specific boot vga pci addr incase detect wrong device"` PingRegionInterval int `default:"60" help:"interval to ping region, deefault is 1 minute"` LogSystemdUnits []string `help:"Systemd units log collected by fluent-bit"` diff --git a/pkg/hostman/storageman/disk_base.go b/pkg/hostman/storageman/disk_base.go index 2eedca879e..38372805c3 100644 --- a/pkg/hostman/storageman/disk_base.go +++ b/pkg/hostman/storageman/disk_base.go @@ -72,10 +72,10 @@ type IDisk interface { encryptInfo *apis.SEncryptInfo, diskId string, back string) (jsonutils.JSONObject, error) PostCreateFromImageFuse() CreateSnapshot(snapshotId string, encryptKey string, encFormat qemuimg.TEncryptFormat, encAlg seclib2.TSymEncAlg) error - DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool) error + DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error DeployGuestFs(diskInfo *deployapi.DiskInfo, guestDesc *desc.SGuestDesc, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error) - ConvertSnapshot(convertSnapshotId string) error + ConvertSnapshot(convertSnapshotId string, encryptInfo apis.SEncryptInfo) error // GetBackupDir() string DiskBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) @@ -145,11 +145,11 @@ func (d *SBaseDisk) CreateSnapshot(snapshotId string, encryptKey string, encForm return errors.Errorf("unsupported operation") } -func (d *SBaseDisk) ConvertSnapshot(convertSnapshotId string) error { +func (d *SBaseDisk) ConvertSnapshot(convertSnapshotId string, encryptInfo apis.SEncryptInfo) error { return errors.Errorf("unsupported operation") } -func (d *SBaseDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool) error { +func (d *SBaseDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error { return errors.Errorf("unsupported operation") } diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index 002dd6b895..3582d19ba0 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -459,7 +459,7 @@ func (d *SLocalDisk) CreateSnapshot(snapshotId string, encryptKey string, encFor return nil } -func (d *SLocalDisk) ConvertSnapshot(convertSnapshotId string) error { +func (d *SLocalDisk) ConvertSnapshot(convertSnapshotId string, encryptInfo apis.SEncryptInfo) error { snapshotDir := d.GetSnapshotDir() snapshotPath := path.Join(snapshotDir, convertSnapshotId) img, err := qemuimg.NewQemuImage(snapshotPath) @@ -482,7 +482,7 @@ func (d *SLocalDisk) ConvertSnapshot(convertSnapshotId string) error { return nil } -func (d *SLocalDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool) error { +func (d *SLocalDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error { snapshotDir := d.GetSnapshotDir() return DeleteLocalSnapshot(snapshotDir, snapshotId, d.getPath(), convertSnapshot, blockStream) } @@ -613,9 +613,15 @@ func (d *SLocalDisk) CleanupSnapshots(ctx context.Context, params interface{}) ( for _, snapshotId := range cleanupParams.DeleteSnapshots { snapId, _ := snapshotId.GetString() - if err := procutils.NewCommand("rm", "-f", path.Join(snapshotDir, snapId)).Run(); err != nil { - log.Errorln(err) - return nil, err + snapPath := path.Join(snapshotDir, snapId) + if options.HostOptions.RecycleDiskfile { + return nil, d.Storage.DeleteDiskfile(snapPath, false) + } else { + log.Infof("Delete disk(%s) snapshot %s", d.Id, snapId) + if err := procutils.NewCommand("rm", "-f", snapPath).Run(); err != nil { + log.Errorln(err) + return nil, err + } } } return nil, nil diff --git a/pkg/hostman/storageman/disk_lvm.go b/pkg/hostman/storageman/disk_lvm.go index 061c24ffcd..8c59fe71fa 100644 --- a/pkg/hostman/storageman/disk_lvm.go +++ b/pkg/hostman/storageman/disk_lvm.go @@ -387,23 +387,12 @@ func (d *SLVMDisk) PrepareSaveToGlance(ctx context.Context, params interface{}) } backupPath := path.Join(destDir, fmt.Sprintf("%s.%s", d.Id, appctx.AppContextTaskId(ctx))) - srcInfo := qemuimg.SImageInfo{ - Path: d.GetPath(), - Format: qemuImg.Format, - IoLevel: qemuimg.IONiceNone, - Password: "", - } - destInfo := qemuimg.SImageInfo{ - Path: backupPath, - Format: qemuimgfmt.QCOW2, - IoLevel: qemuimg.IONiceNone, - Password: "", - } - if err = qemuimg.Convert(srcInfo, destInfo, true, nil); err != nil { + if err := procutils.NewCommand("cp", "--sparse=always", "-f", d.GetPath(), backupPath).Run(); err != nil { log.Errorln(err) procutils.NewCommand("rm", "-f", backupPath).Run() return nil, err } + res := jsonutils.NewDict() res.Set("backup", jsonutils.NewString(backupPath)) return res, nil @@ -595,13 +584,13 @@ func (d *SLVMDisk) ResetFromSnapshot(ctx context.Context, params interface{}) (j return nil, nil } -func (d *SLVMDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool) error { +func (d *SLVMDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error { if blockStream { - if err := ConvertLVMDisk(d.Storage.GetPath(), d.Id); err != nil { + if err := ConvertLVMDisk(d.Storage.GetPath(), d.Id, encryptInfo); err != nil { return err } } else if len(convertSnapshot) > 0 { - if err := d.ConvertSnapshot(convertSnapshot); err != nil { + if err := d.ConvertSnapshot(convertSnapshot, encryptInfo); err != nil { return err } } @@ -626,9 +615,9 @@ func (d *SLVMDisk) DeleteAllSnapshot(skipRecycle bool) error { return nil } -func (d *SLVMDisk) ConvertSnapshot(convertSnapshot string) error { +func (d *SLVMDisk) ConvertSnapshot(convertSnapshot string, encryptInfo apis.SEncryptInfo) error { convertSnapshotName := d.GetSnapshotName(convertSnapshot) - return ConvertLVMDisk(d.Storage.GetPath(), convertSnapshotName) + return ConvertLVMDisk(d.Storage.GetPath(), convertSnapshotName, encryptInfo) } func (d *SLVMDisk) DoDeleteSnapshot(snapshotId string) error { diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index 0c386fdf71..1a324b2870 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -256,11 +256,11 @@ func (d *SRBDDisk) CreateSnapshot(snapshotId string, encryptKey string, encForma return storage.createSnapshot(d.Id, snapshotId) } -func (d *SRBDDisk) ConvertSnapshot(convertSnapshotId string) error { +func (d *SRBDDisk) ConvertSnapshot(convertSnapshotId string, encryptInfo apis.SEncryptInfo) error { return nil } -func (d *SRBDDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool) error { +func (d *SRBDDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error { storage := d.Storage.(*SRbdStorage) return storage.deleteSnapshot(d.Id, snapshotId) } @@ -278,7 +278,7 @@ func (d *SRBDDisk) DiskDeleteSnapshot(ctx context.Context, params interface{}) ( if !ok { return nil, hostutils.ParamsError } - err := d.DeleteSnapshot(snapshotId, "", false) + err := d.DeleteSnapshot(snapshotId, "", false, apis.SEncryptInfo{}) if err != nil { return nil, err } else { diff --git a/pkg/hostman/storageman/disk_slvm.go b/pkg/hostman/storageman/disk_slvm.go index e484206603..e3a4e7840a 100644 --- a/pkg/hostman/storageman/disk_slvm.go +++ b/pkg/hostman/storageman/disk_slvm.go @@ -243,6 +243,46 @@ func (d *SSLVMDisk) ResetFromSnapshot(ctx context.Context, params interface{}) ( return ret, nil } +func (d *SSLVMDisk) GetDiskDesc() jsonutils.JSONObject { + active, err := lvmutils.LvIsActivated(d.GetPath()) + if err != nil { + log.Errorf("failed check active of %s: %s", d.GetPath(), err) + return nil + } + if !active { + if err := lvmutils.LVActive(d.GetPath(), d.Storage.Lvmlockd(), false); err != nil { + log.Errorf("failed active lv %s: %s", d.GetPath(), err) + return nil + } + } + res := d.SLVMDisk.GetDiskDesc() + if !active { + if err := lvmutils.LVDeactivate(d.GetPath()); err != nil { + log.Errorf("failed deactivate lv %s: %s", d.GetPath(), err) + } + } + return res +} + +func (d *SSLVMDisk) PrepareSaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { + active, err := lvmutils.LvIsActivated(d.GetPath()) + if err != nil { + return nil, errors.Wrap(err, "LvIsActivated") + } + if !active { + if err := lvmutils.LVActive(d.GetPath(), d.Storage.Lvmlockd(), false); err != nil { + return nil, errors.Wrap(err, "LVActive") + } + } + res, e := d.SLVMDisk.PrepareSaveToGlance(ctx, params) + if !active { + if err := lvmutils.LVDeactivate(d.GetPath()); err != nil { + log.Errorf("failed deactivate lv %s: %s", d.GetPath(), err) + } + } + return res, e +} + func (d *SSLVMDisk) CreateFromSnapshotLocation(ctx context.Context, snapshotLocation string, size int64, encryptInfo *apis.SEncryptInfo) (jsonutils.JSONObject, error) { ret, err := d.SLVMDisk.CreateRaw(ctx, int(size), "", "", encryptInfo, d.Id, snapshotLocation) if err != nil { @@ -261,3 +301,27 @@ func (d *SSLVMDisk) CreateFromSnapshotLocation(ctx context.Context, snapshotLoca } return ret, nil } + +func (d *SSLVMDisk) DeleteSnapshot(snapshotId, convertSnapshot string, blockStream bool, encryptInfo apis.SEncryptInfo) error { + err := lvmutils.LVActive(d.GetPath(), false, d.Storage.Lvmlockd()) + if err != nil { + return errors.Wrap(err, "LVActive") + } + convertSnapshotPath := d.GetSnapshotPath(convertSnapshot) + err = lvmutils.LVActive(convertSnapshotPath, false, d.Storage.Lvmlockd()) + if err != nil { + return errors.Wrap(err, "LVActive convert snapshot") + } + + err = d.SLVMDisk.DeleteSnapshot(snapshotId, convertSnapshot, blockStream, encryptInfo) + // active disk share mode + e := lvmutils.LVActive(d.GetPath(), d.Storage.Lvmlockd(), false) + if e != nil { + log.Errorf("failed active with share mode: %s", e) + } + e = lvmutils.LVActive(convertSnapshotPath, d.Storage.Lvmlockd(), false) + if e != nil { + log.Errorf("failed active convert snapshot %s with share mode: %s", convertSnapshotPath, e) + } + return err +} diff --git a/pkg/hostman/storageman/storage_lvm.go b/pkg/hostman/storageman/storage_lvm.go index 3a12179ff8..bb9b172589 100644 --- a/pkg/hostman/storageman/storage_lvm.go +++ b/pkg/hostman/storageman/storage_lvm.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient" @@ -38,10 +39,13 @@ import ( "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + identity_modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity" "yunion.io/x/onecloud/pkg/mcclient/modules/image" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/seclib2" ) type SLVMStorage struct { @@ -212,12 +216,12 @@ func (s *SLVMStorage) DeleteSnapshot(ctx context.Context, params interface{}) (j return nil, hostutils.ParamsError } if input.BlockStream { - if err := ConvertLVMDisk(s.GetPath(), input.DiskId); err != nil { + if err := ConvertLVMDisk(s.GetPath(), input.DiskId, input.EncryptInfo); err != nil { return nil, err } } else if len(input.ConvertSnapshot) > 0 { convertSnapshotName := "snap_" + input.ConvertSnapshot - if err := ConvertLVMDisk(s.GetPath(), convertSnapshotName); err != nil { + if err := ConvertLVMDisk(s.GetPath(), convertSnapshotName, input.EncryptInfo); err != nil { return nil, err } } @@ -275,9 +279,28 @@ func (s *SLVMStorage) SaveToGlance(ctx context.Context, input interface{}) (json imageId, _ = data.GetString("image_id") imagePath, _ = data.GetString("image_path") compress = jsonutils.QueryBoolean(data, "compress", true) + encKeyId, _ = data.GetString("encrypt_key_id") err error ) - if err = s.saveToGlance(ctx, imageId, imagePath, compress); err != nil { + + var ( + encKey string + encFormat qemuimg.TEncryptFormat + encAlg seclib2.TSymEncAlg + ) + + if len(encKeyId) > 0 { + session := auth.GetSession(ctx, info.UserCred, consts.GetRegion()) + key, err := identity_modules.Credentials.GetEncryptKey(session, encKeyId) + if err != nil { + return nil, errors.Wrap(err, "GetEncryptKey") + } + encKey = key.Key + encFormat = qemuimg.EncryptFormatLuks + encAlg = key.Alg + } + + if err = s.saveToGlance(ctx, imageId, imagePath, compress, encKey, encFormat, encAlg); err != nil { log.Errorf("Save to glance failed: %s", err) s.onSaveToGlanceFailed(ctx, imageId, err.Error()) } @@ -300,11 +323,19 @@ func (s *SLVMStorage) SaveToGlance(ctx context.Context, input interface{}) (json return nil, nil } -func (s *SLVMStorage) saveToGlance(ctx context.Context, imageId, imagePath string, compress bool) error { +func (s *SLVMStorage) saveToGlance( + ctx context.Context, imageId, imagePath string, compress bool, + encryptKey string, encFormat qemuimg.TEncryptFormat, encAlg seclib2.TSymEncAlg, +) error { log.Infof("saveToGlance %s", imagePath) diskInfo := &deployapi.DiskInfo{ Path: imagePath, } + if len(encryptKey) > 0 { + diskInfo.EncryptPassword = encryptKey + diskInfo.EncryptFormat = string(encFormat) + diskInfo.EncryptAlg = string(encAlg) + } ret, err := deployclient.GetDeployClient().SaveToGlance(ctx, &deployapi.SaveToGlanceParams{DiskInfo: diskInfo, Compress: compress}) if err != nil { @@ -619,7 +650,7 @@ func (d *SLVMStorage) GetDisksPath() ([]string, error) { return disksPath, nil } -func ConvertLVMDisk(vgName, lvName string) error { +func ConvertLVMDisk(vgName, lvName string, encryptInfo apis.SEncryptInfo) error { diskPath := path.Join("/dev", vgName, lvName) qemuImg, err := qemuimg.NewQemuImage(diskPath) if err != nil { @@ -638,16 +669,22 @@ func ConvertLVMDisk(vgName, lvName string) error { return errors.Wrap(err, "delete snapshot LvCreate") } srcInfo := qemuimg.SImageInfo{ - Path: diskPath, - Format: qemuImg.Format, - IoLevel: qemuimg.IONiceNone, - Password: "", + Path: diskPath, + Format: qemuImg.Format, + IoLevel: qemuimg.IONiceNone, + + Password: encryptInfo.Key, + EncryptAlg: encryptInfo.Alg, + EncryptFormat: qemuimg.EncryptFormatLuks, } destInfo := qemuimg.SImageInfo{ - Path: tmpVolumePath, - Format: qemuimgfmt.QCOW2, - IoLevel: qemuimg.IONiceNone, - Password: "", + Path: tmpVolumePath, + Format: qemuimgfmt.QCOW2, + IoLevel: qemuimg.IONiceNone, + + Password: encryptInfo.Key, + EncryptAlg: encryptInfo.Alg, + EncryptFormat: qemuimg.EncryptFormatLuks, } // convert /dev/vg/disk to /dev/vg/disk-convert.tmp if err = qemuimg.Convert(srcInfo, destInfo, false, nil); err != nil { diff --git a/pkg/hostman/storageman/storage_slvm.go b/pkg/hostman/storageman/storage_slvm.go index e6709b7347..cd1faa8185 100644 --- a/pkg/hostman/storageman/storage_slvm.go +++ b/pkg/hostman/storageman/storage_slvm.go @@ -121,7 +121,7 @@ func (s *SSLVMStorage) DeleteSnapshot(ctx context.Context, params interface{}) ( return nil, errors.Wrap(err, "lvactive exclusive") } - err = ConvertLVMDisk(s.GetPath(), input.DiskId) + err = ConvertLVMDisk(s.GetPath(), input.DiskId, input.EncryptInfo) if err != nil { return nil, err } @@ -134,7 +134,7 @@ func (s *SSLVMStorage) DeleteSnapshot(ctx context.Context, params interface{}) ( return nil, errors.Wrap(err, "lvactive exclusive") } - if err := ConvertLVMDisk(s.GetPath(), convertSnapshotName); err != nil { + if err := ConvertLVMDisk(s.GetPath(), convertSnapshotName, input.EncryptInfo); err != nil { return nil, err } } diff --git a/pkg/hostman/storageman/storagehandler/storagehandler.go b/pkg/hostman/storageman/storagehandler/storagehandler.go index 1bc4a02ddc..70b0ba0705 100644 --- a/pkg/hostman/storageman/storagehandler/storagehandler.go +++ b/pkg/hostman/storageman/storagehandler/storagehandler.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -466,6 +467,15 @@ func storageDeleteSnapshot(ctx context.Context, w http.ResponseWriter, r *http.R SnapshotId: snapshotId, } + if body.Contains("encrypt_info") { + encryptInfo := apis.SEncryptInfo{} + if err = body.Unmarshal(&encryptInfo, "encrypt_info"); err != nil { + hostutils.Response(ctx, w, httperrors.NewInputParameterError("unmarshal encrypt_info failed %s", err)) + return + } + input.EncryptInfo = encryptInfo + } + if !blockStream && !autoDeleted { convertSnapshot, err := body.GetString("convert_snapshot") if err != nil { diff --git a/pkg/hostman/storageman/storagehelper.go b/pkg/hostman/storageman/storagehelper.go index ee7a056489..5a0dddf5ae 100644 --- a/pkg/hostman/storageman/storagehelper.go +++ b/pkg/hostman/storageman/storagehelper.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -71,6 +72,7 @@ type SStorageDeleteSnapshot struct { SnapshotId string ConvertSnapshot string BlockStream bool + EncryptInfo apis.SEncryptInfo } type SDiskBackup struct { diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index fa25cc8691..182b88b060 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -372,10 +372,13 @@ func convertEncrypt(srcInfo, destInfo SImageInfo, compact bool, workerOpions []s if err != nil { return errors.Wrapf(err, "NewQemuImage dest %s", destInfo.Path) } - err = target.CreateQcow2(source.GetSizeMB(), compact, "", destInfo.Password, destInfo.EncryptFormat, destInfo.EncryptAlg) - if err != nil { - return errors.Wrapf(err, "Create target image %s", destInfo.Path) + if target.Format != qemuimgfmt.QCOW2 { + err = target.CreateQcow2(source.GetSizeMB(), compact, "", destInfo.Password, destInfo.EncryptFormat, destInfo.EncryptAlg) + if err != nil { + return errors.Wrapf(err, "Create target image %s", destInfo.Path) + } } + cmdline := []string{"-c", strconv.Itoa(int(srcInfo.IoLevel)), qemutils.GetQemuImg(), "convert"} if compact { cmdline = append(cmdline, "-c")