From 913aec9f7f0923cd3544e7f4c3dc3519e0cdbee5 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Mon, 16 May 2022 00:34:37 +0800 Subject: [PATCH] feat(host): create disk from encrypted snapshot (#14256) support create disk from encrypted snapshot, new disk encrypt info must same with snapshot, and should used with libqemuio2.12. area: host,host-image,fetcherfs Signed-off-by: wanyaoqi --- cmd/fetcherfs/fs.go | 8 ++- cmd/fetcherfs/options.go | 4 ++ pkg/hostimage/host_image_service.go | 59 ++++++++++++++++---- pkg/hostimage/image.go | 82 +++++++++++++++++++++++----- pkg/hostman/storageman/disk_local.go | 16 ++++-- pkg/util/fuseutils/fuseutils.go | 14 ++++- pkg/util/qemuimg/qemuimg.go | 10 +++- 7 files changed, 158 insertions(+), 35 deletions(-) diff --git a/cmd/fetcherfs/fs.go b/cmd/fetcherfs/fs.go index 4da73fe93a..e2b9a8f9ba 100644 --- a/cmd/fetcherfs/fs.go +++ b/cmd/fetcherfs/fs.go @@ -63,7 +63,7 @@ func initFetcherFs() (*FetcherFs, error) { url: opt.Url, } if err := fetcherFs.fetchMetaInfo(); err != nil { - return nil, err + return nil, errors.Wrapf(err, "fetch meta info") } segs := strings.Split(opt.Url, "/") if len(segs[len(segs)-1]) == 0 { @@ -271,5 +271,11 @@ func (fs *FetcherFs) destory() error { func NewRequestHeader() http.Header { header := http.Header{} header.Set("X-Auth-Token", opt.Token) + if len(opt.EncryptKey) > 0 { + header.Set("X-Encrypt-Key", opt.EncryptKey) + } + if len(opt.EncryptAlg) > 0 { + header.Set("X-Encrypt-Alg", opt.EncryptAlg) + } return header } diff --git a/cmd/fetcherfs/options.go b/cmd/fetcherfs/options.go index 4a3a113f1b..73e126d97b 100644 --- a/cmd/fetcherfs/options.go +++ b/cmd/fetcherfs/options.go @@ -31,6 +31,10 @@ type Options struct { MountPoint string `help:"mount path of fuse fs" required:"true"` Debug bool `help:"enable debug go fuse"` Foreground bool `help:"run in foreground"` + + // image encrypt info + EncryptAlg string `help:"image encrypt alg"` + EncryptKey string `help:"image encrypt password"` } var opt = &Options{} diff --git a/pkg/hostimage/host_image_service.go b/pkg/hostimage/host_image_service.go index 2af3b44a61..ebb8c5f6e7 100644 --- a/pkg/hostimage/host_image_service.go +++ b/pkg/hostimage/host_image_service.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" @@ -35,6 +36,7 @@ import ( common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/seclib2" ) type SHostImageOptions struct { @@ -162,7 +164,7 @@ func closeImage(ctx context.Context, w http.ResponseWriter, r *http.Request) { } else { f = &SQcow2Image{} } - err = f.Load(imagePath, true) + err = f.Load(imagePath, true, false) if err != nil { httperrors.GeneralServerError(ctx, w, err) return @@ -178,20 +180,35 @@ func getImage(ctx context.Context, w http.ResponseWriter, r *http.Request) { return } - var f IImage - var startPos, endPos int64 - var rateLimit int64 = -1 + var ( + f IImage + startPos, endPos int64 + rateLimit int64 = -1 + encryptInfo *apis.SEncryptInfo + ) if r.Header.Get("X-Read-File") == "true" { f = &SFile{} } else { f = &SQcow2Image{} } - if err = f.Open(imagePath, true); err != nil { - log.Errorf("Open image error: %s", err) - httperrors.GeneralServerError(ctx, w, err) - return + + err = f.Load(imagePath, true, true) + if err != nil { + encryptKey := r.Header.Get("X-Encrypt-Key") + if len(encryptKey) > 0 { + encryptInfo = new(apis.SEncryptInfo) + encryptInfo.Key = encryptKey + encryptInfo.Alg = seclib2.TSymEncAlg(r.Header.Get("X-Encrypt-Alg")) + } + + if err = f.Open(imagePath, true, encryptInfo); err != nil { + log.Errorf("Open image error: %s", err) + httperrors.GeneralServerError(ctx, w, err) + return + } } + defer f.Close() endPos = f.Length() - 1 @@ -293,16 +310,34 @@ func getImageMeta(ctx context.Context, w http.ResponseWriter, r *http.Request) { return } - var f IImage + var ( + f IImage + encryptInfo *apis.SEncryptInfo + ) + if r.Header.Get("X-Read-File") == "true" { f = &SFile{} } else { f = &SQcow2Image{} } - if err = f.Open(imagePath, true); err != nil { - httperrors.GeneralServerError(ctx, w, err) - return + + log.Infof("open image %s", imagePath) + err = f.Load(imagePath, true, true) + if err != nil { + encryptKey := r.Header.Get("X-Encrypt-Key") + if len(encryptKey) > 0 { + encryptInfo = new(apis.SEncryptInfo) + encryptInfo.Key = encryptKey + encryptInfo.Alg = seclib2.TSymEncAlg(r.Header.Get("X-Encrypt-Alg")) + } + + if err = f.Open(imagePath, true, encryptInfo); err != nil { + log.Errorf("Open image error: %s", err) + httperrors.GeneralServerError(ctx, w, err) + return + } } + defer f.Close() w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Length())) w.Header().Set("Content-Type", "application/octet-stream") diff --git a/pkg/hostimage/image.go b/pkg/hostimage/image.go index a5444cb30c..0857f757e0 100644 --- a/pkg/hostimage/image.go +++ b/pkg/hostimage/image.go @@ -27,12 +27,16 @@ import ( "fmt" "io" "os" + "path/filepath" "sync" "sync/atomic" "unsafe" "yunion.io/x/log" "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/util/qemuimg" ) var qemuBlkCache sync.Map @@ -40,6 +44,7 @@ var qemuBlkCache sync.Map type QemuioBlkDev struct { imagePath string readonly bool + encrypted bool refCount int32 blk *C.struct_QemuioBlk @@ -63,20 +68,44 @@ func (qb *QemuioBlkDev) ReadQcow2(offset int64, count int64) ([]byte, int64) { } } -func OpenQcow2(imagePath string, readonly bool) *QemuioBlkDev { - key := fmt.Sprintf("%s_%v", imagePath, readonly) +func OpenQcow2(disk *qemuimg.SImageInfo, readonly bool) *QemuioBlkDev { + key := fmt.Sprintf("%s_%v", disk.Path, readonly) if blk, ok := qemuBlkCache.Load(key); ok { qb := blk.(*QemuioBlkDev) atomic.AddInt32(&qb.refCount, 1) return qb } + qb := &QemuioBlkDev{ - imagePath: imagePath, + imagePath: disk.Path, readonly: readonly, } - cImagePath := C.CString(imagePath) - blk := C.open_qcow2(cImagePath, C.bool(readonly)) - C.free(unsafe.Pointer(cImagePath)) + diskPath := C.CString(disk.Path) + imageOpts := C.CString(disk.ImageOptions()) + + // sec options + var secretOpts *C.char + secOpt := disk.SecretOptions() + if len(secOpt) > 0 { + secretOpts = C.CString(secOpt) + qb.encrypted = true + } + + // qemu io open image + blk := C.open_qcow2(diskPath, imageOpts, secretOpts, C.bool(readonly)) + + C.free(unsafe.Pointer(diskPath)) + C.free(unsafe.Pointer(imageOpts)) + + if secretOpts != nil { + C.free(unsafe.Pointer(secretOpts)) + } + + if blk == nil { + // failed open qemu image + return nil + } + qb.blk = blk qb.refCount = 1 qemuBlkCache.Store(key, qb) @@ -101,16 +130,26 @@ func (qb *QemuioBlkDev) Qcow2GetLength() int64 { func (qb *QemuioBlkDev) CloseQcow2() { if qb.blk != nil && atomic.AddInt32(&qb.refCount, -1) == 0 { - C.close_qcow2(qb.blk) + var secId *C.char + if qb.encrypted { + secId = C.CString(filepath.Base(qb.imagePath)) + } + + C.close_qcow2(qb.blk, secId) + qemuBlkCache.Delete(fmt.Sprintf("%s_%v", qb.imagePath, qb.readonly)) + + if secId != nil { + C.free(unsafe.Pointer(secId)) + } } } type IImage interface { // Open image file and its backing file (if have) - Open(imagePath string, readonly bool) error + Open(imagePath string, readonly bool, encryptInfo *apis.SEncryptInfo) error // load opend qcow2 img form qemu blk cache - Load(imagePath string, readonly bool) error + Load(imagePath string, readonly, reference bool) error // Close may not really close image file handle, just reudce ref count Close() @@ -126,8 +165,22 @@ type SQcow2Image struct { fd *QemuioBlkDev } -func (img *SQcow2Image) Open(imagePath string, readonly bool) error { - fd := OpenQcow2(imagePath, readonly) +func (img *SQcow2Image) newQemuImage(imagePath string, encryptInfo *apis.SEncryptInfo) *qemuimg.SImageInfo { + info := &qemuimg.SImageInfo{ + Path: imagePath, + } + + if encryptInfo != nil { + info.SetSecId(filepath.Base(imagePath)) + info.Password = encryptInfo.Key + info.EncryptAlg = encryptInfo.Alg + } + return info +} + +func (img *SQcow2Image) Open(imagePath string, readonly bool, encryptInfo *apis.SEncryptInfo) error { + disk := img.newQemuImage(imagePath, encryptInfo) + fd := OpenQcow2(disk, readonly) if fd == nil { return fmt.Errorf("open image %s failed", imagePath) } else { @@ -136,11 +189,12 @@ func (img *SQcow2Image) Open(imagePath string, readonly bool) error { } } -func (img *SQcow2Image) Load(imagePath string, readonly bool) error { +func (img *SQcow2Image) Load(imagePath string, readonly, reference bool) error { fd := LoadQcow2(imagePath, readonly) if fd == nil { return fmt.Errorf("image %s readonly: %v not found", imagePath, readonly) } else { + atomic.AddInt32(&fd.refCount, 1) img.fd = fd return nil } @@ -167,7 +221,7 @@ type SFile struct { fd *os.File } -func (f *SFile) Open(imagePath string, readonly bool) error { +func (f *SFile) Open(imagePath string, readonly bool, encryptInfo *apis.SEncryptInfo) error { var mode = os.O_RDWR if readonly { mode = os.O_RDONLY @@ -181,7 +235,7 @@ func (f *SFile) Open(imagePath string, readonly bool) error { } } -func (f *SFile) Load(imagePath string, readonly bool) error { +func (f *SFile) Load(imagePath string, readonly, reference bool) error { return fmt.Errorf("File don't support load") } diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index c07cf42006..b95d745d1d 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -209,16 +209,22 @@ func (d *SLocalDisk) CreateFromImageFuse(ctx context.Context, url string, size i } } if !newImg.IsValid() || newImg.IsChained() { - if err := fuseutils.MountFusefs(options.HostOptions.FetcherfsPath, url, localPath, - auth.GetTokenString(), mntPath, options.HostOptions.FetcherfsBlockSize); err != nil { + if err := fuseutils.MountFusefs( + options.HostOptions.FetcherfsPath, url, localPath, + auth.GetTokenString(), mntPath, options.HostOptions.FetcherfsBlockSize, encryptInfo, + ); err != nil { log.Errorln(err) return err } } if !newImg.IsValid() { - if err := newImg.CreateQcow2(0, false, contentPath, "", "", ""); err != nil { - log.Errorln(err) - return err + if encryptInfo != nil { + err = newImg.CreateQcow2(0, false, contentPath, encryptInfo.Key, qemuimg.EncryptFormatLuks, encryptInfo.Alg) + } else { + err = newImg.CreateQcow2(0, false, contentPath, "", "", "") + } + if err != nil { + return errors.Wrapf(err, "create from fuse") } } diff --git a/pkg/util/fuseutils/fuseutils.go b/pkg/util/fuseutils/fuseutils.go index cb0abf7867..8c8bba0f97 100644 --- a/pkg/util/fuseutils/fuseutils.go +++ b/pkg/util/fuseutils/fuseutils.go @@ -24,11 +24,15 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" ) -func MountFusefs(fetcherfsPath, url, tmpdir, token, mntpath string, blocksize int) error { +func MountFusefs( + fetcherfsPath, url, tmpdir, token, mntpath string, + blocksize int, encryptInfo *apis.SEncryptInfo, +) error { var metaPath = path.Join(mntpath, "meta") if f, err := os.OpenFile(metaPath, os.O_RDONLY, 0644); err == nil { f.Close() @@ -64,6 +68,14 @@ func MountFusefs(fetcherfsPath, url, tmpdir, token, mntpath string, blocksize in "--blocksize", strconv.Itoa(blocksize), "--mount-point", mntpath, } + if encryptInfo != nil && len(encryptInfo.Key) > 0 { + // TODO: base64 encrypt key + cmd = append(cmd, "--encrypt-key", encryptInfo.Key) + } + if encryptInfo != nil && len(encryptInfo.Alg) > 0 { + cmd = append(cmd, "--encrypt-alg", string(encryptInfo.Alg)) + } + log.Infof("%s", strings.Join(cmd, " ")) out, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output() if err != nil { diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 4bfb66423d..92e71ebd43 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -211,6 +211,10 @@ type SImageInfo struct { secId string } +func (info *SImageInfo) SetSecId(id string) { + info.secId = id +} + func (info SImageInfo) ImageOptions() string { opts := make([]string, 0) format := info.Format @@ -238,8 +242,8 @@ func (info SImageInfo) ImageOptions() string { } func (info SImageInfo) SecretOptions() string { - opts := make([]string, 0) if info.Encrypted() { + opts := make([]string, 0) secId := info.secId if len(secId) == 0 { secId = "sec0" @@ -248,8 +252,10 @@ func (info SImageInfo) SecretOptions() string { opts = append(opts, fmt.Sprintf("id=%s", secId)) opts = append(opts, fmt.Sprintf("data=%s", info.Password)) opts = append(opts, "format=base64") + + return strings.Join(opts, ",") } - return strings.Join(opts, ",") + return "" } func (info SImageInfo) Encrypted() bool {