From bc83e2940af003eaa87c8ef1d6595c9744248789 Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 16 Jan 2020 21:00:15 +0800 Subject: [PATCH 1/2] bugfix(esxiagent): PrefetchImageCacheByCopy & SHost.FileUrlPathToDsPath 1. Fix the bug when preparing remote image by copy 2. SHost.FileUrlPathToDsPath will not work when shost has multiple storage. --- .../storageman/imagecachemanager_agent.go | 15 ++++++----- pkg/multicloud/esxi/host.go | 10 +++++--- pkg/multicloud/esxi/manager.go | 4 +-- pkg/multicloud/esxi/shell/store.go | 5 ++-- pkg/multicloud/esxi/storage.go | 25 +++++++++---------- pkg/multicloud/esxi/vdisk.go | 2 +- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/pkg/hostman/storageman/imagecachemanager_agent.go b/pkg/hostman/storageman/imagecachemanager_agent.go index 91c679d00d..4f861638ee 100644 --- a/pkg/hostman/storageman/imagecachemanager_agent.go +++ b/pkg/hostman/storageman/imagecachemanager_agent.go @@ -18,12 +18,14 @@ import ( "context" "fmt" "path/filepath" + "reflect" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/multicloud/esxi" @@ -96,20 +98,20 @@ func (c *SAgentImageCacheManager) prefetchImageCacheByCopy(ctx context.Context, exists := false log.Infof("check file: src=%s, dst=%s", srcPath, dstPath) dstVmdkInfo, err := dstDs.GetVmdkInfo(ctx, dstPath) - if err != nil { + if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound { return nil, err } srcVmdkInfo, err := srcDs.GetVmdkInfo(ctx, srcPath) if err != nil { return nil, err } - if dstVmdkInfo == srcVmdkInfo { + if dstVmdkInfo != nil && reflect.DeepEqual(dstVmdkInfo, srcVmdkInfo) { exists = true } dstUrl := dstDs.GetPathUrl(dstPath) if !exists || data.IsForce { - _, err = dstDs.MakeDir(ctx, dstPath) + err = dstDs.CheckDirC(filepath.Dir(dstPath)) if err != nil { return nil, errors.Wrap(err, "dstDs.MakeDir") } @@ -124,11 +126,12 @@ func (c *SAgentImageCacheManager) prefetchImageCacheByCopy(ctx context.Context, return nil, errors.Wrap(err, "dstDs.GetVmdkInfo") } } + dstPath = dstDs.GetFullPath(dstPath) ret := jsonutils.NewDict() ret.Add(jsonutils.NewInt(dstVmdkInfo.Size()), "size") - ret.Add(jsonutils.NewString(dstUrl), "path") + ret.Add(jsonutils.NewString(dstPath), "path") ret.Add(jsonutils.NewString(data.ImageId), "image_id") - _, err = hostutils.RemoteStoragecacheCacheImage(ctx, data.StoragecacheId, data.ImageId, "ready", dstUrl) + _, err = hostutils.RemoteStoragecacheCacheImage(ctx, data.StoragecacheId, data.ImageId, "ready", dstPath) if err != nil { return nil, err } @@ -189,7 +192,7 @@ func (c *SAgentImageCacheManager) prefetchImageCacheByUpload(ctx context.Context return nil, errors.Wrap(err, "SDatastore.ImportTemplate") } } - remotePath = filepath.Join(ds.GetUrl(), remotePath) + remotePath = ds.GetFullPath(remotePath) remoteImg := localImage.(*jsonutils.JSONDict) remoteImg.Add(jsonutils.NewString(remotePath), "path") diff --git a/pkg/multicloud/esxi/host.go b/pkg/multicloud/esxi/host.go index 7a8cb8346e..5fbc851a05 100644 --- a/pkg/multicloud/esxi/host.go +++ b/pkg/multicloud/esxi/host.go @@ -680,7 +680,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, data *jsonuti if len(cdromPath) != 0 && !strings.HasPrefix(cdromPath, "[") { cdromPath, err = self.FileUrlPathToDsPath(cdromPath) if err != nil { - return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath for cdrom path '%s'", cdromPath) + return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath", cdromPath) } } deviceChange = append(deviceChange, addDevSpec(NewCDROMDev(cdromPath, 16000, 201))) @@ -702,7 +702,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, data *jsonuti } else { imagePath, err = self.FileUrlPathToDsPath(imagePath) if err != nil { - return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath for image path '%s'", imagePath) + return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath", imagePath) } } uuid, _ := disk.GetString("disk_id") @@ -928,10 +928,14 @@ func (host *SHost) FileUrlPathToDsPath(path string) (string, error) { } for _, ds := range dss { rds := ds.(*SDatastore) + log.Debugf("rds: %s", rds.GetUrl()) if strings.HasPrefix(path, rds.GetUrl()) { newPath = fmt.Sprintf("[%s] %s", rds.GetRelName(), path[len(rds.GetUrl()):]) + break } - break + } + if len(newPath) == 0 { + return newPath, fmt.Errorf("path '%s' don't belong any datastore of host '%s'", path, host.GetName()) } return newPath, nil } diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index 6bcbeeaad0..4933585c67 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -504,7 +504,7 @@ func (cli *SESXiClient) CopyDisk(ctx context.Context, src, dst string, isForce b dm := object.NewVirtualDiskManager(cli.client.Client) task, err := dm.CopyVirtualDisk(ctx, src, nil, dst, nil, nil, isForce) if err != nil { - return err + return errors.Wrap(err, "CopyVirtualDisk") } return task.Wait(ctx) } @@ -513,7 +513,7 @@ func (cli *SESXiClient) MoveDisk(ctx context.Context, src, dst string, isForce b dm := object.NewVirtualDiskManager(cli.client.Client) task, err := dm.MoveVirtualDisk(ctx, src, nil, dst, nil, isForce) if err != nil { - return err + return errors.Wrap(err, "MoveVirtualDisk") } return task.Wait(ctx) } diff --git a/pkg/multicloud/esxi/shell/store.go b/pkg/multicloud/esxi/shell/store.go index f67a497869..77e9cbec01 100644 --- a/pkg/multicloud/esxi/shell/store.go +++ b/pkg/multicloud/esxi/shell/store.go @@ -170,12 +170,11 @@ func init() { if err != nil { return err } - ctx := context.Background() - path, err := dsObj.MakeDir(ctx, args.DIR) + err = dsObj.MakeDir(args.DIR) if err != nil { return err } - fmt.Println("Make dir success", path) + fmt.Println("Make dir success") return nil }) diff --git a/pkg/multicloud/esxi/storage.go b/pkg/multicloud/esxi/storage.go index 310110d7af..81f6734917 100644 --- a/pkg/multicloud/esxi/storage.go +++ b/pkg/multicloud/esxi/storage.go @@ -418,7 +418,7 @@ func (self *SDatastore) getPathString(path string) string { return fmt.Sprintf("[%s] %s", self.SManagedObject.GetName(), path) } -func (self *SDatastore) getFullPath(remotePath string) string { +func (self *SDatastore) GetFullPath(remotePath string) string { remotePath = self.cleanPath(remotePath) return path.Join(self.GetUrl(), remotePath) } @@ -438,6 +438,9 @@ func (self *SDatastore) FileGetContent(ctx context.Context, remotePath string) ( var bytes []byte err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode == 404 { + return cloudprovider.ErrNotFound + } if resp.StatusCode >= 400 { return fmt.Errorf("%s", resp.Status) } @@ -690,20 +693,18 @@ func (self *SDatastore) getDatastoreObj() *object.Datastore { return object.NewDatastore(self.manager.client.Client, self.getDatastore().Self) } -func (self *SDatastore) MakeDir(ctx context.Context, remotePath string) (string, error) { - dnm := object.NewDatastoreNamespaceManager(self.manager.client.Client) - +func (self *SDatastore) MakeDir(remotePath string) error { remotePath = self.cleanPath(remotePath) - objDS := self.getDatastoreObj() - - return dnm.CreateDirectory(ctx, objDS, remotePath, "") + m := object.NewFileManager(self.manager.client.Client) + path := fmt.Sprintf("[%s] %s", self.GetRelName(), remotePath) + return m.MakeDirectory(self.manager.context, path, self.datacenter.getObjectDatacenter(), true) } func (self *SDatastore) RemoveDir(ctx context.Context, remotePath string) error { dnm := object.NewDatastoreNamespaceManager(self.manager.client.Client) - remotePath = self.getFullPath(remotePath) + remotePath = self.GetFullPath(remotePath) dc, err := self.GetDatacenter() if err != nil { @@ -717,17 +718,15 @@ func (self *SDatastore) RemoveDir(ctx context.Context, remotePath string) error // CheckDirC will check that Dir 'remotePath' is exist, if not, create one. func (self *SDatastore) CheckDirC(remotePath string) error { - _, err := self.CheckFile(context.Background(), remotePath) + _, err := self.CheckFile(self.manager.context, remotePath) if err == nil { return nil } if errors.Cause(err) != cloudprovider.ErrNotFound { return err } - m := object.NewFileManager(self.manager.client.Client) - path := fmt.Sprintf("[%s] %s", self.GetRelName(), remotePath) - return m.MakeDirectory(self.manager.context, path, self.datacenter.getObjectDatacenter(), - true) + return self.MakeDir(remotePath) + } func (self *SDatastore) IsSysDiskStore() bool { diff --git a/pkg/multicloud/esxi/vdisk.go b/pkg/multicloud/esxi/vdisk.go index d16a8775a8..dfe9ce5ebb 100644 --- a/pkg/multicloud/esxi/vdisk.go +++ b/pkg/multicloud/esxi/vdisk.go @@ -203,7 +203,7 @@ func (disk *SVirtualDisk) GetAccessPath() string { return "" } ds := istore.(*SDatastore) - return ds.getFullPath(disk.getBackingInfo().GetFileName()) + return ds.GetFullPath(disk.getBackingInfo().GetFileName()) } func (disk *SVirtualDisk) GetDiskFormat() string { From 5b1c97af58111980b7135c99ea8766cd65435750 Mon Sep 17 00:00:00 2001 From: Rain Date: Thu, 16 Jan 2020 21:00:15 +0800 Subject: [PATCH 2/2] bugfix(esxiagent): PrefetchImageCacheByCopy & SHost.FileUrlPathToDsPath 1. Fix the bug when preparing remote image by copy 2. SHost.FileUrlPathToDsPath will not work when shost has multiple storage. --- pkg/multicloud/esxi/host.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/multicloud/esxi/host.go b/pkg/multicloud/esxi/host.go index 5fbc851a05..f7e9a42d40 100644 --- a/pkg/multicloud/esxi/host.go +++ b/pkg/multicloud/esxi/host.go @@ -680,7 +680,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, data *jsonuti if len(cdromPath) != 0 && !strings.HasPrefix(cdromPath, "[") { cdromPath, err = self.FileUrlPathToDsPath(cdromPath) if err != nil { - return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath", cdromPath) + return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath") } } deviceChange = append(deviceChange, addDevSpec(NewCDROMDev(cdromPath, 16000, 201))) @@ -702,7 +702,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, data *jsonuti } else { imagePath, err = self.FileUrlPathToDsPath(imagePath) if err != nil { - return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath", imagePath) + return nil, errors.Wrapf(err, "SHost.FileUrlPathToDsPath") } } uuid, _ := disk.GetString("disk_id")