From 959306646b6c1b453557cd173201a6d7329ecf64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 20 Feb 2019 16:40:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0rbd=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gopkg.toml | 4 + pkg/compute/models/storages.go | 2 +- pkg/compute/storagedrivers/rbd.go | 2 +- pkg/hostman/hostinfo/hostinfo.go | 4 +- pkg/hostman/storageman/core.go | 18 +- .../storageman/{diskbase.go => disk_base.go} | 0 .../{disklocal.go => disk_local.go} | 0 pkg/hostman/storageman/disk_rbd.go | 98 +- pkg/hostman/storageman/imagecache_base.go | 19 + ...{imagecachebase.go => imagecache_local.go} | 20 +- pkg/hostman/storageman/imagecache_rbd.go | 94 ++ .../storageman/imagecachemanager_agent.go | 10 + .../storageman/imagecachemanager_base.go | 42 + ...emanager.go => imagecachemanager_local.go} | 52 - .../storageman/imagecachemanager_rbd.go | 117 +++ .../{storagebase.go => storage_base.go} | 20 + .../{storagelocal.go => storage_local.go} | 0 pkg/hostman/storageman/storage_rbd.go | 245 ++++- pkg/util/qemuimg/qemuimg.go | 2 +- vendor/github.com/ceph/go-ceph/LICENSE | 21 + vendor/github.com/ceph/go-ceph/rados/conn.go | 328 ++++++ vendor/github.com/ceph/go-ceph/rados/doc.go | 4 + vendor/github.com/ceph/go-ceph/rados/ioctx.go | 890 ++++++++++++++++ vendor/github.com/ceph/go-ceph/rados/rados.go | 85 ++ vendor/github.com/ceph/go-ceph/rbd/doc.go | 4 + vendor/github.com/ceph/go-ceph/rbd/rbd.go | 991 ++++++++++++++++++ 26 files changed, 2962 insertions(+), 110 deletions(-) rename pkg/hostman/storageman/{diskbase.go => disk_base.go} (100%) rename pkg/hostman/storageman/{disklocal.go => disk_local.go} (100%) create mode 100644 pkg/hostman/storageman/imagecache_base.go rename pkg/hostman/storageman/{imagecachebase.go => imagecache_local.go} (95%) create mode 100644 pkg/hostman/storageman/imagecache_rbd.go create mode 100644 pkg/hostman/storageman/imagecachemanager_agent.go create mode 100644 pkg/hostman/storageman/imagecachemanager_base.go rename pkg/hostman/storageman/{imagecachemanager.go => imagecachemanager_local.go} (74%) create mode 100644 pkg/hostman/storageman/imagecachemanager_rbd.go rename pkg/hostman/storageman/{storagebase.go => storage_base.go} (93%) rename pkg/hostman/storageman/{storagelocal.go => storage_local.go} (100%) create mode 100644 vendor/github.com/ceph/go-ceph/LICENSE create mode 100644 vendor/github.com/ceph/go-ceph/rados/conn.go create mode 100644 vendor/github.com/ceph/go-ceph/rados/doc.go create mode 100644 vendor/github.com/ceph/go-ceph/rados/ioctx.go create mode 100644 vendor/github.com/ceph/go-ceph/rados/rados.go create mode 100644 vendor/github.com/ceph/go-ceph/rbd/doc.go create mode 100644 vendor/github.com/ceph/go-ceph/rbd/rbd.go diff --git a/Gopkg.toml b/Gopkg.toml index 5c1fc99166..a43a3f429f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -203,3 +203,7 @@ [[constraint]] name = "github.com/shirou/gopsutil" version = "2.18.10" + +[[constraint]] + name="github.com/ceph/go-ceph" + branch = "master" \ No newline at end of file diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index a62bf1344c..132f8e7fc9 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -129,7 +129,7 @@ type SStorage struct { StoragecacheId string `width:"36" charset:"ascii" nullable:"true" list:"admin" get:"admin" update:"admin" create:"optional"` Enabled bool `nullable:"false" default:"true" list:"user" create:"optional"` - Status string `width:"36" charset:"ascii" nullable:"false" default:"offline" list:"user" create:"optional"` + Status string `width:"36" charset:"ascii" nullable:"false" default:"offline" update:"admin" list:"user" create:"optional"` // indicating whether system disk can be allocated in this storage IsSysDiskStore bool `nullable:"false" default:"true" list:"user" create:"optional" update:"admin"` diff --git a/pkg/compute/storagedrivers/rbd.go b/pkg/compute/storagedrivers/rbd.go index afc1b27746..3ded06cd54 100644 --- a/pkg/compute/storagedrivers/rbd.go +++ b/pkg/compute/storagedrivers/rbd.go @@ -33,7 +33,7 @@ func (self *SRbdStorageDriver) ValidateCreateData(ctx context.Context, userCred return nil, httperrors.NewMissingParameterError(v) } value, _ := data.GetString(v) - conf.Add(jsonutils.NewString(value), strings.TrimLeft(v, "rbd_")) + conf.Add(jsonutils.NewString(value), strings.TrimPrefix(v, "rbd_")) } if key, _ := data.GetString("rbd_key"); len(key) > 0 { conf.Add(jsonutils.NewString(key), "key") diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index d1e7e12bde..ec980dd089 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -1114,15 +1114,17 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { if !utils.IsInStringArray(storagetype, storagetypes.Local) { storage := storageManager.NewSharedStorageInstance(mountPoint, storagetype) if storage != nil { + storage.SetStoragecacheId(storagecacheId) + storage.SetStorageInfo(storageId, storageName, storageConf) storageManager.Storages = append(storageManager.Storages, storage) storageManager.InitSharedStorageImageCache( storagetype, storagecacheId, imagecachePath, storage) - storage.SetStorageInfo(storageId, storageName, storageConf) } } else { // Storage type local storage := storageManager.GetStorageByPath(mountPoint) if storage != nil { + storage.SetStoragecacheId(storagecacheId) storage.SetStorageInfo(storageId, storageName, storageConf) } else { // XXX hack: storage type baremetal is a converted host，reserve storage diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 7d825099b7..00567490fb 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -260,12 +260,24 @@ func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheI // TODO // s.InitNfsStorageImagecache(storagecacheId, imagecachePath) } else if storageType == storagetypes.STORAGE_RBD { - if s.GetStoragecacheById(storagecacheId) == nil { - // TODO - // s.AddRbdStorageImagecache(imagecachePath, rbdStorage, storagecacheId) + if rbdStorage := s.GetStoragecacheById(storagecacheId); rbdStorage == nil { + // Done + s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId) } } +} +func (s *SStorageManager) AddRbdStorageImagecache(imagecachePath string, storage IStorage, storagecacheId string) { + if s.RbdStorageImagecacheManagers == nil { + s.RbdStorageImagecacheManagers = map[string]IImageCacheManger{} + } + if _, ok := s.RbdStorageImagecacheManagers[storagecacheId]; !ok { + if imagecache := NewRbdImageCacheManager(s, imagecachePath, storage, storagecacheId); imagecache != nil { + s.RbdStorageImagecacheManagers[storagecacheId] = imagecache + return + } + log.Errorf("failed init storagecache %s for storage %s", storagecacheId, storage.GetStorageName()) + } } var storageManager *SStorageManager diff --git a/pkg/hostman/storageman/diskbase.go b/pkg/hostman/storageman/disk_base.go similarity index 100% rename from pkg/hostman/storageman/diskbase.go rename to pkg/hostman/storageman/disk_base.go diff --git a/pkg/hostman/storageman/disklocal.go b/pkg/hostman/storageman/disk_local.go similarity index 100% rename from pkg/hostman/storageman/disklocal.go rename to pkg/hostman/storageman/disk_local.go diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index 2a99b1f047..c401b9989e 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -2,10 +2,13 @@ package storageman import ( "context" + "fmt" + "github.com/ceph/go-ceph/rbd" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" "yunion.io/x/onecloud/pkg/hostman/guestfs" + "yunion.io/x/onecloud/pkg/hostman/hostutils" ) type SRBDDisk struct { @@ -23,11 +26,24 @@ func (d *SRBDDisk) GetType() string { } func (d *SRBDDisk) Probe() error { - return nil + storage := d.Storage.(*SRbdStorage) + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + _, err := storage.withImage(pool, d.Id, true, func(image *rbd.Image) error { + return nil + }) + return err +} + +func (d *SRBDDisk) getPath() string { + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + return fmt.Sprintf("rbd:%s/%s", pool, d.Id) } func (d *SRBDDisk) GetPath() string { - return "" + storage := d.Storage.(*SRbdStorage) + return fmt.Sprintf("%s%s", d.getPath(), storage.getStorageConfString()) } func (d *SRBDDisk) GetSnapshotDir() string { @@ -35,23 +51,46 @@ func (d *SRBDDisk) GetSnapshotDir() string { } func (d *SRBDDisk) GetDiskDesc() jsonutils.JSONObject { - return nil + storage := d.Storage.(*SRbdStorage) + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + desc := map[string]interface{}{ + "disk_id": d.Id, + "disk_format": "raw", + "disk_path": d.GetPath(), + "disk_size": storage.getImageSizeMb(pool, d.Id), + } + return jsonutils.Marshal(desc) } func (d *SRBDDisk) GetDiskSetupScripts(idx int) string { - return "" + return fmt.Sprintf(`DISK_%d=%s\n`, idx, d.GetPath()) } func (d *SRBDDisk) DeleteAllSnapshot() error { - return nil + return fmt.Errorf("Not Impl") } func (d *SRBDDisk) Delete(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { - return nil, nil + storage := d.Storage.(*SRbdStorage) + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + return nil, storage.deleteImage(pool, d.Id) } func (d *SRBDDisk) Resize(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { - return nil, nil + diskInfo, ok := params.(*jsonutils.JSONDict) + if !ok { + return nil, hostutils.ParamsError + } + storage := d.Storage.(*SRbdStorage) + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + sizeMb, _ := diskInfo.Int("size") + if err := storage.resizeImage(pool, d.Id, uint64(sizeMb)); err != nil { + return nil, err + } + return d.GetDiskDesc(), nil } func (d *SRBDDisk) PrepareSaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { @@ -63,7 +102,9 @@ func (d *SRBDDisk) ResetFromSnapshot(ctx context.Context, params interface{}) (j } func (d *SRBDDisk) CleanupSnapshots(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { - return nil, nil + storage := d.Storage.(*SRbdStorage) + pool, _ := storage.StorageConf.GetString("pool") + return nil, storage.deleteSnapshot(pool, d.Id, "") } func (d *SRBDDisk) PrepareMigrate(liveMigrate bool) (string, error) { @@ -74,8 +115,30 @@ func (d *SRBDDisk) CreateFromUrl(context.Context, string) error { return nil } -func (d *SRBDDisk) CreateFromTemplate(context.Context, string, string, int64) (jsonutils.JSONObject, error) { - return nil, nil +func (d *SRBDDisk) CreateFromTemplate(ctx context.Context, imageId string, format string, size int64) (jsonutils.JSONObject, error) { + ret, err := d.createFromTemplate(ctx, imageId, format) + if err != nil { + return nil, err + } + return ret, nil +} + +func (d *SRBDDisk) createFromTemplate(ctx context.Context, imageId, format string) (jsonutils.JSONObject, error) { + var imageCacheManager = storageManager.GetStoragecacheById(d.Storage.GetStoragecacheId()) + if imageCacheManager == nil { + return nil, fmt.Errorf("failed to find image cache manger for storage %s", d.Storage.GetStorageName()) + } + imageCache := imageCacheManager.AcquireImage(ctx, imageId, d.GetZone(), "", "") + if imageCache == nil { + return nil, fmt.Errorf("failed to qcquire image for storage %s", d.Storage.GetStorageName()) + } + defer imageCacheManager.ReleaseImage(imageId) + storage := d.Storage.(*SRbdStorage) + destPool, _ := storage.StorageConf.GetString("pool") + if err := storage.copyImage(imageCacheManager.GetPath(), imageCache.GetName(), destPool, d.Id); err != nil { + return nil, err + } + return d.GetDiskDesc(), nil } func (d *SRBDDisk) CreateFromImageFuse(context.Context, string) error { @@ -83,7 +146,12 @@ func (d *SRBDDisk) CreateFromImageFuse(context.Context, string) error { } func (d *SRBDDisk) CreateRaw(ctx context.Context, sizeMb int, diskFromat string, fsFormat string, encryption bool, diskId string, back string) (jsonutils.JSONObject, error) { - return nil, nil + storage := d.Storage.(*SRbdStorage) + pool, _ := storage.StorageConf.GetString("pool") + if err := storage.createImage(pool, diskId, uint64(sizeMb)); err != nil { + return nil, err + } + return d.GetDiskDesc(), nil } func (d *SRBDDisk) PostCreateFromImageFuse() { @@ -91,11 +159,15 @@ func (d *SRBDDisk) PostCreateFromImageFuse() { } func (d *SRBDDisk) CreateSnapshot(snapshotId string) error { - return nil + storage := d.Storage.(*SRbdStorage) + pool, _ := storage.StorageConf.GetString("pool") + return storage.createSnapshot(pool, d.Id, snapshotId) } func (d *SRBDDisk) DeleteSnapshot(snapshotId, convertSnapshot string, pendingDelete bool) error { - return nil + storage := d.Storage.(*SRbdStorage) + pool, _ := storage.StorageConf.GetString("pool") + return storage.deleteSnapshot(pool, d.Id, snapshotId) } func (d *SRBDDisk) DeployGuestFs(diskPath string, guestDesc *jsonutils.JSONDict, deployInfo *guestfs.SDeployInfo) (jsonutils.JSONObject, error) { diff --git a/pkg/hostman/storageman/imagecache_base.go b/pkg/hostman/storageman/imagecache_base.go new file mode 100644 index 0000000000..c26ce4a9fd --- /dev/null +++ b/pkg/hostman/storageman/imagecache_base.go @@ -0,0 +1,19 @@ +package storageman + +import ( + "context" + + "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" +) + +type IImageCache interface { + GetPath() string + GetName() string + Load() bool + Acquire(ctx context.Context, zone, srcUrl, format string) bool + Release() + Remove(ctx context.Context) error + GetImageId() string + + GetDesc() *remotefile.SImageDesc +} diff --git a/pkg/hostman/storageman/imagecachebase.go b/pkg/hostman/storageman/imagecache_local.go similarity index 95% rename from pkg/hostman/storageman/imagecachebase.go rename to pkg/hostman/storageman/imagecache_local.go index a1800981cd..8a0dc6b66f 100644 --- a/pkg/hostman/storageman/imagecachebase.go +++ b/pkg/hostman/storageman/imagecache_local.go @@ -26,17 +26,6 @@ const ( CHECK_TIMEOUT = 3600 * time.Second ) -type IImageCache interface { - GetPath() string - Load() bool - Acquire(ctx context.Context, zone, srcUrl, format string) bool - Release() - Remove(ctx context.Context) error - GetImageId() string - - GetDesc() *remotefile.SImageDesc -} - type SLocalImageCache struct { imageId string Manager IImageCacheManger @@ -66,6 +55,10 @@ func (l *SLocalImageCache) GetImageId() string { return l.imageId } +func (l *SLocalImageCache) GetName() string { + return l.imageId +} + func (l *SLocalImageCache) Load() bool { var ( imgPath = l.GetPath() @@ -270,8 +263,3 @@ func (l *SLocalImageCache) GetSize() int64 { return fi.Size() } } - -type SRbdImageCache struct { - imageId string - Manager IImageCacheManger -} diff --git a/pkg/hostman/storageman/imagecache_rbd.go b/pkg/hostman/storageman/imagecache_rbd.go new file mode 100644 index 0000000000..58a58a856d --- /dev/null +++ b/pkg/hostman/storageman/imagecache_rbd.go @@ -0,0 +1,94 @@ +package storageman + +import ( + "context" + "fmt" + "sync" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/qemutils" +) + +type SRbdImageCache struct { + imageId string + cond *sync.Cond + Manager IImageCacheManger +} + +func NewRbdImageCache(imageId string, imagecacheManager IImageCacheManger) *SRbdImageCache { + imageCache := new(SRbdImageCache) + imageCache.imageId = imageId + imageCache.Manager = imagecacheManager + imageCache.cond = sync.NewCond(new(sync.Mutex)) + return imageCache +} + +func (r *SRbdImageCache) GetName() string { + imageCacheManger := r.Manager.(*SRbdImageCacheManager) + return fmt.Sprintf("%s%s", imageCacheManger.Prefix, r.imageId) +} + +func (r *SRbdImageCache) GetPath() string { + imageCacheManger := r.Manager.(*SRbdImageCacheManager) + storage := imageCacheManger.storage.(*SRbdStorage) + return fmt.Sprintf("rbd:%s/%s%s", imageCacheManger.GetPath(), r.GetName(), storage.getStorageConfString()) +} + +func (r *SRbdImageCache) Load() bool { + log.Debugf("loading rbd imagecache %s", r.GetPath()) + origin, err := qemuimg.NewQemuImage(r.GetPath()) + if err != nil { + return false + } + return origin.IsValid() +} + +func (r *SRbdImageCache) Acquire(ctx context.Context, zone, srcUrl, format string) bool { + localImageCache := storageManager.LocalStorageImagecacheManager.AcquireImage(ctx, r.imageId, zone, srcUrl, format) + if localImageCache == nil { + log.Errorf("failed to acquireimage %s ", r.imageId) + return false + } + log.Debugf("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath()) + + _, err := procutils.NewCommand(qemutils.GetQemuImg(), "convert", "-O", "raw", localImageCache.GetPath(), r.GetName()).Run() + if err != nil { + log.Errorf("failed to convert image %s", options.HostOptions.ServersPath) + } + return r.Load() +} + +func (r *SRbdImageCache) Release() { + return +} + +func (r *SRbdImageCache) Remove(ctx context.Context) error { + imageCacheManger := r.Manager.(*SRbdImageCacheManager) + storage := imageCacheManger.storage.(*SRbdStorage) + if err := storage.deleteImage(r.Manager.GetPath(), r.GetName()); err != nil { + return err + } + + go func() { + _, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx), + r.Manager.GetId(), r.imageId, nil) + if err != nil { + log.Errorf("Fail to delete host cached image: %s", err) + } + }() + return nil +} + +func (r *SRbdImageCache) GetDesc() *remotefile.SImageDesc { + return nil +} + +func (r *SRbdImageCache) GetImageId() string { + return r.imageId +} diff --git a/pkg/hostman/storageman/imagecachemanager_agent.go b/pkg/hostman/storageman/imagecachemanager_agent.go new file mode 100644 index 0000000000..5415a14165 --- /dev/null +++ b/pkg/hostman/storageman/imagecachemanager_agent.go @@ -0,0 +1,10 @@ +package storageman + +// not need to impl, using esxi agent +type SAgentImageCacheManager struct { + storagemanager *SStorageManager +} + +func NewAgentImageCacheManager(storagemanager *SStorageManager) *SAgentImageCacheManager { + return &SAgentImageCacheManager{storagemanager} +} diff --git a/pkg/hostman/storageman/imagecachemanager_base.go b/pkg/hostman/storageman/imagecachemanager_base.go new file mode 100644 index 0000000000..b94e974083 --- /dev/null +++ b/pkg/hostman/storageman/imagecachemanager_base.go @@ -0,0 +1,42 @@ +package storageman + +import ( + "context" + "sync" + + "yunion.io/x/jsonutils" +) + +type IImageCacheManger interface { + GetId() string + GetPath() string + SetStoragecacheId(string) + + // for diskhandler + PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) + DeleteImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) + + AcquireImage(ctx context.Context, imageId, zone, srcUrl, format string) IImageCache + ReleaseImage(imageId string) + LoadImageCache(imageId string) +} + +type SBaseImageCacheManager struct { + storagemanager *SStorageManager + storagecacaheId string + cachePath string + cachedImages map[string]IImageCache + mutex *sync.Mutex +} + +func (c *SBaseImageCacheManager) GetPath() string { + return c.cachePath +} + +func (c *SBaseImageCacheManager) GetId() string { + return c.storagecacaheId +} + +func (c *SBaseImageCacheManager) SetStoragecacheId(scid string) { + c.storagecacaheId = scid +} diff --git a/pkg/hostman/storageman/imagecachemanager.go b/pkg/hostman/storageman/imagecachemanager_local.go similarity index 74% rename from pkg/hostman/storageman/imagecachemanager.go rename to pkg/hostman/storageman/imagecachemanager_local.go index 26a39a973e..dad28e3640 100644 --- a/pkg/hostman/storageman/imagecachemanager.go +++ b/pkg/hostman/storageman/imagecachemanager_local.go @@ -14,40 +14,6 @@ import ( "yunion.io/x/pkg/util/regutils" ) -type IImageCacheManger interface { - GetId() string - GetPath() string - SetStoragecacheId(string) - - // for diskhandler - PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) - DeleteImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) - - AcquireImage(ctx context.Context, imageId, zone, srcUrl, format string) IImageCache - ReleaseImage(imageId string) - LoadImageCache(imageId string) -} - -type SBaseImageCacheManager struct { - storagemanager *SStorageManager - storagecacaheId string - cachePath string - cachedImages map[string]IImageCache - mutex *sync.Mutex -} - -func (c *SBaseImageCacheManager) GetPath() string { - return c.cachePath -} - -func (c *SBaseImageCacheManager) GetId() string { - return c.storagecacaheId -} - -func (c *SBaseImageCacheManager) SetStoragecacheId(scid string) { - c.storagecacaheId = scid -} - type SLocalImageCacheManager struct { SBaseImageCacheManager limit int @@ -183,21 +149,3 @@ func (c *SLocalImageCacheManager) PrefetchImageCache(ctx context.Context, data i } // TODO: AgentImageCacheManager -type SAgentImageCacheManager struct { - storagemanager *SStorageManager -} - -func NewAgentImageCacheManager(storagemanager *SStorageManager) *SAgentImageCacheManager { - return &SAgentImageCacheManager{storagemanager} -} - -type SRbdImageCacheManager struct { - SBaseImageCacheManager - pool, prefix string - storage IStorage -} - -func NewRbdImageCacheManager() *SRbdImageCacheManager { - // TODO - return nil -} diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go new file mode 100644 index 0000000000..0d03e790c0 --- /dev/null +++ b/pkg/hostman/storageman/imagecachemanager_rbd.go @@ -0,0 +1,117 @@ +package storageman + +import ( + "context" + "strings" + "sync" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/hostman/hostutils" +) + +type SRbdImageCacheManager struct { + SBaseImageCacheManager + Pool, Prefix string + storage IStorage +} + +func NewRbdImageCacheManager(manager *SStorageManager, cachePath string, storage IStorage, storagecacheId string) *SRbdImageCacheManager { + imageCacheManager := new(SRbdImageCacheManager) + + imageCacheManager.storagemanager = manager + imageCacheManager.storagecacaheId = storagecacheId + imageCacheManager.storage = storage + + // cachePath like `rbd:pool/imagecache` or `rbd:pool` + cachePath = strings.TrimPrefix(cachePath, "rbd:") + poolInfo := strings.Split(cachePath, "/") + if len(poolInfo) == 2 { + imageCacheManager.Pool, imageCacheManager.Prefix = poolInfo[0], poolInfo[1] + } else { + imageCacheManager.Pool, imageCacheManager.Prefix = cachePath, "image_cache_" + } + imageCacheManager.cachedImages = make(map[string]IImageCache, 0) + imageCacheManager.mutex = new(sync.Mutex) + imageCacheManager.loadCache() + return imageCacheManager +} + +func (c *SRbdImageCacheManager) loadCache() { + c.mutex.Lock() + defer c.mutex.Unlock() + storage := c.storage.(*SRbdStorage) + + images, err := storage.listImages(c.Pool) + if err != nil { + log.Errorf("get storage %s images error; %v", c.storage.GetStorageName(), err) + return + } + for _, image := range images { + if strings.HasPrefix(image, c.Prefix) { + imageId := strings.TrimPrefix(image, c.Prefix) + c.LoadImageCache(imageId) + } else { + log.Debugf("find image %s from stroage %s", image, c.storage.GetStorageName()) + } + } +} + +func (c *SRbdImageCacheManager) LoadImageCache(imageId string) { + imageCache := NewRbdImageCache(imageId, c) + if imageCache.Load() { + c.cachedImages[imageId] = imageCache + } +} + +func (c *SRbdImageCacheManager) GetPath() string { + return c.Pool +} + +func (c *SRbdImageCacheManager) PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) { + return nil, nil +} + +func (c *SRbdImageCacheManager) DeleteImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) { + body, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil, hostutils.ParamsError + } + + imageId, _ := body.GetString("image_id") + return nil, c.removeImage(ctx, imageId) +} + +func (c *SRbdImageCacheManager) removeImage(ctx context.Context, imageId string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + if img, ok := c.cachedImages[imageId]; ok { + delete(c.cachedImages, imageId) + return img.Remove(ctx) + } + return nil +} + +func (c *SRbdImageCacheManager) AcquireImage(ctx context.Context, imageId, zone, srcUrl, format string) IImageCache { + c.mutex.Lock() + defer c.mutex.Unlock() + + img, ok := c.cachedImages[imageId] + if !ok { + img = NewRbdImageCache(imageId, c) + c.cachedImages[imageId] = img + } + if img.Acquire(ctx, zone, srcUrl, format) { + return img + } + return nil +} + +func (c *SRbdImageCacheManager) ReleaseImage(imageId string) { + c.mutex.Lock() + defer c.mutex.Unlock() + if img, ok := c.cachedImages[imageId]; ok { + img.Release() + } +} diff --git a/pkg/hostman/storageman/storagebase.go b/pkg/hostman/storageman/storage_base.go similarity index 93% rename from pkg/hostman/storageman/storagebase.go rename to pkg/hostman/storageman/storage_base.go index 9c185b55bd..1f8fc27009 100644 --- a/pkg/hostman/storageman/storagebase.go +++ b/pkg/hostman/storageman/storage_base.go @@ -18,11 +18,15 @@ var ( type IStorage interface { GetId() string + GetStorageName() string GetZone() string SetStorageInfo(storageId, storageName string, conf jsonutils.JSONObject) SyncStorageInfo() (jsonutils.JSONObject, error) StorageType() string + GetStorageConf() *jsonutils.JSONDict + GetStoragecacheId() string + SetStoragecacheId(storagecacheId string) SetPath(string) GetPath() string @@ -78,6 +82,18 @@ func (s *SBaseStorage) GetId() string { return s.StorageId } +func (s *SBaseStorage) GetStorageName() string { + return s.StorageName +} + +func (s *SBaseStorage) GetStoragecacheId() string { + return s.StoragecacheId +} + +func (s *SBaseStorage) SetStoragecacheId(storagecacheId string) { + s.StoragecacheId = storagecacheId +} + func (s *SBaseStorage) GetName(generateName func() string) string { if len(s.StorageName) > 0 { return s.StorageName @@ -102,6 +118,10 @@ func (s *SBaseStorage) GetCapacity() int { return s.GetAvailSizeMb() } +func (s *SBaseStorage) GetStorageConf() *jsonutils.JSONDict { + return s.StorageConf +} + func (s *SBaseStorage) GetAvailSizeMb() int { return s.GetTotalSizeMb() } diff --git a/pkg/hostman/storageman/storagelocal.go b/pkg/hostman/storageman/storage_local.go similarity index 100% rename from pkg/hostman/storageman/storagelocal.go rename to pkg/hostman/storageman/storage_local.go diff --git a/pkg/hostman/storageman/storage_rbd.go b/pkg/hostman/storageman/storage_rbd.go index d88768ccfe..e2f944be3d 100644 --- a/pkg/hostman/storageman/storage_rbd.go +++ b/pkg/hostman/storageman/storage_rbd.go @@ -2,50 +2,252 @@ package storageman import ( "context" + "fmt" + "strings" + "github.com/ceph/go-ceph/rados" + "github.com/ceph/go-ceph/rbd" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/mcclient/modules" ) -type SRBDStorage struct { +const ( + RBD_FEATURE = 2 + RBD_ORDER = 22 //为rbd对应到rados中每个对象的大小，默认为4MB +) + +type SRbdStorage struct { SBaseStorage } -func NewRBDStorage(manager *SStorageManager, path string) *SRBDStorage { - var ret = new(SRBDStorage) +func NewRBDStorage(manager *SStorageManager, path string) *SRbdStorage { + var ret = new(SRbdStorage) ret.SBaseStorage = *NewBaseStorage(manager, path) return ret } -func (s *SRBDStorage) StorageType() string { +func (s *SRbdStorage) StorageType() string { return storagetypes.STORAGE_RBD } -func (s *SRBDStorage) GetSnapshotPathByIds(diskId, snapshotId string) string { +func (s *SRbdStorage) GetSnapshotPathByIds(diskId, snapshotId string) string { return "" } -func (s *SRBDStorage) GetSnapshotDir() string { +func (s *SRbdStorage) GetSnapshotDir() string { return "" } -func (s *SRBDStorage) GetFuseTmpPath() string { +func (s *SRbdStorage) GetFuseTmpPath() string { return "" } -func (s *SRBDStorage) GetFuseMountPath() string { +func (s *SRbdStorage) GetFuseMountPath() string { return "" } -func (s *SRBDStorage) GetImgsaveBackupPath() string { +func (s *SRbdStorage) GetImgsaveBackupPath() string { return "" } -func (s *SRBDStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { - return nil, nil +//Tip Configuration values containing :, @, or = can be escaped with a leading \ character. +func (s *SRbdStorage) getStorageConfString() string { + conf := "" + for _, key := range []string{"mon_host", "key", "rados_osd_op_timeout", "rados_mon_op_timeout", "client_mount_timeout", "rbd_default_format"} { + if value, _ := s.StorageConf.GetString(key); len(value) > 0 { + if key == "mon_host" { + value = strings.Replace(value, ",", `\;`, -1) + } + for _, keyworkd := range []string{":", "@", "="} { + if strings.Index(value, keyworkd) != -1 { + value = strings.Replace(value, keyworkd, fmt.Sprintf(`\%s`, keyworkd), -1) + } + } + conf += fmt.Sprintf(":%s=%s", key, value) + } + } + return conf } -func (s *SRBDStorage) GetDiskById(diskId string) IDisk { +func (s *SRbdStorage) getImageSizeMb(pool string, name string) uint64 { + stat, err := s.withImage(pool, name, true, func(image *rbd.Image) error { + return nil + }) + if err != nil { + log.Errorf("get image error: %v", err) + return 0 + } + return stat.Size / 1024 / 1024 +} + +func (s *SRbdStorage) resizeImage(pool string, name string, sizeMb uint64) error { + _, err := s.withImage(pool, name, true, func(image *rbd.Image) error { + return image.Resize(sizeMb * 1024 * 1024) + }) + return err +} + +func (s *SRbdStorage) deleteImage(pool string, name string) error { + _, err := s.withImage(pool, name, false, func(image *rbd.Image) error { + return image.Remove() + }) + return err +} + +func (s *SRbdStorage) copyImage(srcPool string, srcImage string, destPool string, destImage string) error { + _, err := s.withImage(srcPool, srcImage, true, func(src *rbd.Image) error { + imageSize, err := src.GetSize() + if err != nil { + return err + } + if err := s.createImage(destPool, destImage, imageSize/1024/1024); err != nil { + log.Errorf("create image dest pool: %s dest image: %s image size: %dMb error: %v", destPool, destImage, imageSize/1024/1024, err) + return err + } + _, err = s.withImage(destPool, destImage, true, func(dest *rbd.Image) error { + return src.Copy(*dest) + }) + return err + }) + return err +} + +func (s *SRbdStorage) withImage(pool string, name string, closeImage bool, doFunc func(*rbd.Image) error) (*rbd.ImageInfo, error) { + stat, err := s.withIOContext(pool, func(ioctx *rados.IOContext) (interface{}, error) { + image := rbd.GetImage(ioctx, name) + if err := image.Open(); err != nil { + log.Errorf("open image %s name error: %v", name, err) + return nil, err + } + stat, err := image.Stat() + if err != nil { + log.Errorf("get image %s stat error: %v", name, err) + return nil, err + } + if closeImage { + defer image.Close() + } + if err := doFunc(image); err != nil { + return nil, err + } + return stat, nil + }) + if err != nil { + return nil, err + } + return stat.(*rbd.ImageInfo), nil +} + +func (s *SRbdStorage) withIOContext(pool string, doFunc func(*rados.IOContext) (interface{}, error)) (interface{}, error) { + return s.withCluster(func(conn *rados.Conn) (interface{}, error) { + ioctx, err := conn.OpenIOContext(pool) + if err != nil { + log.Errorf("get ioctx for pool %s error: %v", pool, err) + return nil, err + } + return doFunc(ioctx) + }) +} + +func (s *SRbdStorage) listImages(pool string) ([]string, error) { + images, err := s.withIOContext(pool, func(ioctx *rados.IOContext) (interface{}, error) { + return rbd.GetImageNames(ioctx) + }) + if err != nil { + return nil, err + } + return images.([]string), nil +} + +func (s *SRbdStorage) withCluster(doFunc func(*rados.Conn) (interface{}, error)) (interface{}, error) { + conn, _ := rados.NewConn() + for _, key := range []string{"mon_host", "key"} { + if value, _ := s.StorageConf.GetString(key); len(value) > 0 { + if err := conn.SetConfigOption(key, value); err != nil { + return nil, err + } + } + } + if err := conn.Connect(); err != nil { + log.Errorf("connect rbd cluster %s error: %v", s.StorageName, err) + return nil, err + } + defer conn.Shutdown() + return doFunc(conn) +} + +func (s *SRbdStorage) createImage(pool string, name string, sizeMb uint64) error { + _, err := s.withIOContext(pool, func(ioctx *rados.IOContext) (interface{}, error) { + image, err := rbd.Create(ioctx, name, sizeMb*1024*1024, RBD_ORDER, RBD_FEATURE) + if err != nil { + return nil, err + } + defer image.Close() + return nil, nil + }) + return err +} + +func (s *SRbdStorage) createSnapshot(pool string, diskId string, snapshotId string) error { + _, err := s.withImage(pool, diskId, true, func(image *rbd.Image) error { + _, err := image.CreateSnapshot(snapshotId) + return err + }) + return err +} + +func (s *SRbdStorage) deleteSnapshot(pool string, diskId string, snapshotId string) error { + _, err := s.withImage(pool, diskId, true, func(image *rbd.Image) error { + snapshots, err := image.GetSnapshotNames() + if err != nil { + return err + } + for _, snapshot := range snapshots { + if len(snapshotId) == 0 || snapshot.Name == snapshotId { + if err := image.GetSnapshot(snapshot.Name).Remove(); err != nil { + return err + } + } + } + return nil + }) + return err +} + +func (s *SRbdStorage) getCapacity() (uint64, error) { + _stats, err := s.withCluster(func(conn *rados.Conn) (interface{}, error) { + return conn.GetClusterStats() + }) + if err != nil { + return 0, err + } + stats := _stats.(rados.ClusterStat) + return stats.Kb / 1024, nil +} + +func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { + content := map[string]interface{}{} + if len(s.StorageId) > 0 { + capacity, err := s.getCapacity() + if err != nil { + return nil, err + } + content = map[string]interface{}{ + "name": s.StorageName, + "capacity": capacity, + "status": models.STORAGE_ONLINE, + "zone": s.GetZone(), + } + return modules.Storages.Put(hostutils.GetComputeSession(context.Background()), s.StorageId, jsonutils.Marshal(content)) + } + return modules.Storages.Get(hostutils.GetComputeSession(context.Background()), s.StorageName, jsonutils.Marshal(content)) +} + +func (s *SRbdStorage) GetDiskById(diskId string) IDisk { s.DiskLock.Lock() defer s.DiskLock.Unlock() for i := 0; i < len(s.Disks); i++ { @@ -64,7 +266,7 @@ func (s *SRBDStorage) GetDiskById(diskId string) IDisk { } } -func (s *SRBDStorage) CreateDisk(diskId string) IDisk { +func (s *SRbdStorage) CreateDisk(diskId string) IDisk { s.DiskLock.Lock() defer s.DiskLock.Unlock() disk := NewRBDDisk(s, diskId) @@ -72,22 +274,21 @@ func (s *SRBDStorage) CreateDisk(diskId string) IDisk { return disk } -func (s *SRBDStorage) Accessible() bool { - return true +func (s *SRbdStorage) Accessible() bool { + _, err := s.withCluster(func(conn *rados.Conn) (interface{}, error) { + return conn.ListPools() + }) + return err == nil } -func (s *SRBDStorage) DeleteDiskfile(diskpath string) error { - return nil -} - -func (s *SRBDStorage) SaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { +func (s *SRbdStorage) SaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { return nil, nil } -func (s *SRBDStorage) CreateSnapshotFormUrl(ctx context.Context, snapshotUrl, diskId, snapshotPath string) error { +func (s *SRbdStorage) CreateSnapshotFormUrl(ctx context.Context, snapshotUrl, diskId, snapshotPath string) error { return nil } -func (s *SRBDStorage) DeleteSnapshots(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { +func (s *SRbdStorage) DeleteSnapshots(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { return nil, nil } diff --git a/pkg/util/qemuimg/qemuimg.go b/pkg/util/qemuimg/qemuimg.go index 3623c1e066..8e86a6f422 100644 --- a/pkg/util/qemuimg/qemuimg.go +++ b/pkg/util/qemuimg/qemuimg.go @@ -132,7 +132,7 @@ func (img *SQemuImage) parse() error { } } } - if img.Format == RAW { + if img.Format == RAW && fileutils2.IsFile(img.Path) { // test if it is an ISO blkType := fileutils2.GetBlkidType(img.Path) if utils.IsInStringArray(blkType, []string{"iso9660", "udf"}) { diff --git a/vendor/github.com/ceph/go-ceph/LICENSE b/vendor/github.com/ceph/go-ceph/LICENSE new file mode 100644 index 0000000000..08d70bfc05 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Noah Watkins + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/ceph/go-ceph/rados/conn.go b/vendor/github.com/ceph/go-ceph/rados/conn.go new file mode 100644 index 0000000000..3dfb30bfbb --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rados/conn.go @@ -0,0 +1,328 @@ +package rados + +// #cgo LDFLAGS: -lrados +// #include +// #include +import "C" + +import "unsafe" +import "bytes" +import "fmt" + +// ClusterStat represents Ceph cluster statistics. +type ClusterStat struct { + Kb uint64 + Kb_used uint64 + Kb_avail uint64 + Num_objects uint64 +} + +// Conn is a connection handle to a Ceph cluster. +type Conn struct { + cluster C.rados_t + connected bool +} + +// PingMonitor sends a ping to a monitor and returns the reply. +func (c *Conn) PingMonitor(id string) (string, error) { + c_id := C.CString(id) + defer C.free(unsafe.Pointer(c_id)) + + var strlen C.size_t + var strout *C.char + + ret := C.rados_ping_monitor(c.cluster, c_id, &strout, &strlen) + defer C.rados_buffer_free(strout) + + if ret == 0 { + reply := C.GoStringN(strout, (C.int)(strlen)) + return reply, nil + } else { + return "", RadosError(int(ret)) + } +} + +// Connect establishes a connection to a RADOS cluster. It returns an error, +// if any. +func (c *Conn) Connect() error { + ret := C.rados_connect(c.cluster) + if ret != 0 { + return RadosError(int(ret)) + } + c.connected = true + return nil +} + +// Shutdown disconnects from the cluster. +func (c *Conn) Shutdown() { + if err := c.ensure_connected(); err != nil { + return + } + C.rados_shutdown(c.cluster) +} + +// ReadConfigFile configures the connection using a Ceph configuration file. +func (c *Conn) ReadConfigFile(path string) error { + c_path := C.CString(path) + defer C.free(unsafe.Pointer(c_path)) + ret := C.rados_conf_read_file(c.cluster, c_path) + if ret == 0 { + return nil + } else { + return RadosError(int(ret)) + } +} + +// ReadDefaultConfigFile configures the connection using a Ceph configuration +// file located at default locations. +func (c *Conn) ReadDefaultConfigFile() error { + ret := C.rados_conf_read_file(c.cluster, nil) + if ret == 0 { + return nil + } else { + return RadosError(int(ret)) + } +} + +func (c *Conn) OpenIOContext(pool string) (*IOContext, error) { + c_pool := C.CString(pool) + defer C.free(unsafe.Pointer(c_pool)) + ioctx := &IOContext{} + ret := C.rados_ioctx_create(c.cluster, c_pool, &ioctx.ioctx) + if ret == 0 { + return ioctx, nil + } else { + return nil, RadosError(int(ret)) + } +} + +// ListPools returns the names of all existing pools. +func (c *Conn) ListPools() (names []string, err error) { + buf := make([]byte, 4096) + for { + ret := int(C.rados_pool_list(c.cluster, + (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + if ret < 0 { + return nil, RadosError(int(ret)) + } + + if ret > len(buf) { + buf = make([]byte, ret) + continue + } + + tmp := bytes.SplitAfter(buf[:ret-1], []byte{0}) + for _, s := range tmp { + if len(s) > 0 { + name := C.GoString((*C.char)(unsafe.Pointer(&s[0]))) + names = append(names, name) + } + } + + return names, nil + } +} + +// SetConfigOption sets the value of the configuration option identified by +// the given name. +func (c *Conn) SetConfigOption(option, value string) error { + c_opt, c_val := C.CString(option), C.CString(value) + defer C.free(unsafe.Pointer(c_opt)) + defer C.free(unsafe.Pointer(c_val)) + ret := C.rados_conf_set(c.cluster, c_opt, c_val) + if ret < 0 { + return RadosError(int(ret)) + } else { + return nil + } +} + +// GetConfigOption returns the value of the Ceph configuration option +// identified by the given name. +func (c *Conn) GetConfigOption(name string) (value string, err error) { + buf := make([]byte, 4096) + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_name)) + ret := int(C.rados_conf_get(c.cluster, c_name, + (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + // FIXME: ret may be -ENAMETOOLONG if the buffer is not large enough. We + // can handle this case, but we need a reliable way to test for + // -ENAMETOOLONG constant. Will the syscall/Errno stuff in Go help? + if ret == 0 { + value = C.GoString((*C.char)(unsafe.Pointer(&buf[0]))) + return value, nil + } else { + return "", RadosError(ret) + } +} + +// WaitForLatestOSDMap blocks the caller until the latest OSD map has been +// retrieved. +func (c *Conn) WaitForLatestOSDMap() error { + ret := C.rados_wait_for_latest_osdmap(c.cluster) + if ret < 0 { + return RadosError(int(ret)) + } else { + return nil + } +} + +func (c *Conn) ensure_connected() error { + if c.connected { + return nil + } else { + return RadosError(1) + } +} + +// GetClusterStat returns statistics about the cluster associated with the +// connection. +func (c *Conn) GetClusterStats() (stat ClusterStat, err error) { + if err := c.ensure_connected(); err != nil { + return ClusterStat{}, err + } + c_stat := C.struct_rados_cluster_stat_t{} + ret := C.rados_cluster_stat(c.cluster, &c_stat) + if ret < 0 { + return ClusterStat{}, RadosError(int(ret)) + } else { + return ClusterStat{ + Kb: uint64(c_stat.kb), + Kb_used: uint64(c_stat.kb_used), + Kb_avail: uint64(c_stat.kb_avail), + Num_objects: uint64(c_stat.num_objects), + }, nil + } +} + +// ParseCmdLineArgs configures the connection from command line arguments. +func (c *Conn) ParseCmdLineArgs(args []string) error { + // add an empty element 0 -- Ceph treats the array as the actual contents + // of argv and skips the first element (the executable name) + argc := C.int(len(args) + 1) + argv := make([]*C.char, argc) + + // make the first element a string just in case it is ever examined + argv[0] = C.CString("placeholder") + defer C.free(unsafe.Pointer(argv[0])) + + for i, arg := range args { + argv[i+1] = C.CString(arg) + defer C.free(unsafe.Pointer(argv[i+1])) + } + + ret := C.rados_conf_parse_argv(c.cluster, argc, &argv[0]) + if ret < 0 { + return RadosError(int(ret)) + } else { + return nil + } +} + +// ParseDefaultConfigEnv configures the connection from the default Ceph +// environment variable(s). +func (c *Conn) ParseDefaultConfigEnv() error { + ret := C.rados_conf_parse_env(c.cluster, nil) + if ret == 0 { + return nil + } else { + return RadosError(int(ret)) + } +} + +// GetFSID returns the fsid of the cluster as a hexadecimal string. The fsid +// is a unique identifier of an entire Ceph cluster. +func (c *Conn) GetFSID() (fsid string, err error) { + buf := make([]byte, 37) + ret := int(C.rados_cluster_fsid(c.cluster, + (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + // FIXME: the success case isn't documented correctly in librados.h + if ret == 36 { + fsid = C.GoString((*C.char)(unsafe.Pointer(&buf[0]))) + return fsid, nil + } else { + return "", RadosError(int(ret)) + } +} + +// GetInstanceID returns a globally unique identifier for the cluster +// connection instance. +func (c *Conn) GetInstanceID() uint64 { + // FIXME: are there any error cases for this? + return uint64(C.rados_get_instance_id(c.cluster)) +} + +// MakePool creates a new pool with default settings. +func (c *Conn) MakePool(name string) error { + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_name)) + ret := int(C.rados_pool_create(c.cluster, c_name)) + if ret == 0 { + return nil + } else { + return RadosError(ret) + } +} + +// DeletePool deletes a pool and all the data inside the pool. +func (c *Conn) DeletePool(name string) error { + if err := c.ensure_connected(); err != nil { + fmt.Println("NOT CONNECTED WHOOPS") + return err + } + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_name)) + ret := int(C.rados_pool_delete(c.cluster, c_name)) + if ret == 0 { + return nil + } else { + return RadosError(ret) + } +} + +// MonCommand sends a command to one of the monitors +func (c *Conn) MonCommand(args []byte) (buffer []byte, info string, err error) { + return c.monCommand(args, nil) +} + +// MonCommand sends a command to one of the monitors, with an input buffer +func (c *Conn) MonCommandWithInputBuffer(args, inputBuffer []byte) (buffer []byte, info string, err error) { + return c.monCommand(args, inputBuffer) +} + +func (c *Conn) monCommand(args, inputBuffer []byte) (buffer []byte, info string, err error) { + argv := C.CString(string(args)) + defer C.free(unsafe.Pointer(argv)) + + var ( + outs, outbuf *C.char + outslen, outbuflen C.size_t + ) + inbuf := C.CString(string(inputBuffer)) + inbufLen := len(inputBuffer) + defer C.free(unsafe.Pointer(inbuf)) + + ret := C.rados_mon_command(c.cluster, + &argv, 1, + inbuf, // bulk input (e.g. crush map) + C.size_t(inbufLen), // length inbuf + &outbuf, // buffer + &outbuflen, // buffer length + &outs, // status string + &outslen) + + if outslen > 0 { + info = C.GoStringN(outs, C.int(outslen)) + C.free(unsafe.Pointer(outs)) + } + if outbuflen > 0 { + buffer = C.GoBytes(unsafe.Pointer(outbuf), C.int(outbuflen)) + C.free(unsafe.Pointer(outbuf)) + } + if ret != 0 { + err = RadosError(int(ret)) + return nil, info, err + } + + return +} diff --git a/vendor/github.com/ceph/go-ceph/rados/doc.go b/vendor/github.com/ceph/go-ceph/rados/doc.go new file mode 100644 index 0000000000..14babe93a7 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rados/doc.go @@ -0,0 +1,4 @@ +/* +Set of wrappers around librados API. +*/ +package rados diff --git a/vendor/github.com/ceph/go-ceph/rados/ioctx.go b/vendor/github.com/ceph/go-ceph/rados/ioctx.go new file mode 100644 index 0000000000..0b7207e67d --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rados/ioctx.go @@ -0,0 +1,890 @@ +package rados + +// #cgo LDFLAGS: -lrados +// #include +// #include +// #include +// +// char* nextChunk(char **idx) { +// char *copy; +// copy = strdup(*idx); +// *idx += strlen(*idx) + 1; +// return copy; +// } +// +// #if __APPLE__ +// #define ceph_time_t __darwin_time_t +// #define ceph_suseconds_t __darwin_suseconds_t +// #elif __GLIBC__ +// #define ceph_time_t __time_t +// #define ceph_suseconds_t __suseconds_t +// #else +// #define ceph_time_t time_t +// #define ceph_suseconds_t suseconds_t +// #endif +import "C" + +import ( + "syscall" + "time" + "unsafe" +) + +// PoolStat represents Ceph pool statistics. +type PoolStat struct { + // space used in bytes + Num_bytes uint64 + // space used in KB + Num_kb uint64 + // number of objects in the pool + Num_objects uint64 + // number of clones of objects + Num_object_clones uint64 + // num_objects * num_replicas + Num_object_copies uint64 + Num_objects_missing_on_primary uint64 + // number of objects found on no OSDs + Num_objects_unfound uint64 + // number of objects replicated fewer times than they should be + // (but found on at least one OSD) + Num_objects_degraded uint64 + Num_rd uint64 + Num_rd_kb uint64 + Num_wr uint64 + Num_wr_kb uint64 +} + +// ObjectStat represents an object stat information +type ObjectStat struct { + // current length in bytes + Size uint64 + // last modification time + ModTime time.Time +} + +// LockInfo represents information on a current Ceph lock +type LockInfo struct { + NumLockers int + Exclusive bool + Tag string + Clients []string + Cookies []string + Addrs []string +} + +// IOContext represents a context for performing I/O within a pool. +type IOContext struct { + ioctx C.rados_ioctx_t +} + +// Pointer returns a uintptr representation of the IOContext. +func (ioctx *IOContext) Pointer() uintptr { + return uintptr(ioctx.ioctx) +} + +// SetNamespace sets the namespace for objects within this IO context (pool). +// Setting namespace to a empty or zero length string sets the pool to the default namespace. +func (ioctx *IOContext) SetNamespace(namespace string) { + var c_ns *C.char + if len(namespace) > 0 { + c_ns = C.CString(namespace) + defer C.free(unsafe.Pointer(c_ns)) + } + C.rados_ioctx_set_namespace(ioctx.ioctx, c_ns) +} + +// Write writes len(data) bytes to the object with key oid starting at byte +// offset offset. It returns an error, if any. +func (ioctx *IOContext) Write(oid string, data []byte, offset uint64) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + dataPointer := unsafe.Pointer(nil) + if len(data) > 0 { + dataPointer = unsafe.Pointer(&data[0]) + } + + ret := C.rados_write(ioctx.ioctx, c_oid, + (*C.char)(dataPointer), + (C.size_t)(len(data)), + (C.uint64_t)(offset)) + + return GetRadosError(int(ret)) +} + +// WriteFull writes len(data) bytes to the object with key oid. +// The object is filled with the provided data. If the object exists, +// it is atomically truncated and then written. It returns an error, if any. +func (ioctx *IOContext) WriteFull(oid string, data []byte) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + ret := C.rados_write_full(ioctx.ioctx, c_oid, + (*C.char)(unsafe.Pointer(&data[0])), + (C.size_t)(len(data))) + return GetRadosError(int(ret)) +} + +// Append appends len(data) bytes to the object with key oid. +// The object is appended with the provided data. If the object exists, +// it is atomically appended to. It returns an error, if any. +func (ioctx *IOContext) Append(oid string, data []byte) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + ret := C.rados_append(ioctx.ioctx, c_oid, + (*C.char)(unsafe.Pointer(&data[0])), + (C.size_t)(len(data))) + return GetRadosError(int(ret)) +} + +// Read reads up to len(data) bytes from the object with key oid starting at byte +// offset offset. It returns the number of bytes read and an error, if any. +func (ioctx *IOContext) Read(oid string, data []byte, offset uint64) (int, error) { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + var buf *C.char + if len(data) > 0 { + buf = (*C.char)(unsafe.Pointer(&data[0])) + } + + ret := C.rados_read( + ioctx.ioctx, + c_oid, + buf, + (C.size_t)(len(data)), + (C.uint64_t)(offset)) + + if ret >= 0 { + return int(ret), nil + } else { + return 0, GetRadosError(int(ret)) + } +} + +// Delete deletes the object with key oid. It returns an error, if any. +func (ioctx *IOContext) Delete(oid string) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + return GetRadosError(int(C.rados_remove(ioctx.ioctx, c_oid))) +} + +// Truncate resizes the object with key oid to size size. If the operation +// enlarges the object, the new area is logically filled with zeroes. If the +// operation shrinks the object, the excess data is removed. It returns an +// error, if any. +func (ioctx *IOContext) Truncate(oid string, size uint64) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + return GetRadosError(int(C.rados_trunc(ioctx.ioctx, c_oid, (C.uint64_t)(size)))) +} + +// Destroy informs librados that the I/O context is no longer in use. +// Resources associated with the context may not be freed immediately, and the +// context should not be used again after calling this method. +func (ioctx *IOContext) Destroy() { + C.rados_ioctx_destroy(ioctx.ioctx) +} + +// Stat returns a set of statistics about the pool associated with this I/O +// context. +func (ioctx *IOContext) GetPoolStats() (stat PoolStat, err error) { + c_stat := C.struct_rados_pool_stat_t{} + ret := C.rados_ioctx_pool_stat(ioctx.ioctx, &c_stat) + if ret < 0 { + return PoolStat{}, GetRadosError(int(ret)) + } else { + return PoolStat{ + Num_bytes: uint64(c_stat.num_bytes), + Num_kb: uint64(c_stat.num_kb), + Num_objects: uint64(c_stat.num_objects), + Num_object_clones: uint64(c_stat.num_object_clones), + Num_object_copies: uint64(c_stat.num_object_copies), + Num_objects_missing_on_primary: uint64(c_stat.num_objects_missing_on_primary), + Num_objects_unfound: uint64(c_stat.num_objects_unfound), + Num_objects_degraded: uint64(c_stat.num_objects_degraded), + Num_rd: uint64(c_stat.num_rd), + Num_rd_kb: uint64(c_stat.num_rd_kb), + Num_wr: uint64(c_stat.num_wr), + Num_wr_kb: uint64(c_stat.num_wr_kb), + }, nil + } +} + +// GetPoolName returns the name of the pool associated with the I/O context. +func (ioctx *IOContext) GetPoolName() (name string, err error) { + buf := make([]byte, 128) + for { + ret := C.rados_ioctx_get_pool_name(ioctx.ioctx, + (*C.char)(unsafe.Pointer(&buf[0])), C.unsigned(len(buf))) + if ret == -C.ERANGE { + buf = make([]byte, len(buf)*2) + continue + } else if ret < 0 { + return "", GetRadosError(int(ret)) + } + name = C.GoStringN((*C.char)(unsafe.Pointer(&buf[0])), ret) + return name, nil + } +} + +// ObjectListFunc is the type of the function called for each object visited +// by ListObjects. +type ObjectListFunc func(oid string) + +// ListObjects lists all of the objects in the pool associated with the I/O +// context, and called the provided listFn function for each object, passing +// to the function the name of the object. Call SetNamespace with +// RadosAllNamespaces before calling this function to return objects from all +// namespaces +func (ioctx *IOContext) ListObjects(listFn ObjectListFunc) error { + var ctx C.rados_list_ctx_t + ret := C.rados_nobjects_list_open(ioctx.ioctx, &ctx) + if ret < 0 { + return GetRadosError(int(ret)) + } + defer func() { C.rados_nobjects_list_close(ctx) }() + + for { + var c_entry *C.char + ret := C.rados_nobjects_list_next(ctx, &c_entry, nil, nil) + if ret == -C.ENOENT { + return nil + } else if ret < 0 { + return GetRadosError(int(ret)) + } + listFn(C.GoString(c_entry)) + } +} + +// Stat returns the size of the object and its last modification time +func (ioctx *IOContext) Stat(object string) (stat ObjectStat, err error) { + var c_psize C.uint64_t + var c_pmtime C.time_t + c_object := C.CString(object) + defer C.free(unsafe.Pointer(c_object)) + + ret := C.rados_stat( + ioctx.ioctx, + c_object, + &c_psize, + &c_pmtime) + + if ret < 0 { + return ObjectStat{}, GetRadosError(int(ret)) + } else { + return ObjectStat{ + Size: uint64(c_psize), + ModTime: time.Unix(int64(c_pmtime), 0), + }, nil + } +} + +// GetXattr gets an xattr with key `name`, it returns the length of +// the key read or an error if not successful +func (ioctx *IOContext) GetXattr(object string, name string, data []byte) (int, error) { + c_object := C.CString(object) + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_object)) + defer C.free(unsafe.Pointer(c_name)) + + ret := C.rados_getxattr( + ioctx.ioctx, + c_object, + c_name, + (*C.char)(unsafe.Pointer(&data[0])), + (C.size_t)(len(data))) + + if ret >= 0 { + return int(ret), nil + } else { + return 0, GetRadosError(int(ret)) + } +} + +// Sets an xattr for an object with key `name` with value as `data` +func (ioctx *IOContext) SetXattr(object string, name string, data []byte) error { + c_object := C.CString(object) + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_object)) + defer C.free(unsafe.Pointer(c_name)) + + ret := C.rados_setxattr( + ioctx.ioctx, + c_object, + c_name, + (*C.char)(unsafe.Pointer(&data[0])), + (C.size_t)(len(data))) + + return GetRadosError(int(ret)) +} + +// function that lists all the xattrs for an object, since xattrs are +// a k-v pair, this function returns a map of k-v pairs on +// success, error code on failure +func (ioctx *IOContext) ListXattrs(oid string) (map[string][]byte, error) { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + var it C.rados_xattrs_iter_t + + ret := C.rados_getxattrs(ioctx.ioctx, c_oid, &it) + if ret < 0 { + return nil, GetRadosError(int(ret)) + } + defer func() { C.rados_getxattrs_end(it) }() + m := make(map[string][]byte) + for { + var c_name, c_val *C.char + var c_len C.size_t + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_val)) + + ret := C.rados_getxattrs_next(it, &c_name, &c_val, &c_len) + if ret < 0 { + return nil, GetRadosError(int(ret)) + } + // rados api returns a null name,val & 0-length upon + // end of iteration + if c_name == nil { + return m, nil // stop iteration + } + m[C.GoString(c_name)] = C.GoBytes(unsafe.Pointer(c_val), (C.int)(c_len)) + } +} + +// Remove an xattr with key `name` from object `oid` +func (ioctx *IOContext) RmXattr(oid string, name string) error { + c_oid := C.CString(oid) + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + + ret := C.rados_rmxattr( + ioctx.ioctx, + c_oid, + c_name) + + return GetRadosError(int(ret)) +} + +// Append the map `pairs` to the omap `oid` +func (ioctx *IOContext) SetOmap(oid string, pairs map[string][]byte) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + var s C.size_t + var c *C.char + ptrSize := unsafe.Sizeof(c) + + c_keys := C.malloc(C.size_t(len(pairs)) * C.size_t(ptrSize)) + c_values := C.malloc(C.size_t(len(pairs)) * C.size_t(ptrSize)) + c_lengths := C.malloc(C.size_t(len(pairs)) * C.size_t(unsafe.Sizeof(s))) + + defer C.free(unsafe.Pointer(c_keys)) + defer C.free(unsafe.Pointer(c_values)) + defer C.free(unsafe.Pointer(c_lengths)) + + i := 0 + for key, value := range pairs { + // key + c_key_ptr := (**C.char)(unsafe.Pointer(uintptr(c_keys) + uintptr(i)*ptrSize)) + *c_key_ptr = C.CString(key) + defer C.free(unsafe.Pointer(*c_key_ptr)) + + // value and its length + c_value_ptr := (**C.char)(unsafe.Pointer(uintptr(c_values) + uintptr(i)*ptrSize)) + + var c_length C.size_t + if len(value) > 0 { + *c_value_ptr = (*C.char)(unsafe.Pointer(&value[0])) + c_length = C.size_t(len(value)) + } else { + *c_value_ptr = nil + c_length = C.size_t(0) + } + + c_length_ptr := (*C.size_t)(unsafe.Pointer(uintptr(c_lengths) + uintptr(i)*ptrSize)) + *c_length_ptr = c_length + + i++ + } + + op := C.rados_create_write_op() + C.rados_write_op_omap_set( + op, + (**C.char)(c_keys), + (**C.char)(c_values), + (*C.size_t)(c_lengths), + C.size_t(len(pairs))) + + ret := C.rados_write_op_operate(op, ioctx.ioctx, c_oid, nil, 0) + C.rados_release_write_op(op) + + return GetRadosError(int(ret)) +} + +// OmapListFunc is the type of the function called for each omap key +// visited by ListOmapValues +type OmapListFunc func(key string, value []byte) + +// Iterate on a set of keys and their values from an omap +// `startAfter`: iterate only on the keys after this specified one +// `filterPrefix`: iterate only on the keys beginning with this prefix +// `maxReturn`: iterate no more than `maxReturn` key/value pairs +// `listFn`: the function called at each iteration +func (ioctx *IOContext) ListOmapValues(oid string, startAfter string, filterPrefix string, maxReturn int64, listFn OmapListFunc) error { + c_oid := C.CString(oid) + c_start_after := C.CString(startAfter) + c_filter_prefix := C.CString(filterPrefix) + c_max_return := C.uint64_t(maxReturn) + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_start_after)) + defer C.free(unsafe.Pointer(c_filter_prefix)) + + op := C.rados_create_read_op() + + var c_iter C.rados_omap_iter_t + var c_prval C.int + C.rados_read_op_omap_get_vals2( + op, + c_start_after, + c_filter_prefix, + c_max_return, + &c_iter, + nil, + &c_prval, + ) + + ret := C.rados_read_op_operate(op, ioctx.ioctx, c_oid, 0) + + if int(ret) != 0 { + return GetRadosError(int(ret)) + } else if int(c_prval) != 0 { + return RadosError(int(c_prval)) + } + + for { + var c_key *C.char + var c_val *C.char + var c_len C.size_t + + ret = C.rados_omap_get_next(c_iter, &c_key, &c_val, &c_len) + + if int(ret) != 0 { + return GetRadosError(int(ret)) + } + + if c_key == nil { + break + } + + listFn(C.GoString(c_key), C.GoBytes(unsafe.Pointer(c_val), C.int(c_len))) + } + + C.rados_omap_get_end(c_iter) + C.rados_release_read_op(op) + + return nil +} + +// Fetch a set of keys and their values from an omap and returns then as a map +// `startAfter`: retrieve only the keys after this specified one +// `filterPrefix`: retrieve only the keys beginning with this prefix +// `maxReturn`: retrieve no more than `maxReturn` key/value pairs +func (ioctx *IOContext) GetOmapValues(oid string, startAfter string, filterPrefix string, maxReturn int64) (map[string][]byte, error) { + omap := map[string][]byte{} + + err := ioctx.ListOmapValues( + oid, startAfter, filterPrefix, maxReturn, + func(key string, value []byte) { + omap[key] = value + }, + ) + + return omap, err +} + +// Fetch all the keys and their values from an omap and returns then as a map +// `startAfter`: retrieve only the keys after this specified one +// `filterPrefix`: retrieve only the keys beginning with this prefix +// `iteratorSize`: internal number of keys to fetch during a read operation +func (ioctx *IOContext) GetAllOmapValues(oid string, startAfter string, filterPrefix string, iteratorSize int64) (map[string][]byte, error) { + omap := map[string][]byte{} + omapSize := 0 + + for { + err := ioctx.ListOmapValues( + oid, startAfter, filterPrefix, iteratorSize, + func(key string, value []byte) { + omap[key] = value + startAfter = key + }, + ) + + if err != nil { + return omap, err + } + + // End of omap + if len(omap) == omapSize { + break + } + + omapSize = len(omap) + } + + return omap, nil +} + +// Remove the specified `keys` from the omap `oid` +func (ioctx *IOContext) RmOmapKeys(oid string, keys []string) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + var c *C.char + ptrSize := unsafe.Sizeof(c) + + c_keys := C.malloc(C.size_t(len(keys)) * C.size_t(ptrSize)) + defer C.free(unsafe.Pointer(c_keys)) + + i := 0 + for _, key := range keys { + c_key_ptr := (**C.char)(unsafe.Pointer(uintptr(c_keys) + uintptr(i)*ptrSize)) + *c_key_ptr = C.CString(key) + defer C.free(unsafe.Pointer(*c_key_ptr)) + i++ + } + + op := C.rados_create_write_op() + C.rados_write_op_omap_rm_keys( + op, + (**C.char)(c_keys), + C.size_t(len(keys))) + + ret := C.rados_write_op_operate(op, ioctx.ioctx, c_oid, nil, 0) + C.rados_release_write_op(op) + + return GetRadosError(int(ret)) +} + +// Clear the omap `oid` +func (ioctx *IOContext) CleanOmap(oid string) error { + c_oid := C.CString(oid) + defer C.free(unsafe.Pointer(c_oid)) + + op := C.rados_create_write_op() + C.rados_write_op_omap_clear(op) + + ret := C.rados_write_op_operate(op, ioctx.ioctx, c_oid, nil, 0) + C.rados_release_write_op(op) + + return GetRadosError(int(ret)) +} + +type Iter struct { + ctx C.rados_list_ctx_t + err error + entry string + namespace string +} + +type IterToken uint32 + +// Return a Iterator object that can be used to list the object names in the current pool +func (ioctx *IOContext) Iter() (*Iter, error) { + iter := Iter{} + if cerr := C.rados_nobjects_list_open(ioctx.ioctx, &iter.ctx); cerr < 0 { + return nil, GetRadosError(int(cerr)) + } + return &iter, nil +} + +// Returns a token marking the current position of the iterator. To be used in combination with Iter.Seek() +func (iter *Iter) Token() IterToken { + return IterToken(C.rados_nobjects_list_get_pg_hash_position(iter.ctx)) +} + +func (iter *Iter) Seek(token IterToken) { + C.rados_nobjects_list_seek(iter.ctx, C.uint32_t(token)) +} + +// Next retrieves the next object name in the pool/namespace iterator. +// Upon a successful invocation (return value of true), the Value method should +// be used to obtain the name of the retrieved object name. When the iterator is +// exhausted, Next returns false. The Err method should used to verify whether the +// end of the iterator was reached, or the iterator received an error. +// +// Example: +// iter := pool.Iter() +// defer iter.Close() +// for iter.Next() { +// fmt.Printf("%v\n", iter.Value()) +// } +// return iter.Err() +// +func (iter *Iter) Next() bool { + var c_entry *C.char + var c_namespace *C.char + if cerr := C.rados_nobjects_list_next(iter.ctx, &c_entry, nil, &c_namespace); cerr < 0 { + iter.err = GetRadosError(int(cerr)) + return false + } + iter.entry = C.GoString(c_entry) + iter.namespace = C.GoString(c_namespace) + return true +} + +// Returns the current value of the iterator (object name), after a successful call to Next. +func (iter *Iter) Value() string { + if iter.err != nil { + return "" + } + return iter.entry +} + +// Returns the namespace associated with the current value of the iterator (object name), after a successful call to Next. +func (iter *Iter) Namespace() string { + if iter.err != nil { + return "" + } + return iter.namespace +} + +// Checks whether the iterator has encountered an error. +func (iter *Iter) Err() error { + if iter.err == RadosErrorNotFound { + return nil + } + return iter.err +} + +// Closes the iterator cursor on the server. Be aware that iterators are not closed automatically +// at the end of iteration. +func (iter *Iter) Close() { + C.rados_nobjects_list_close(iter.ctx) +} + +// Take an exclusive lock on an object. +func (ioctx *IOContext) LockExclusive(oid, name, cookie, desc string, duration time.Duration, flags *byte) (int, error) { + c_oid := C.CString(oid) + c_name := C.CString(name) + c_cookie := C.CString(cookie) + c_desc := C.CString(desc) + + var c_duration C.struct_timeval + if duration != 0 { + tv := syscall.NsecToTimeval(duration.Nanoseconds()) + c_duration = C.struct_timeval{tv_sec: C.ceph_time_t(tv.Sec), tv_usec: C.ceph_suseconds_t(tv.Usec)} + } + + var c_flags C.uint8_t + if flags != nil { + c_flags = C.uint8_t(*flags) + } + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_cookie)) + defer C.free(unsafe.Pointer(c_desc)) + + ret := C.rados_lock_exclusive( + ioctx.ioctx, + c_oid, + c_name, + c_cookie, + c_desc, + &c_duration, + c_flags) + + // 0 on success, negative error code on failure + // -EBUSY if the lock is already held by another (client, cookie) pair + // -EEXIST if the lock is already held by the same (client, cookie) pair + + switch ret { + case 0: + return int(ret), nil + case -C.EBUSY: + return int(ret), nil + case -C.EEXIST: + return int(ret), nil + default: + return int(ret), RadosError(int(ret)) + } +} + +// Take a shared lock on an object. +func (ioctx *IOContext) LockShared(oid, name, cookie, tag, desc string, duration time.Duration, flags *byte) (int, error) { + c_oid := C.CString(oid) + c_name := C.CString(name) + c_cookie := C.CString(cookie) + c_tag := C.CString(tag) + c_desc := C.CString(desc) + + var c_duration C.struct_timeval + if duration != 0 { + tv := syscall.NsecToTimeval(duration.Nanoseconds()) + c_duration = C.struct_timeval{tv_sec: C.ceph_time_t(tv.Sec), tv_usec: C.ceph_suseconds_t(tv.Usec)} + } + + var c_flags C.uint8_t + if flags != nil { + c_flags = C.uint8_t(*flags) + } + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_cookie)) + defer C.free(unsafe.Pointer(c_tag)) + defer C.free(unsafe.Pointer(c_desc)) + + ret := C.rados_lock_shared( + ioctx.ioctx, + c_oid, + c_name, + c_cookie, + c_tag, + c_desc, + &c_duration, + c_flags) + + // 0 on success, negative error code on failure + // -EBUSY if the lock is already held by another (client, cookie) pair + // -EEXIST if the lock is already held by the same (client, cookie) pair + + switch ret { + case 0: + return int(ret), nil + case -C.EBUSY: + return int(ret), nil + case -C.EEXIST: + return int(ret), nil + default: + return int(ret), RadosError(int(ret)) + } +} + +// Release a shared or exclusive lock on an object. +func (ioctx *IOContext) Unlock(oid, name, cookie string) (int, error) { + c_oid := C.CString(oid) + c_name := C.CString(name) + c_cookie := C.CString(cookie) + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_cookie)) + + // 0 on success, negative error code on failure + // -ENOENT if the lock is not held by the specified (client, cookie) pair + + ret := C.rados_unlock( + ioctx.ioctx, + c_oid, + c_name, + c_cookie) + + switch ret { + case 0: + return int(ret), nil + case -C.ENOENT: + return int(ret), nil + default: + return int(ret), RadosError(int(ret)) + } +} + +// List clients that have locked the named object lock and information about the lock. +// The number of bytes required in each buffer is put in the corresponding size out parameter. +// If any of the provided buffers are too short, -ERANGE is returned after these sizes are filled in. +func (ioctx *IOContext) ListLockers(oid, name string) (*LockInfo, error) { + c_oid := C.CString(oid) + c_name := C.CString(name) + + c_tag := (*C.char)(C.malloc(C.size_t(1024))) + c_clients := (*C.char)(C.malloc(C.size_t(1024))) + c_cookies := (*C.char)(C.malloc(C.size_t(1024))) + c_addrs := (*C.char)(C.malloc(C.size_t(1024))) + + var c_exclusive C.int + c_tag_len := C.size_t(1024) + c_clients_len := C.size_t(1024) + c_cookies_len := C.size_t(1024) + c_addrs_len := C.size_t(1024) + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_tag)) + defer C.free(unsafe.Pointer(c_clients)) + defer C.free(unsafe.Pointer(c_cookies)) + defer C.free(unsafe.Pointer(c_addrs)) + + ret := C.rados_list_lockers( + ioctx.ioctx, + c_oid, + c_name, + &c_exclusive, + c_tag, + &c_tag_len, + c_clients, + &c_clients_len, + c_cookies, + &c_cookies_len, + c_addrs, + &c_addrs_len) + + splitCString := func(items *C.char, itemsLen C.size_t) []string { + currLen := 0 + clients := []string{} + for currLen < int(itemsLen) { + client := C.GoString(C.nextChunk(&items)) + clients = append(clients, client) + currLen += len(client) + 1 + } + return clients + } + + if ret < 0 { + return nil, RadosError(int(ret)) + } else { + return &LockInfo{int(ret), c_exclusive == 1, C.GoString(c_tag), splitCString(c_clients, c_clients_len), splitCString(c_cookies, c_cookies_len), splitCString(c_addrs, c_addrs_len)}, nil + } +} + +// Releases a shared or exclusive lock on an object, which was taken by the specified client. +func (ioctx *IOContext) BreakLock(oid, name, client, cookie string) (int, error) { + c_oid := C.CString(oid) + c_name := C.CString(name) + c_client := C.CString(client) + c_cookie := C.CString(cookie) + + defer C.free(unsafe.Pointer(c_oid)) + defer C.free(unsafe.Pointer(c_name)) + defer C.free(unsafe.Pointer(c_client)) + defer C.free(unsafe.Pointer(c_cookie)) + + // 0 on success, negative error code on failure + // -ENOENT if the lock is not held by the specified (client, cookie) pair + // -EINVAL if the client cannot be parsed + + ret := C.rados_break_lock( + ioctx.ioctx, + c_oid, + c_name, + c_client, + c_cookie) + + switch ret { + case 0: + return int(ret), nil + case -C.ENOENT: + return int(ret), nil + case -C.EINVAL: // -EINVAL + return int(ret), nil + default: + return int(ret), RadosError(int(ret)) + } +} diff --git a/vendor/github.com/ceph/go-ceph/rados/rados.go b/vendor/github.com/ceph/go-ceph/rados/rados.go new file mode 100644 index 0000000000..b6af97af9b --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rados/rados.go @@ -0,0 +1,85 @@ +package rados + +// #cgo LDFLAGS: -lrados +// #include +// #include +// #include +import "C" + +import ( + "fmt" + "unsafe" +) + +type RadosError int + +func (e RadosError) Error() string { + return fmt.Sprintf("rados: %s", C.GoString(C.strerror(C.int(-e)))) +} + +var RadosAllNamespaces = C.LIBRADOS_ALL_NSPACES + +var RadosErrorNotFound = RadosError(-C.ENOENT) +var RadosErrorPermissionDenied = RadosError(-C.EPERM) + +func GetRadosError(err int) error { + if err == 0 { + return nil + } + return RadosError(err) +} + +// Version returns the major, minor, and patch components of the version of +// the RADOS library linked against. +func Version() (int, int, int) { + var c_major, c_minor, c_patch C.int + C.rados_version(&c_major, &c_minor, &c_patch) + return int(c_major), int(c_minor), int(c_patch) +} + +func makeConn() *Conn { + return &Conn{connected: false} +} + +func newConn(user *C.char) (*Conn, error) { + conn := makeConn() + ret := C.rados_create(&conn.cluster, user) + + if ret == 0 { + return conn, nil + } else { + return nil, RadosError(int(ret)) + } +} + +// NewConn creates a new connection object. It returns the connection and an +// error, if any. +func NewConn() (*Conn, error) { + return newConn(nil) +} + +// NewConnWithUser creates a new connection object with a custom username. +// It returns the connection and an error, if any. +func NewConnWithUser(user string) (*Conn, error) { + c_user := C.CString(user) + defer C.free(unsafe.Pointer(c_user)) + return newConn(c_user) +} + +// NewConnWithClusterAndUser creates a new connection object for a specific cluster and username. +// It returns the connection and an error, if any. +func NewConnWithClusterAndUser(clusterName string, userName string) (*Conn, error) { + c_cluster_name := C.CString(clusterName) + defer C.free(unsafe.Pointer(c_cluster_name)) + + c_name := C.CString(userName) + defer C.free(unsafe.Pointer(c_name)) + + conn := makeConn() + ret := C.rados_create2(&conn.cluster, c_cluster_name, c_name, 0) + if ret == 0 { + return conn, nil + } else { + return nil, RadosError(int(ret)) + } +} diff --git a/vendor/github.com/ceph/go-ceph/rbd/doc.go b/vendor/github.com/ceph/go-ceph/rbd/doc.go new file mode 100644 index 0000000000..70e70f3c63 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rbd/doc.go @@ -0,0 +1,4 @@ +/* +Wrappers around librbd. +*/ +package rbd diff --git a/vendor/github.com/ceph/go-ceph/rbd/rbd.go b/vendor/github.com/ceph/go-ceph/rbd/rbd.go new file mode 100644 index 0000000000..e754fde648 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rbd/rbd.go @@ -0,0 +1,991 @@ +package rbd + +// #cgo LDFLAGS: -lrbd +// #include +// #include +// #include +// #include +// #include +import "C" + +import ( + "bytes" + "errors" + "fmt" + "github.com/ceph/go-ceph/rados" + "io" + "time" + "unsafe" +) + +const ( + // RBD features. + RbdFeatureLayering = C.RBD_FEATURE_LAYERING + RbdFeatureStripingV2 = C.RBD_FEATURE_STRIPINGV2 + RbdFeatureExclusiveLock = C.RBD_FEATURE_EXCLUSIVE_LOCK + RbdFeatureObjectMap = C.RBD_FEATURE_OBJECT_MAP + RbdFeatureFastDiff = C.RBD_FEATURE_FAST_DIFF + RbdFeatureDeepFlatten = C.RBD_FEATURE_DEEP_FLATTEN + RbdFeatureJournaling = C.RBD_FEATURE_JOURNALING + RbdFeatureDataPool = C.RBD_FEATURE_DATA_POOL + + RbdFeaturesDefault = C.RBD_FEATURES_DEFAULT + + // Features that make an image inaccessible for read or write by clients that don't understand + // them. + RbdFeaturesIncompatible = C.RBD_FEATURES_INCOMPATIBLE + + // Features that make an image unwritable by clients that don't understand them. + RbdFeaturesRwIncompatible = C.RBD_FEATURES_RW_INCOMPATIBLE + + // Features that may be dynamically enabled or disabled. + RbdFeaturesMutable = C.RBD_FEATURES_MUTABLE + + // Features that only work when used with a single client using the image for writes. + RbdFeaturesSingleClient = C.RBD_FEATURES_SINGLE_CLIENT +) + +// +type RBDError int + +var ( + RbdErrorImageNotOpen = errors.New("RBD image not open") + RbdErrorNotFound = errors.New("RBD image not found") +) + +// +type ImageInfo struct { + Size uint64 + Obj_size uint64 + Num_objs uint64 + Order int + Block_name_prefix string + Parent_pool int64 + Parent_name string +} + +// +type SnapInfo struct { + Id uint64 + Size uint64 + Name string +} + +// +type Locker struct { + Client string + Cookie string + Addr string +} + +// +type Image struct { + io.Reader + io.Writer + io.Seeker + io.ReaderAt + io.WriterAt + name string + offset int64 + ioctx *rados.IOContext + image C.rbd_image_t +} + +// +type Snapshot struct { + image *Image + name string +} + +// TrashInfo contains information about trashed RBDs. +type TrashInfo struct { + Id string // Id string, required to remove / restore trashed RBDs. + Name string // Original name of trashed RBD. + DeletionTime time.Time // Date / time at which the RBD was moved to the trash. + DefermentEndTime time.Time // Date / time after which the trashed RBD may be permanently deleted. +} + +// +func split(buf []byte) (values []string) { + tmp := bytes.Split(buf[:len(buf)-1], []byte{0}) + for _, s := range tmp { + if len(s) > 0 { + go_s := C.GoString((*C.char)(unsafe.Pointer(&s[0]))) + values = append(values, go_s) + } + } + return values +} + +// +func (e RBDError) Error() string { + return fmt.Sprintf("rbd: ret=%d", e) +} + +// +func GetError(err C.int) error { + if err != 0 { + if err == -C.ENOENT { + return RbdErrorNotFound + } + return RBDError(err) + } else { + return nil + } +} + +// +func Version() (int, int, int) { + var c_major, c_minor, c_patch C.int + C.rbd_version(&c_major, &c_minor, &c_patch) + return int(c_major), int(c_minor), int(c_patch) +} + +// GetImageNames returns the list of current RBD images. +func GetImageNames(ioctx *rados.IOContext) (names []string, err error) { + buf := make([]byte, 4096) + for { + size := C.size_t(len(buf)) + ret := C.rbd_list(C.rados_ioctx_t(ioctx.Pointer()), + (*C.char)(unsafe.Pointer(&buf[0])), &size) + if ret == -C.ERANGE { + buf = make([]byte, size) + continue + } else if ret < 0 { + return nil, RBDError(ret) + } + tmp := bytes.Split(buf[:size-1], []byte{0}) + for _, s := range tmp { + if len(s) > 0 { + name := C.GoString((*C.char)(unsafe.Pointer(&s[0]))) + names = append(names, name) + } + } + return names, nil + } +} + +// +func GetImage(ioctx *rados.IOContext, name string) *Image { + return &Image{ + ioctx: ioctx, + name: name, + } +} + +// int rbd_create(rados_ioctx_t io, const char *name, uint64_t size, int *order); +// int rbd_create2(rados_ioctx_t io, const char *name, uint64_t size, +// uint64_t features, int *order); +// int rbd_create3(rados_ioctx_t io, const char *name, uint64_t size, +// uint64_t features, int *order, +// uint64_t stripe_unit, uint64_t stripe_count); +func Create(ioctx *rados.IOContext, name string, size uint64, order int, + args ...uint64) (image *Image, err error) { + var ret C.int + + c_order := C.int(order) + c_name := C.CString(name) + + defer C.free(unsafe.Pointer(c_name)) + + switch len(args) { + case 3: + ret = C.rbd_create3(C.rados_ioctx_t(ioctx.Pointer()), + c_name, C.uint64_t(size), + C.uint64_t(args[0]), &c_order, + C.uint64_t(args[1]), C.uint64_t(args[2])) + case 1: + ret = C.rbd_create2(C.rados_ioctx_t(ioctx.Pointer()), + c_name, C.uint64_t(size), + C.uint64_t(args[0]), &c_order) + case 0: + ret = C.rbd_create(C.rados_ioctx_t(ioctx.Pointer()), + c_name, C.uint64_t(size), &c_order) + default: + return nil, errors.New("Wrong number of argument") + } + + if ret < 0 { + return nil, RBDError(ret) + } + + return &Image{ + ioctx: ioctx, + name: name, + }, nil +} + +// int rbd_clone(rados_ioctx_t p_ioctx, const char *p_name, +// const char *p_snapname, rados_ioctx_t c_ioctx, +// const char *c_name, uint64_t features, int *c_order); +// int rbd_clone2(rados_ioctx_t p_ioctx, const char *p_name, +// const char *p_snapname, rados_ioctx_t c_ioctx, +// const char *c_name, uint64_t features, int *c_order, +// uint64_t stripe_unit, int stripe_count); +func (image *Image) Clone(snapname string, c_ioctx *rados.IOContext, c_name string, features uint64, order int) (*Image, error) { + c_order := C.int(order) + c_p_name := C.CString(image.name) + c_p_snapname := C.CString(snapname) + c_c_name := C.CString(c_name) + + defer C.free(unsafe.Pointer(c_p_name)) + defer C.free(unsafe.Pointer(c_p_snapname)) + defer C.free(unsafe.Pointer(c_c_name)) + + ret := C.rbd_clone(C.rados_ioctx_t(image.ioctx.Pointer()), + c_p_name, c_p_snapname, + C.rados_ioctx_t(c_ioctx.Pointer()), + c_c_name, C.uint64_t(features), &c_order) + if ret < 0 { + return nil, RBDError(ret) + } + + return &Image{ + ioctx: c_ioctx, + name: c_name, + }, nil +} + +// int rbd_remove(rados_ioctx_t io, const char *name); +// int rbd_remove_with_progress(rados_ioctx_t io, const char *name, +// librbd_progress_fn_t cb, void *cbdata); +func (image *Image) Remove() error { + c_name := C.CString(image.name) + defer C.free(unsafe.Pointer(c_name)) + return GetError(C.rbd_remove(C.rados_ioctx_t(image.ioctx.Pointer()), c_name)) +} + +// Trash will move an image into the RBD trash, where it will be protected (i.e., salvageable) for +// at least the specified delay. +func (image *Image) Trash(delay time.Duration) error { + c_name := C.CString(image.name) + defer C.free(unsafe.Pointer(c_name)) + + return GetError(C.rbd_trash_move(C.rados_ioctx_t(image.ioctx.Pointer()), c_name, + C.uint64_t(delay.Seconds()))) +} + +// int rbd_rename(rados_ioctx_t src_io_ctx, const char *srcname, const char *destname); +func (image *Image) Rename(destname string) error { + c_srcname := C.CString(image.name) + c_destname := C.CString(destname) + + defer C.free(unsafe.Pointer(c_srcname)) + defer C.free(unsafe.Pointer(c_destname)) + + err := RBDError(C.rbd_rename(C.rados_ioctx_t(image.ioctx.Pointer()), + c_srcname, c_destname)) + if err == 0 { + image.name = destname + return nil + } + return err +} + +// int rbd_open(rados_ioctx_t io, const char *name, rbd_image_t *image, const char *snap_name); +// int rbd_open_read_only(rados_ioctx_t io, const char *name, rbd_image_t *image, +// const char *snap_name); +func (image *Image) Open(args ...interface{}) error { + var c_image C.rbd_image_t + var c_snap_name *C.char + var ret C.int + var read_only bool + + c_name := C.CString(image.name) + + defer C.free(unsafe.Pointer(c_name)) + for _, arg := range args { + switch t := arg.(type) { + case string: + if t != "" { + c_snap_name = C.CString(t) + defer C.free(unsafe.Pointer(c_snap_name)) + } + case bool: + read_only = t + default: + return errors.New("Unexpected argument") + } + } + + if read_only { + ret = C.rbd_open_read_only(C.rados_ioctx_t(image.ioctx.Pointer()), c_name, + &c_image, c_snap_name) + } else { + ret = C.rbd_open(C.rados_ioctx_t(image.ioctx.Pointer()), c_name, + &c_image, c_snap_name) + } + + image.image = c_image + + return GetError(ret) +} + +// int rbd_close(rbd_image_t image); +func (image *Image) Close() error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + if ret := C.rbd_close(image.image); ret != 0 { + return RBDError(ret) + } + + image.image = nil + return nil +} + +// int rbd_resize(rbd_image_t image, uint64_t size); +func (image *Image) Resize(size uint64) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + return GetError(C.rbd_resize(image.image, C.uint64_t(size))) +} + +// int rbd_stat(rbd_image_t image, rbd_image_info_t *info, size_t infosize); +func (image *Image) Stat() (info *ImageInfo, err error) { + if image.image == nil { + return nil, RbdErrorImageNotOpen + } + + var c_stat C.rbd_image_info_t + + if ret := C.rbd_stat(image.image, &c_stat, C.size_t(unsafe.Sizeof(info))); ret < 0 { + return info, RBDError(ret) + } + + return &ImageInfo{ + Size: uint64(c_stat.size), + Obj_size: uint64(c_stat.obj_size), + Num_objs: uint64(c_stat.num_objs), + Order: int(c_stat.order), + Block_name_prefix: C.GoString((*C.char)(&c_stat.block_name_prefix[0])), + Parent_pool: int64(c_stat.parent_pool), + Parent_name: C.GoString((*C.char)(&c_stat.parent_name[0]))}, nil +} + +// int rbd_get_old_format(rbd_image_t image, uint8_t *old); +func (image *Image) IsOldFormat() (old_format bool, err error) { + if image.image == nil { + return false, RbdErrorImageNotOpen + } + + var c_old_format C.uint8_t + ret := C.rbd_get_old_format(image.image, + &c_old_format) + if ret < 0 { + return false, RBDError(ret) + } + + return c_old_format != 0, nil +} + +// int rbd_size(rbd_image_t image, uint64_t *size); +func (image *Image) GetSize() (size uint64, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if ret := C.rbd_get_size(image.image, (*C.uint64_t)(&size)); ret < 0 { + return 0, RBDError(ret) + } + + return size, nil +} + +// int rbd_get_features(rbd_image_t image, uint64_t *features); +func (image *Image) GetFeatures() (features uint64, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if ret := C.rbd_get_features(image.image, (*C.uint64_t)(&features)); ret < 0 { + return 0, RBDError(ret) + } + + return features, nil +} + +// int rbd_get_stripe_unit(rbd_image_t image, uint64_t *stripe_unit); +func (image *Image) GetStripeUnit() (stripe_unit uint64, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if ret := C.rbd_get_stripe_unit(image.image, (*C.uint64_t)(&stripe_unit)); ret < 0 { + return 0, RBDError(ret) + } + + return stripe_unit, nil +} + +// int rbd_get_stripe_count(rbd_image_t image, uint64_t *stripe_count); +func (image *Image) GetStripeCount() (stripe_count uint64, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if ret := C.rbd_get_stripe_count(image.image, (*C.uint64_t)(&stripe_count)); ret < 0 { + return 0, RBDError(ret) + } + + return stripe_count, nil +} + +// int rbd_get_overlap(rbd_image_t image, uint64_t *overlap); +func (image *Image) GetOverlap() (overlap uint64, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if ret := C.rbd_get_overlap(image.image, (*C.uint64_t)(&overlap)); ret < 0 { + return overlap, RBDError(ret) + } + + return overlap, nil +} + +// int rbd_copy(rbd_image_t image, rados_ioctx_t dest_io_ctx, const char *destname); +// int rbd_copy2(rbd_image_t src, rbd_image_t dest); +// int rbd_copy_with_progress(rbd_image_t image, rados_ioctx_t dest_p, const char *destname, +// librbd_progress_fn_t cb, void *cbdata); +// int rbd_copy_with_progress2(rbd_image_t src, rbd_image_t dest, +// librbd_progress_fn_t cb, void *cbdata); +func (image *Image) Copy(args ...interface{}) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + switch t := args[0].(type) { + case rados.IOContext: + switch t2 := args[1].(type) { + case string: + c_destname := C.CString(t2) + defer C.free(unsafe.Pointer(c_destname)) + return RBDError(C.rbd_copy(image.image, + C.rados_ioctx_t(t.Pointer()), + c_destname)) + default: + return errors.New("Must specify destname") + } + case Image: + var dest Image = t + if dest.image == nil { + return errors.New(fmt.Sprintf("RBD image %s is not open", dest.name)) + } + return GetError(C.rbd_copy2(image.image, dest.image)) + default: + return errors.New("Must specify either destination pool or destination image") + } +} + +// int rbd_flatten(rbd_image_t image); +func (image *Image) Flatten() error { + if image.image == nil { + return errors.New(fmt.Sprintf("RBD image %s is not open", image.name)) + } + + return GetError(C.rbd_flatten(image.image)) +} + +// ssize_t rbd_list_children(rbd_image_t image, char *pools, size_t *pools_len, +// char *images, size_t *images_len); +func (image *Image) ListChildren() (pools []string, images []string, err error) { + if image.image == nil { + return nil, nil, RbdErrorImageNotOpen + } + + var c_pools_len, c_images_len C.size_t + + ret := C.rbd_list_children(image.image, + nil, &c_pools_len, + nil, &c_images_len) + if ret == 0 { + return nil, nil, nil + } + if ret < 0 && ret != -C.ERANGE { + return nil, nil, RBDError(ret) + } + + pools_buf := make([]byte, c_pools_len) + images_buf := make([]byte, c_images_len) + + ret = C.rbd_list_children(image.image, + (*C.char)(unsafe.Pointer(&pools_buf[0])), + &c_pools_len, + (*C.char)(unsafe.Pointer(&images_buf[0])), + &c_images_len) + if ret < 0 { + return nil, nil, RBDError(ret) + } + + tmp := bytes.Split(pools_buf[:c_pools_len-1], []byte{0}) + for _, s := range tmp { + if len(s) > 0 { + name := C.GoString((*C.char)(unsafe.Pointer(&s[0]))) + pools = append(pools, name) + } + } + + tmp = bytes.Split(images_buf[:c_images_len-1], []byte{0}) + for _, s := range tmp { + if len(s) > 0 { + name := C.GoString((*C.char)(unsafe.Pointer(&s[0]))) + images = append(images, name) + } + } + + return pools, images, nil +} + +// ssize_t rbd_list_lockers(rbd_image_t image, int *exclusive, +// char *tag, size_t *tag_len, +// char *clients, size_t *clients_len, +// char *cookies, size_t *cookies_len, +// char *addrs, size_t *addrs_len); +func (image *Image) ListLockers() (tag string, lockers []Locker, err error) { + if image.image == nil { + return "", nil, RbdErrorImageNotOpen + } + + var c_exclusive C.int + var c_tag_len, c_clients_len, c_cookies_len, c_addrs_len C.size_t + var c_locker_cnt C.ssize_t + + C.rbd_list_lockers(image.image, &c_exclusive, + nil, (*C.size_t)(&c_tag_len), + nil, (*C.size_t)(&c_clients_len), + nil, (*C.size_t)(&c_cookies_len), + nil, (*C.size_t)(&c_addrs_len)) + + // no locker held on rbd image when either c_clients_len, + // c_cookies_len or c_addrs_len is *0*, so just quickly returned + if int(c_clients_len) == 0 || int(c_cookies_len) == 0 || + int(c_addrs_len) == 0 { + lockers = make([]Locker, 0) + return "", lockers, nil + } + + tag_buf := make([]byte, c_tag_len) + clients_buf := make([]byte, c_clients_len) + cookies_buf := make([]byte, c_cookies_len) + addrs_buf := make([]byte, c_addrs_len) + + c_locker_cnt = C.rbd_list_lockers(image.image, &c_exclusive, + (*C.char)(unsafe.Pointer(&tag_buf[0])), (*C.size_t)(&c_tag_len), + (*C.char)(unsafe.Pointer(&clients_buf[0])), (*C.size_t)(&c_clients_len), + (*C.char)(unsafe.Pointer(&cookies_buf[0])), (*C.size_t)(&c_cookies_len), + (*C.char)(unsafe.Pointer(&addrs_buf[0])), (*C.size_t)(&c_addrs_len)) + + // rbd_list_lockers returns negative value for errors + // and *0* means no locker held on rbd image. + // but *0* is unexpected here because first rbd_list_lockers already + // dealt with no locker case + if int(c_locker_cnt) <= 0 { + return "", nil, RBDError(c_locker_cnt) + } + + clients := split(clients_buf) + cookies := split(cookies_buf) + addrs := split(addrs_buf) + + lockers = make([]Locker, c_locker_cnt) + for i := 0; i < int(c_locker_cnt); i++ { + lockers[i] = Locker{Client: clients[i], + Cookie: cookies[i], + Addr: addrs[i]} + } + + return string(tag_buf), lockers, nil +} + +// int rbd_lock_exclusive(rbd_image_t image, const char *cookie); +func (image *Image) LockExclusive(cookie string) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + c_cookie := C.CString(cookie) + defer C.free(unsafe.Pointer(c_cookie)) + + return GetError(C.rbd_lock_exclusive(image.image, c_cookie)) +} + +// int rbd_lock_shared(rbd_image_t image, const char *cookie, const char *tag); +func (image *Image) LockShared(cookie string, tag string) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + c_cookie := C.CString(cookie) + c_tag := C.CString(tag) + defer C.free(unsafe.Pointer(c_cookie)) + defer C.free(unsafe.Pointer(c_tag)) + + return GetError(C.rbd_lock_shared(image.image, c_cookie, c_tag)) +} + +// int rbd_lock_shared(rbd_image_t image, const char *cookie, const char *tag); +func (image *Image) Unlock(cookie string) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + c_cookie := C.CString(cookie) + defer C.free(unsafe.Pointer(c_cookie)) + + return GetError(C.rbd_unlock(image.image, c_cookie)) +} + +// int rbd_break_lock(rbd_image_t image, const char *client, const char *cookie); +func (image *Image) BreakLock(client string, cookie string) error { + if image.image == nil { + return RbdErrorImageNotOpen + } + + c_client := C.CString(client) + c_cookie := C.CString(cookie) + defer C.free(unsafe.Pointer(c_client)) + defer C.free(unsafe.Pointer(c_cookie)) + + return GetError(C.rbd_break_lock(image.image, c_client, c_cookie)) +} + +// ssize_t rbd_read(rbd_image_t image, uint64_t ofs, size_t len, char *buf); +// TODO: int64_t rbd_read_iterate(rbd_image_t image, uint64_t ofs, size_t len, +// int (*cb)(uint64_t, size_t, const char *, void *), void *arg); +// TODO: int rbd_read_iterate2(rbd_image_t image, uint64_t ofs, uint64_t len, +// int (*cb)(uint64_t, size_t, const char *, void *), void *arg); +// TODO: int rbd_diff_iterate(rbd_image_t image, +// const char *fromsnapname, +// uint64_t ofs, uint64_t len, +// int (*cb)(uint64_t, size_t, int, void *), void *arg); +func (image *Image) Read(data []byte) (n int, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if len(data) == 0 { + return 0, nil + } + + ret := int(C.rbd_read( + image.image, + (C.uint64_t)(image.offset), + (C.size_t)(len(data)), + (*C.char)(unsafe.Pointer(&data[0])))) + + if ret < 0 { + return 0, RBDError(ret) + } + + image.offset += int64(ret) + if ret < n { + return ret, io.EOF + } + + return ret, nil +} + +// ssize_t rbd_write(rbd_image_t image, uint64_t ofs, size_t len, const char *buf); +func (image *Image) Write(data []byte) (n int, err error) { + ret := int(C.rbd_write(image.image, C.uint64_t(image.offset), + C.size_t(len(data)), (*C.char)(unsafe.Pointer(&data[0])))) + + if ret >= 0 { + image.offset += int64(ret) + } + + if ret != len(data) { + err = RBDError(-C.EPERM) + } + + return ret, err +} + +func (image *Image) Seek(offset int64, whence int) (int64, error) { + switch whence { + case 0: + image.offset = offset + case 1: + image.offset += offset + case 2: + stats, err := image.Stat() + if err != nil { + return 0, err + } + image.offset = int64(stats.Size) - offset + default: + return 0, errors.New("Wrong value for whence") + } + return image.offset, nil +} + +// int rbd_discard(rbd_image_t image, uint64_t ofs, uint64_t len); +func (image *Image) Discard(ofs uint64, length uint64) error { + return RBDError(C.rbd_discard(image.image, C.uint64_t(ofs), + C.uint64_t(length))) +} + +func (image *Image) ReadAt(data []byte, off int64) (n int, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if len(data) == 0 { + return 0, nil + } + + ret := int(C.rbd_read( + image.image, + (C.uint64_t)(off), + (C.size_t)(len(data)), + (*C.char)(unsafe.Pointer(&data[0])))) + + if ret < 0 { + return 0, RBDError(ret) + } + + if ret < n { + return ret, io.EOF + } + + return ret, nil +} + +func (image *Image) WriteAt(data []byte, off int64) (n int, err error) { + if image.image == nil { + return 0, RbdErrorImageNotOpen + } + + if len(data) == 0 { + return 0, nil + } + + ret := int(C.rbd_write(image.image, C.uint64_t(off), + C.size_t(len(data)), (*C.char)(unsafe.Pointer(&data[0])))) + + if ret != len(data) { + err = RBDError(-C.EPERM) + } + + return ret, err +} + +// int rbd_flush(rbd_image_t image); +func (image *Image) Flush() error { + return GetError(C.rbd_flush(image.image)) +} + +// int rbd_snap_list(rbd_image_t image, rbd_snap_info_t *snaps, int *max_snaps); +// void rbd_snap_list_end(rbd_snap_info_t *snaps); +func (image *Image) GetSnapshotNames() (snaps []SnapInfo, err error) { + if image.image == nil { + return nil, RbdErrorImageNotOpen + } + + var c_max_snaps C.int + + ret := C.rbd_snap_list(image.image, nil, &c_max_snaps) + + c_snaps := make([]C.rbd_snap_info_t, c_max_snaps) + snaps = make([]SnapInfo, c_max_snaps) + + ret = C.rbd_snap_list(image.image, + &c_snaps[0], &c_max_snaps) + if ret < 0 { + return nil, RBDError(ret) + } + + for i, s := range c_snaps { + snaps[i] = SnapInfo{Id: uint64(s.id), + Size: uint64(s.size), + Name: C.GoString(s.name)} + } + + C.rbd_snap_list_end(&c_snaps[0]) + return snaps[:len(snaps)-1], nil +} + +// int rbd_snap_create(rbd_image_t image, const char *snapname); +func (image *Image) CreateSnapshot(snapname string) (*Snapshot, error) { + if image.image == nil { + return nil, RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapname) + defer C.free(unsafe.Pointer(c_snapname)) + + ret := C.rbd_snap_create(image.image, c_snapname) + if ret < 0 { + return nil, RBDError(ret) + } + + return &Snapshot{ + image: image, + name: snapname, + }, nil +} + +// +func (image *Image) GetSnapshot(snapname string) *Snapshot { + return &Snapshot{ + image: image, + name: snapname, + } +} + +// int rbd_get_parent_info(rbd_image_t image, +// char *parent_pool_name, size_t ppool_namelen, char *parent_name, +// size_t pnamelen, char *parent_snap_name, size_t psnap_namelen) +func (image *Image) GetParentInfo(p_pool, p_name, p_snapname []byte) error { + ret := C.rbd_get_parent_info( + image.image, + (*C.char)(unsafe.Pointer(&p_pool[0])), + (C.size_t)(len(p_pool)), + (*C.char)(unsafe.Pointer(&p_name[0])), + (C.size_t)(len(p_name)), + (*C.char)(unsafe.Pointer(&p_snapname[0])), + (C.size_t)(len(p_snapname))) + if ret == 0 { + return nil + } else { + return RBDError(ret) + } +} + +// int rbd_snap_remove(rbd_image_t image, const char *snapname); +func (snapshot *Snapshot) Remove() error { + if snapshot.image.image == nil { + return RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + return GetError(C.rbd_snap_remove(snapshot.image.image, c_snapname)) +} + +// int rbd_snap_rollback(rbd_image_t image, const char *snapname); +// int rbd_snap_rollback_with_progress(rbd_image_t image, const char *snapname, +// librbd_progress_fn_t cb, void *cbdata); +func (snapshot *Snapshot) Rollback() error { + if snapshot.image.image == nil { + return RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + return GetError(C.rbd_snap_rollback(snapshot.image.image, c_snapname)) +} + +// int rbd_snap_protect(rbd_image_t image, const char *snap_name); +func (snapshot *Snapshot) Protect() error { + if snapshot.image.image == nil { + return RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + return GetError(C.rbd_snap_protect(snapshot.image.image, c_snapname)) +} + +// int rbd_snap_unprotect(rbd_image_t image, const char *snap_name); +func (snapshot *Snapshot) Unprotect() error { + if snapshot.image.image == nil { + return RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + return GetError(C.rbd_snap_unprotect(snapshot.image.image, c_snapname)) +} + +// int rbd_snap_is_protected(rbd_image_t image, const char *snap_name, +// int *is_protected); +func (snapshot *Snapshot) IsProtected() (bool, error) { + if snapshot.image.image == nil { + return false, RbdErrorImageNotOpen + } + + var c_is_protected C.int + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + ret := C.rbd_snap_is_protected(snapshot.image.image, c_snapname, + &c_is_protected) + if ret < 0 { + return false, RBDError(ret) + } + + return c_is_protected != 0, nil +} + +// int rbd_snap_set(rbd_image_t image, const char *snapname); +func (snapshot *Snapshot) Set() error { + if snapshot.image.image == nil { + return RbdErrorImageNotOpen + } + + c_snapname := C.CString(snapshot.name) + defer C.free(unsafe.Pointer(c_snapname)) + + return GetError(C.rbd_snap_set(snapshot.image.image, c_snapname)) +} + +// GetTrashList returns a slice of TrashInfo structs, containing information about all RBD images +// currently residing in the trash. +func GetTrashList(ioctx *rados.IOContext) ([]TrashInfo, error) { + var num_entries C.size_t + + // Call rbd_trash_list with nil pointer to get number of trash entries. + if C.rbd_trash_list(C.rados_ioctx_t(ioctx.Pointer()), nil, &num_entries); num_entries == 0 { + return nil, nil + } + + c_entries := make([]C.rbd_trash_image_info_t, num_entries) + trashList := make([]TrashInfo, num_entries) + + if ret := C.rbd_trash_list(C.rados_ioctx_t(ioctx.Pointer()), &c_entries[0], &num_entries); ret < 0 { + return nil, RBDError(ret) + } + + for i, ti := range c_entries { + trashList[i] = TrashInfo{ + Id: C.GoString(ti.id), + Name: C.GoString(ti.name), + DeletionTime: time.Unix(int64(ti.deletion_time), 0), + DefermentEndTime: time.Unix(int64(ti.deferment_end_time), 0), + } + } + + // Free rbd_trash_image_info_t pointers + C.rbd_trash_list_cleanup(&c_entries[0], num_entries) + + return trashList, nil +} + +// TrashRemove permanently deletes the trashed RBD with the specified id. +func TrashRemove(ioctx *rados.IOContext, id string, force bool) error { + c_id := C.CString(id) + defer C.free(unsafe.Pointer(c_id)) + + return GetError(C.rbd_trash_remove(C.rados_ioctx_t(ioctx.Pointer()), c_id, C.bool(force))) +} + +// TrashRestore restores the trashed RBD with the specified id back to the pool from whence it +// came, with the specified new name. +func TrashRestore(ioctx *rados.IOContext, id, name string) error { + c_id := C.CString(id) + c_name := C.CString(name) + defer C.free(unsafe.Pointer(c_id)) + defer C.free(unsafe.Pointer(c_name)) + + return GetError(C.rbd_trash_restore(C.rados_ioctx_t(ioctx.Pointer()), c_id, c_name)) +}