opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release/2.1.0' of ssh://git.yunion.io/~qiujian/onecloud into feature/qj-elastic-ip

Browse Source
This commit is contained in:
Qiu Jian
2018-09-04 23:46:43 +08:00
parent c3f5e8289d 031ed3a3e8
commit 45f7f6febe
122 changed files with 3993 additions and 893 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/aliyun/disk.go
+114
View File
@@ -61,6 +61,8 @@ func (self *SDisk) GetMetadata() *jsonutils.JSONDict {
priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type)
data.Add(jsonutils.NewString(priceKey), "price_key")
data.Add(jsonutils.NewString(models.HYPERVISOR_ALIYUN), "hypervisor")
return data
}
@@ -107,6 +109,11 @@ func (self *SDisk) GetId() string {
}
func (self *SDisk) Delete() error {
if _, err := self.storage.zone.region.getDisk(self.DiskId); err == cloudprovider.ErrNotFound {
// 未找到disk, 说明disk已经被删除了. 避免回收站中disk-delete循环删除失败
log.Errorf("Failed to find disk %s when delete", self.DiskId)
return nil
}
return self.storage.zone.region.deleteDisk(self.DiskId)
}
@@ -260,3 +267,110 @@ func (self *SRegion) resizeDisk(diskId string, size int64) error {
return nil
}
func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) {
if snapshotId, err := self.storage.zone.region.CreateSnapshot(self.DiskId, name, desc); err != nil {
log.Errorf("createSnapshot fail %s", err)
return nil, err
} else if snapshot, err := self.getSnapshot(snapshotId); err != nil {
return nil, err
} else {
snapshot.disk = self
if err := cloudprovider.WaitStatus(snapshot, string(SnapshotStatusAccoplished), 15*time.Second, 3600*time.Second); err != nil {
return nil, err
}
return snapshot, nil
}
}
func (self *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) {
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["DiskId"] = diskId
params["SnapshotName"] = name
params["Description"] = desc
if body, err := self.ecsRequest("CreateSnapshot", params); err != nil {
log.Errorf("CreateSnapshot fail %s", err)
return "", err
} else {
return body.GetString("SnapshotId")
}
}
func (self *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
if snapshot, err := self.getSnapshot(snapshotId); err != nil {
return nil, err
} else {
snapshot.disk = self
return snapshot, nil
}
}
func (self *SDisk) getSnapshot(snapshotId string) (*SSnapshot, error) {
if snapshots, total, err := self.storage.zone.region.GetSnapshots("", "", "", []string{snapshotId}, 0, 1); err != nil {
return nil, err
} else if total != 1 {
return nil, cloudprovider.ErrNotFound
} else {
return &snapshots[0], nil
}
}
func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
snapshots := make([]SSnapshot, 0)
for {
if parts, total, err := self.storage.zone.region.GetSnapshots("", self.DiskId, "", []string{}, 0, 20); err != nil {
log.Errorf("GetDisks fail %s", err)
return nil, err
} else {
snapshots = append(snapshots, parts...)
if len(snapshots) >= total {
break
}
}
}
isnapshots := make([]cloudprovider.ICloudSnapshot, len(snapshots))
for i := 0; i < len(snapshots); i++ {
snapshots[i].disk = self
isnapshots[i] = &snapshots[i]
}
return isnapshots, nil
}
func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName string, snapshotIds []string, offset int, limit int) ([]SSnapshot, int, error) {
if limit > 50 || limit <= 0 {
limit = 50
}
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["PageSize"] = fmt.Sprintf("%d", limit)
params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1)
if len(instanceId) > 0 {
params["InstanceId"] = instanceId
}
if len(diskId) > 0 {
params["diskId"] = diskId
}
if len(snapshotName) > 0 {
params["SnapshotName"] = snapshotName
}
if snapshotIds != nil && len(snapshotIds) > 0 {
params["SnapshotIds"] = jsonutils.Marshal(snapshotIds).String()
}
if body, err := self.ecsRequest("DescribeSnapshots", params); err != nil {
log.Errorf("GetSnapshots fail %s", err)
return nil, 0, err
} else {
snapshots := make([]SSnapshot, 0)
if err := body.Unmarshal(&snapshots, "Snapshots", "Snapshot"); err != nil {
log.Errorf("Unmarshal snapshot details fail %s", err)
return nil, 0, err
}
total, _ := body.Int("TotalCount")
return snapshots, int(total), nil
}
}
pkg/util/aliyun/image.go
+31
View File
@@ -5,6 +5,7 @@ import (
"strings"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -71,6 +72,10 @@ func (self *SImage) IsEmulated() bool {
return false
}
func (self *SImage) Delete() error {
return self.storageCache.region.DeleteImage(self.ImageId)
}
func (self *SImage) GetGlobalId() string {
return fmt.Sprintf("%s-%s")
}
@@ -102,6 +107,32 @@ func (self *SImage) Refresh() error {
return jsonutils.Update(self, new)
}
type ImageExportTask struct {
ImageId string
RegionId string
// RequestId string
TaskId string
}
func (self *SRegion) ExportImage(imageId string, bucket *oss.Bucket) (*ImageExportTask, error) {
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["ImageId"] = imageId
params["OssBucket"] = bucket.BucketName
params["OssPrefix"] = fmt.Sprintf("%sexport", strings.Replace(imageId, "-", "", -1))
if body, err := self.ecsRequest("ExportImage", params); err != nil {
return nil, err
} else {
result := ImageExportTask{}
if err := body.Unmarshal(&result); err != nil {
log.Errorf("unmarshal result error %s", err)
return nil, err
}
return &result, nil
}
}
// {"ImageId":"m-j6c1qlpa7oebbg1n2k60","RegionId":"cn-hongkong","RequestId":"F8B2F6A1-F6AA-4C92-A54C-C4A309CF811F","TaskId":"t-j6c1qlpa7oebbg1rcl9t"}
type ImageImportTask struct {
pkg/util/aliyun/shell/snapshot.go
+45
View File
@@ -0,0 +1,45 @@
package shell
import (
"yunion.io/x/onecloud/pkg/util/aliyun"
"yunion.io/x/onecloud/pkg/util/shellutils"
)
func init() {
type SnapshotListOptions struct {
DiskId string `help:"Disk ID"`
InstanceId string `help:"Instance ID"`
SnapshotIds []string `helo:"Snapshot ids"`
Name string `help:"Snapshot Name"`
Limit int `help:"page size"`
Offset int `help:"page offset"`
}
shellutils.R(&SnapshotListOptions{}, "snapshot-list", "List snapshot", func(cli *aliyun.SRegion, args *SnapshotListOptions) error {
if snapshots, total, err := cli.GetSnapshots(args.InstanceId, args.DiskId, args.Name, args.SnapshotIds, args.Offset, args.Limit); err != nil {
return err
} else {
printList(snapshots, total, args.Offset, args.Limit, []string{})
return nil
}
})
type SnapshotDeleteOptions struct {
ID string `help:"Snapshot ID"`
}
shellutils.R(&SnapshotDeleteOptions{}, "snapshot-delete", "Delete snapshot", func(cli *aliyun.SRegion, args *SnapshotDeleteOptions) error {
return cli.DeleteSnapshot(args.ID)
})
type SnapshotCreateOptions struct {
DiskId string `help:"Disk ID"`
Name string `help:"Snapeshot Name"`
Desc string `help:"Snapshot Desc"`
}
shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *aliyun.SRegion, args *SnapshotCreateOptions) error {
_, err := cli.CreateSnapshot(args.DiskId, args.Name, args.Desc)
return err
})
}
pkg/util/aliyun/snapshot.go
+73
View File
@@ -0,0 +1,73 @@
package aliyun
import (
"fmt"
"yunion.io/x/jsonutils"
)
type SnapshotStatusType string
const (
SnapshotStatusAccoplished SnapshotStatusType = "accomplished"
SnapshotStatusProgress SnapshotStatusType = "progressing"
)
type SSnapshot struct {
disk *SDisk
Progress string
SnapshotId string
SnapshotName string
SourceDiskId string
SourceDiskSize int32
SourceDiskType string
Status SnapshotStatusType
Usage string
}
func (self *SSnapshot) GetId() string {
return self.SnapshotId
}
func (self *SSnapshot) GetName() string {
return self.SnapshotName
}
func (self *SSnapshot) GetStatus() string {
return string(self.Status)
}
func (self *SSnapshot) Refresh() error {
if snapshot, err := self.disk.getSnapshot(self.SnapshotId); err != nil {
return err
} else if err := jsonutils.Update(self, snapshot); err != nil {
return err
}
return nil
}
func (self *SSnapshot) GetGlobalId() string {
return fmt.Sprintf("%s", self.SnapshotId)
}
func (self *SSnapshot) IsEmulated() bool {
return false
}
func (self *SRegion) DeleteSnapshot(snapshotId string) error {
params := make(map[string]string)
params["SnapshotId"] = snapshotId
_, err := self.ecsRequest("DeleteSnapshot", params)
return err
}
func (self *SSnapshot) Delete() error {
if self.disk == nil {
return fmt.Errorf("not init disk for snapshot %s", self.SnapshotId)
}
return self.disk.storage.zone.region.DeleteSnapshot(self.SnapshotId)
}
func (self *SSnapshot) GetMetadata() *jsonutils.JSONDict {
return nil
}
pkg/util/aliyun/storagecache.go
+128
View File
@@ -2,13 +2,17 @@ package aliyun
import (
"fmt"
"os"
"strings"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudprovider"
compute "yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -173,3 +177,127 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
return task.ImageId, nil
}
func (self *SStoragecache) CreateIImage(snapshoutId, imageName, imageDesc string) (cloudprovider.ICloudImage, error) {
if imageId, err := self.region.createIImage(snapshoutId, imageName, imageDesc); err != nil {
return nil, err
} else if image, err := self.region.GetImage(imageId); err != nil {
return nil, err
} else {
image.storageCache = self
iimage := make([]cloudprovider.ICloudImage, 1)
iimage[0] = image
if err := cloudprovider.WaitStatus(iimage[0], compute.IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second); err != nil {
return nil, err
}
return iimage[0], nil
}
}
func (self *SRegion) CheckBucket(bucketName string) (*oss.Bucket, error) {
return self.checkBucket(bucketName)
}
func (self *SRegion) checkBucket(bucketName string) (*oss.Bucket, error) {
oss, err := self.GetOssClient()
if err != nil {
log.Errorf("GetOssClient err %s", err)
return nil, err
}
if exist, err := oss.IsBucketExist(bucketName); err != nil {
log.Errorf("IsBucketExist err %s", err)
return nil, err
} else if !exist {
log.Debugf("Bucket %s not exists, to create ...", bucketName)
if err := oss.CreateBucket(bucketName); err != nil {
log.Errorf("Create bucket error %s", err)
return nil, err
}
}
log.Debugf("Bucket %s exists", bucketName)
if bucket, err := oss.Bucket(bucketName); err != nil {
log.Errorf("Bucket error %s %s", bucketName, err)
return nil, err
} else {
return bucket, nil
}
}
func (self *SRegion) createIImage(snapshoutId, imageName, imageDesc string) (string, error) {
params := make(map[string]string)
params["RegionId"] = self.RegionId
params["OssBucket"] = strings.ToLower(fmt.Sprintf("imgcache-%s", self.GetId()))
params["SnapshotId"] = snapshoutId
params["ImageName"] = imageName
params["Description"] = imageDesc
if _, err := self.checkBucket(params["OssBucket"]); err != nil {
return "", err
}
if body, err := self.ecsRequest("CreateImage", params); err != nil {
log.Errorf("CreateImage fail %s", err)
return "", err
} else {
log.Infof("%s", body)
return body.GetString("ImageId")
}
}
func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) {
return self.downloadImage(userCred, imageId, extId)
}
// 定义进度条监听器。
type OssProgressListener struct {
}
// 定义进度变更事件处理函数。
func (listener *OssProgressListener) ProgressChanged(event *oss.ProgressEvent) {
switch event.EventType {
case oss.TransferStartedEvent:
log.Debugf("Transfer Started, ConsumedBytes: %d, TotalBytes %d.\n",
event.ConsumedBytes, event.TotalBytes)
case oss.TransferDataEvent:
log.Debugf("\rTransfer Data, ConsumedBytes: %d, TotalBytes %d, %d%%.",
event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes)
case oss.TransferCompletedEvent:
log.Debugf("\nTransfer Completed, ConsumedBytes: %d, TotalBytes %d.\n",
event.ConsumedBytes, event.TotalBytes)
case oss.TransferFailedEvent:
log.Debugf("\nTransfer Failed, ConsumedBytes: %d, TotalBytes %d.\n",
event.ConsumedBytes, event.TotalBytes)
default:
}
}
func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) {
tmpImageFile := fmt.Sprintf("/tmp/%s", extId)
bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId()))
if bucket, err := self.region.checkBucket(bucketName); err != nil {
return nil, err
} else if _, err := self.region.GetImage(extId); err != nil {
return nil, err
} else if task, err := self.region.ExportImage(extId, bucket); err != nil {
return nil, err
} else if err := self.region.waitTaskStatus(ExportImageTask, task.TaskId, "Finished", 15*time.Second, 3600*time.Second); err != nil {
return nil, err
} else if imageList, err := bucket.ListObjects(oss.Prefix(fmt.Sprintf("%sexport", strings.Replace(extId, "-", "", -1)))); err != nil {
return nil, err
} else if len(imageList.Objects) != 1 {
return nil, httperrors.NewResourceNotFoundError("exported image not find")
} else if err := bucket.DownloadFile(imageList.Objects[0].Key, tmpImageFile, 12*1024*1024, oss.Routines(3), oss.Progress(&OssProgressListener{})); err != nil {
return nil, err
} else {
s := auth.GetAdminSession(options.Options.Region, "")
params := jsonutils.Marshal(map[string]string{"image_id": imageId, "disk-format": "raw"})
if file, err := os.Open(tmpImageFile); err != nil {
return nil, err
} else if result, err := modules.Images.Upload(s, params, file, imageList.Objects[0].Size); err != nil {
return nil, err
} else {
os.Remove(tmpImageFile)
return result, nil
}
}
}
pkg/util/logclient/doc.go
+1
View File
@@ -0,0 +1 @@
package logclient // import "yunion.io/x/onecloud/pkg/util/logclient"
pkg/util/logclient/logclient.go
+97
View File
@@ -0,0 +1,97 @@
package logclient
import (
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/mcclient/modules"
"yunion.io/x/pkg/util/stringutils"
)
const (
ACT_ADDTAG = "添加标签"
ACT_ALLOCATE = "分配"
ACT_BM_CONVERT_HYPER = "转换为宿主机"
ACT_BM_MAINTENANCE = "进入离线状态"
ACT_BM_UNCONVERT_HYPER = "转换为受管物理机"
ACT_BM_UNMAINTENANCE = "退出离线状态"
ACT_CANCEL_DELETE = "恢复"
ACT_CHANGE_OWNER = "更改项目"
ACT_CLOUD_FULLSYNC = "全量同步"
ACT_CLOUD_SYNC = "同步"
ACT_CREATE = "创建"
ACT_DELETE = "删除"
ACT_DISABLE = "禁用"
ACT_ENABLE = "启用"
ACT_GUEST_ATTACH_ISOLATED_DEVICE = "挂载透传设备"
ACT_GUEST_DETACH_ISOLATED_DEVICE = "卸载透传设备"
ACT_MERGE = "合并"
ACT_OFFLINE = "下线"
ACT_ONLINE = "上线"
ACT_PRIVATE = "设为私有"
ACT_PUBLIC = "设为共享"
ACT_RELEASE_IP = "释放IP"
ACT_RESERVE_IP = "预留IP"
ACT_RESIZE = "扩容"
ACT_RMTAG = "删除标签"
ACT_SPLIT = "分割"
ACT_UNCACHED_IMAGE = "清除缓存"
ACT_UPDATE = "更新"
ACT_VM_ATTACH_DISK = "挂载磁盘"
ACT_VM_BIND_KEYPAIR = "绑定密钥"
ACT_VM_CHANGE_FLAVOR = "调整配置"
ACT_VM_DEPLOY = "部署"
ACT_VM_DETACH_DISK = "卸载磁盘"
ACT_VM_PURGE = "清除"
ACT_VM_REBUILD = "重装系统"
ACT_VM_RESET_PSWD = "重置密码"
ACT_VM_START = "开机"
ACT_VM_STOP = "关机"
ACT_VM_SYNC_CONF = "同步配置"
ACT_VM_SYNC_STATUS = "同步状态"
ACT_VM_UNBIND_KEYPAIR = "解绑密钥"
)
type IObject interface {
GetId() string
GetName() string
Keyword() string
}
func AddActionLog(model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
token := userCred
notes := stringutils.Interface2String(iNotes)
s := auth.GetSession(userCred, "", "")
logentry := jsonutils.NewDict()
logentry.Add(jsonutils.NewString(model.GetName()), "obj_name")
logentry.Add(jsonutils.NewString(model.Keyword()), "obj_type")
logentry.Add(jsonutils.NewString(model.GetId()), "obj_id")
logentry.Add(jsonutils.NewString(action), "action")
logentry.Add(jsonutils.NewString(token.GetUserId()), "user_id")
logentry.Add(jsonutils.NewString(token.GetUserName()), "user")
logentry.Add(jsonutils.NewString(token.GetTenantId()), "tenant_id")
logentry.Add(jsonutils.NewString(token.GetTenantName()), "tenant")
if !success {
// 失败日志
logentry.Add(jsonutils.JSONFalse, "success")
} else {
// 成功日志
logentry.Add(jsonutils.JSONTrue, "success")
}
// TODO delete tag when done.
notes = fmt.Sprintf("[a2]%s", notes)
logentry.Add(jsonutils.NewString(notes), "notes")
_, err := modules.Actions.Create(s, logentry)
if err != nil {
fmt.Printf("create action log failed %s", err)
} else {
fmt.Println("create action log success")
}
}