From 5e8b7b3ce3a13788f4dfad9eaa3ee884d0fc9cc6 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 21 Aug 2018 20:49:42 +0800 Subject: [PATCH] fix auto-merge conflict --- Gopkg.lock | 6 + Makefile | 12 +- cmd/climc/climc.go | 52 +- cmd/climc/promputils/executor.go | 5 + cmd/climc/shell/disks.go | 14 +- cmd/climc/shell/secgrouprules.go | 2 - cmd/climc/shell/servers.go | 14 +- pkg/appsrv/dispatcher/dispatcher.go | 10 +- pkg/cloudcommon/cronman/cronman.go | 2 + pkg/cloudcommon/db/db_dispatcher.go | 23 +- pkg/cloudcommon/db/quotas/quotas.go | 4 +- pkg/cloudcommon/db/taskman/localtaskworker.go | 6 +- pkg/cloudcommon/db/taskman/subtasks.go | 2 +- pkg/cloudcommon/db/taskman/tasks.go | 26 +- pkg/cloudcommon/db/virtualresource.go | 4 +- pkg/cloudcommon/options.go | 1 + pkg/cloudprovider/resources.go | 8 + pkg/compute/guestdrivers/aliyun.go | 23 + pkg/compute/guestdrivers/baremetals.go | 22 +- pkg/compute/guestdrivers/base.go | 65 +- pkg/compute/guestdrivers/esxi.go | 30 + pkg/compute/guestdrivers/kvm.go | 184 ++++- pkg/compute/guestdrivers/managedvirtual.go | 2 +- pkg/compute/guestdrivers/virtualization.go | 36 + pkg/compute/hostdrivers/aliyun.go | 104 +++ pkg/compute/hostdrivers/kvm.go | 126 ++++ pkg/compute/models/cachedimages.go | 53 +- pkg/compute/models/cloudregions.go | 2 +- pkg/compute/models/disks.go | 280 +++++++- pkg/compute/models/guestdisks.go | 50 +- pkg/compute/models/guestdrivers.go | 26 +- pkg/compute/models/guestnetworks.go | 2 + pkg/compute/models/guests.go | 652 +++++++++++++++++- pkg/compute/models/hostdrivers.go | 39 ++ pkg/compute/models/hosts.go | 37 +- pkg/compute/models/networks.go | 1 + pkg/compute/models/secgrouprules.go | 187 ++++- pkg/compute/models/secgroups.go | 137 +++- pkg/compute/models/storagecachedimages.go | 56 +- pkg/compute/models/storagecaches.go | 54 +- pkg/compute/models/storages.go | 30 +- pkg/compute/models/vpcs.go | 10 +- pkg/compute/service/service.go | 19 +- .../tasks/cloud_provider_sync_info_task.go | 26 +- pkg/compute/tasks/disk_base_task.go | 15 + pkg/compute/tasks/disk_create_task.go | 76 ++ pkg/compute/tasks/disk_delete_task.go | 16 +- pkg/compute/tasks/disk_resize_task.go | 108 +++ pkg/compute/tasks/guest_change_config_task.go | 208 ++++++ pkg/compute/tasks/guest_create_disk_task.go | 124 ++++ pkg/compute/tasks/guest_delete_task.go | 13 +- .../tasks/guest_detach_all_disks_task.go | 1 + pkg/compute/tasks/guest_detach_disk_task.go | 83 +++ pkg/compute/tasks/guest_insert_iso_task.go | 58 +- pkg/compute/tasks/guest_rebuild_root_task.go | 144 ++++ pkg/compute/tasks/guest_reset_task.go | 58 ++ pkg/compute/tasks/guest_start_task.go | 3 +- pkg/compute/tasks/guest_stop_task.go | 1 + pkg/compute/tasks/guest_suspend_task.go | 47 ++ pkg/compute/tasks/guest_sync_task.go | 69 ++ pkg/compute/tasks/guest_syncstatus_task.go | 4 +- pkg/compute/tasks/guest_undeploy_task.go | 3 +- pkg/compute/tasks/storage_cache_image_task.go | 26 +- pkg/httperrors/errors.go | 7 + pkg/mcclient/mcclient.go | 11 + pkg/mcclient/session.go | 4 + pkg/util/aliyun/disk.go | 17 + pkg/util/aliyun/host.go | 4 +- pkg/util/aliyun/instance.go | 31 + pkg/util/aliyun/region.go | 1 + pkg/util/aliyun/securitygroup.go | 399 ++++++++++- pkg/util/aliyun/storage.go | 9 + pkg/util/aliyun/vpc.go | 48 +- pkg/util/esxi/storage.go | 4 + pkg/util/esxi/virtualmachine.go | 10 +- vendor/github.com/serialx/hashring/LICENSE | 21 + vendor/github.com/serialx/hashring/README.md | 70 ++ .../github.com/serialx/hashring/hashring.go | 268 +++++++ 78 files changed, 4202 insertions(+), 203 deletions(-) create mode 100644 pkg/compute/hostdrivers/aliyun.go create mode 100644 pkg/compute/hostdrivers/kvm.go create mode 100644 pkg/compute/models/hostdrivers.go create mode 100644 pkg/compute/tasks/disk_create_task.go create mode 100644 pkg/compute/tasks/disk_resize_task.go create mode 100644 pkg/compute/tasks/guest_change_config_task.go create mode 100644 pkg/compute/tasks/guest_create_disk_task.go create mode 100644 pkg/compute/tasks/guest_rebuild_root_task.go create mode 100644 pkg/compute/tasks/guest_reset_task.go create mode 100644 pkg/compute/tasks/guest_suspend_task.go create mode 100644 pkg/compute/tasks/guest_sync_task.go create mode 100644 vendor/github.com/serialx/hashring/LICENSE create mode 100644 vendor/github.com/serialx/hashring/README.md create mode 100644 vendor/github.com/serialx/hashring/hashring.go diff --git a/Gopkg.lock b/Gopkg.lock index 2499f89c7f..ccd7d5e941 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -667,6 +667,12 @@ [[projects]] digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + branch = "master" + name = "github.com/serialx/hashring" + packages = ["."] + revision = "49a4782e9908fe098c907022a1bd7519c79803d6" + +[[projects]] name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" diff --git a/Makefile b/Makefile index 228037d447..6537fbd733 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell pwd) +ROOT_DIR := $(shell readlink -f `pwd`) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh @@ -84,6 +84,16 @@ output_dir: @mkdir -p $(BUILD_DIR) +dep: + cd $(ROOT_DIR) && dep ensure -v + +dep_clean: + rm -fr $(GOPATH)/pkg/dep/sources/* + +dep_install: + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + + .PHONY: all build prepare_dir clean fmt rpm diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index 175543ccc8..1d4bdec473 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -1,11 +1,14 @@ package main import ( + "encoding/json" "fmt" + "io/ioutil" "os" "strings" + "time" - "github.com/c-bata/go-prompt" + prompt "github.com/c-bata/go-prompt" "yunion.io/x/log" "yunion.io/x/pkg/util/version" "yunion.io/x/structarg" @@ -117,18 +120,51 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { options.Timeout, options.Debug, options.Insecure) - token, err := client.Authenticate(options.OsUsername, - options.OsPassword, - options.OsDomainName, - options.OsProjectName) - if err != nil { - return nil, err + + var cacheToken mcclient.TokenCredential + cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN") + if err == nil && cacheFile != nil { + fileInfo, _ := cacheFile.Stat() + dur, err := time.ParseDuration("-24h") + if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { + bytesToken, err := ioutil.ReadAll(cacheFile) + if err == nil { + token := client.NewAuthTokenCredential() + err := json.Unmarshal(bytesToken, token) + if err != nil { + fmt.Printf("Unmarshal token error:%s", err) + } else { + cacheToken = token + } + } + cacheFile.Close() + } + } + if cacheToken == nil { + token, err := client.Authenticate(options.OsUsername, + options.OsPassword, + options.OsDomainName, + options.OsProjectName) + if err != nil { + return nil, err + } + cacheToken = token + bytesCacheToken, err := json.Marshal(cacheToken) + if err != nil { + fmt.Printf("Marshal token error:%s", err) + } else { + fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN") + fo.Write(bytesCacheToken) + fo.Close() + } + } else { + fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********") } session := client.NewSession(options.OsRegionName, options.OsZoneName, options.OsEndpointType, - token, + cacheToken, options.ApiVersion) return session, nil } diff --git a/cmd/climc/promputils/executor.go b/cmd/climc/promputils/executor.go index a3f579dfea..2b48225b17 100644 --- a/cmd/climc/promputils/executor.go +++ b/cmd/climc/promputils/executor.go @@ -56,6 +56,11 @@ func Executor(s string) { e := parser.ParseArgs(utils.ArgsStringToArray(s), false) subcmd := parser.GetSubcommand() subparser := subcmd.GetSubParser() + if args[0] == "--debug" { + session.GetClient().SetDebug(true) + } else { + session.GetClient().SetDebug(false) + } if e != nil { if subparser != nil { fmt.Print(subparser.Usage()) diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 3f4d35033b..36654af96e 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -10,11 +10,12 @@ import ( func init() { type DiskListOptions struct { BaseListOptions - Unused bool `help:"Show unused disks"` - Share bool `help:"Show Share storage disks"` - Local bool `help:"Show Local storage disks"` - Guest string `help:"Guest ID or name"` - Storage string `help:"Storage ID or name"` + Unused bool `help:"Show unused disks"` + Share bool `help:"Show Share storage disks"` + Local bool `help:"Show Local storage disks"` + Guest string `help:"Guest ID or name"` + Storage string `help:"Storage ID or name"` + Provider string `help:"Provider for disk" choices:"Aliyun|VMware"` } R(&DiskListOptions{}, "disk-list", "List virtual disks", func(s *mcclient.ClientSession, suboptions *DiskListOptions) error { params := FetchPagingParams(suboptions.BaseListOptions) @@ -33,6 +34,9 @@ func init() { if len(suboptions.Storage) > 0 { params.Add(jsonutils.NewString(suboptions.Storage), "storage") } + if len(suboptions.Provider) > 0 { + params.Add(jsonutils.NewString(suboptions.Provider), "provider") + } result, err := modules.Disks.List(s, params) if err != nil { return err diff --git a/cmd/climc/shell/secgrouprules.go b/cmd/climc/shell/secgrouprules.go index eb9fd4b621..13fdd49730 100644 --- a/cmd/climc/shell/secgrouprules.go +++ b/cmd/climc/shell/secgrouprules.go @@ -59,7 +59,6 @@ func init() { }) type SecGroupRulesCreateOptions struct { - NAME string `help:"Name of security group rule to create"` SECGROUP string `help:"Secgroup ID or Name" metavar:"Secgroup"` Direction string `help:"Direction of rule" choices:"in|out"` Action string `help:"Action of rule" choices:"allow|deny"` @@ -72,7 +71,6 @@ func init() { R(&SecGroupRulesCreateOptions{}, "secgroup-rule-create", "Create all security group rule", func(s *mcclient.ClientSession, args *SecGroupRulesCreateOptions) error { params := jsonutils.NewDict() - params.Add(jsonutils.NewString(args.NAME), "name") if len(args.Desc) > 0 { params.Add(jsonutils.NewString(args.Desc), "description") } diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index fe663182b6..0a3b49acfb 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -315,7 +315,7 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-sync", "Sync servers status", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { + R(&ServerOpsOptions{}, "server-sync", "Sync servers configures", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { ret := modules.Servers.BatchPerformAction(s, args.ID, "sync", nil) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil @@ -343,8 +343,16 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { - ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", nil) + type ServerResetOptions struct { + ServerOpsOptions + Hard bool `help:"Hard reset or not; default soft"` + } + R(&ServerResetOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerResetOptions) error { + params := jsonutils.NewDict() + if args.Hard { + params.Add(jsonutils.JSONTrue, "is_hard") + } + ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", params) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil }) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index 56cb359b66..1aeff85240 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/httperrors" @@ -201,7 +202,14 @@ func createInContextHandler(ctx context.Context, w http.ResponseWriter, r *http. func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { manager, params, query, body := fetchEnv(ctx, w, r) - data, _ := body.Get(manager.KeywordPlural()) + var data jsonutils.JSONObject + if body != nil { + data, _ = body.Get(manager.KeywordPlural()) + // about string ?? + if data == nil { + data = body.(*jsonutils.JSONDict) + } + } if data == nil { data = jsonutils.NewDict() } diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index d01cc096d5..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -89,6 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index fe06fc2f9d..8b4179f50d 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -9,18 +9,18 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/filterclause" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/httputils" ) type DBModelDispatcher struct { @@ -841,7 +841,12 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act managerValue := reflect.ValueOf(dispatcher.modelManager) if action == "check-create-data" { - return dispatcher.modelManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data.(*jsonutils.JSONDict)) + manager := dispatcher.modelManager + if body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword()); err != nil { + return nil, httperrors.NewGeneralError(err) + } else { + return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, body.(*jsonutils.JSONDict)) + } } return objectPerformAction(dispatcher, managerValue, ctx, userCred, action, query, data) } @@ -1039,13 +1044,13 @@ func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCr err := model.ValidateDeleteCondition(ctx) if err != nil { log.Errorf("validate delete condition error: %s", err) - return nil, httperrors.NewGeneralError(err) + return nil, httperrors.NewNotAcceptableError(err.Error()) } err = model.CustomizeDelete(ctx, userCred, query, data) if err != nil { log.Errorf("customize delete error: %s", err) - return nil, httperrors.NewGeneralError(err) + return nil, httperrors.NewNotAcceptableError(err.Error()) } details, err := getItemDetails(manager, model, ctx, userCred, query) diff --git a/pkg/cloudcommon/db/quotas/quotas.go b/pkg/cloudcommon/db/quotas/quotas.go index b26700d073..b80fb3c1d0 100644 --- a/pkg/cloudcommon/db/quotas/quotas.go +++ b/pkg/cloudcommon/db/quotas/quotas.go @@ -63,7 +63,9 @@ func (manager *SQuotaManager) _cancelPendingUsage(ctx context.Context, userCred log.Errorf("%s", err) return err } - localUsage.Sub(cancelUsage) + if localUsage != nil { + localUsage.Sub(cancelUsage) + } quota.Sub(cancelUsage) err = manager.pendingStore.SetQuota(ctx, userCred, projectId, quota) if err != nil { diff --git a/pkg/cloudcommon/db/taskman/localtaskworker.go b/pkg/cloudcommon/db/taskman/localtaskworker.go index 2e5ac395e9..f60e8b146f 100644 --- a/pkg/cloudcommon/db/taskman/localtaskworker.go +++ b/pkg/cloudcommon/db/taskman/localtaskworker.go @@ -15,7 +15,7 @@ func init() { localTaskWorkerMan = appsrv.NewWorkerManager("LocalTaskWorkerManager", 4, 10) } -func error2TaskData(err error) jsonutils.JSONObject { +func Error2TaskData(err error) jsonutils.JSONObject { errJson := jsonutils.NewDict() errJson.Add(jsonutils.NewString("ERROR"), "__status__") errJson.Add(jsonutils.NewString(err.Error()), "reason") @@ -32,12 +32,12 @@ func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) { if r := recover(); r != nil { log.Errorf("LocalTaskRun error: %s", r) debug.PrintStack() - task.ScheduleRun(error2TaskData(fmt.Errorf("LocalTaskRun error: %s", r))) + task.ScheduleRun(Error2TaskData(fmt.Errorf("LocalTaskRun error: %s", r))) } }() data, err := proc() if err != nil { - task.ScheduleRun(error2TaskData(err)) + task.ScheduleRun(Error2TaskData(err)) } else { task.ScheduleRun(data) } diff --git a/pkg/cloudcommon/db/taskman/subtasks.go b/pkg/cloudcommon/db/taskman/subtasks.go index 8ad2c2902e..d4d0bf50ed 100644 --- a/pkg/cloudcommon/db/taskman/subtasks.go +++ b/pkg/cloudcommon/db/taskman/subtasks.go @@ -40,7 +40,7 @@ func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SS err := manager.Query().Equals("task_id", ptaskId).Equals("subtask_id", subtaskId).First(&subtask) if err != nil { if err != sql.ErrNoRows { - log.Errorf("GetSubTask fail %s", err) + log.Errorf("GetSubTask fail %s %s %s", err, ptaskId, subtaskId) } return nil } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..8e4f45acf6 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -307,7 +311,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, var stageName string if taskFailed { - stageName = fmt.Sprintf("%s_failed", task.Stage) + stageName = fmt.Sprintf("%sFailed", task.Stage) } else { stageName = task.Stage } @@ -315,6 +319,10 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, funcValue := taskValue.MethodByName(stageName) if !funcValue.IsValid() || funcValue.IsNil() { + log.Debugf("Stage %s not found, try kebab to camel and find again", stageName) + if taskFailed { + stageName = fmt.Sprintf("%s_failed", task.Stage) + } stageName = utils.Kebab2Camel(stageName, "_") funcValue = taskValue.MethodByName(stageName) @@ -387,7 +395,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, params[2] = reflect.ValueOf(data) - log.Debugf("Call %s with %s", funcValue, params) + log.Debugf("Call %s: %s with %s", stageName, funcValue, params) funcValue.Call(params) @@ -408,9 +416,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 28360ccb55..db56d599f4 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -8,13 +8,13 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SVirtualResourceBaseManager struct { diff --git a/pkg/cloudcommon/options.go b/pkg/cloudcommon/options.go index 7b91719b70..a6e4fa7336 100644 --- a/pkg/cloudcommon/options.go +++ b/pkg/cloudcommon/options.go @@ -33,6 +33,7 @@ type Options struct { NotifyAdminUser string `default:"sysadmin" help:"System administrator user ID or name to notify"` GlobalVirtualResourceNamespace bool `help:"Per project namespace or global namespace for virtual resources"` + DebugSqlchemy bool `default:"False" help:"Print SQL executed by sqlchemy"` structarg.BaseOptions } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 2ff3e397b9..f388d66feb 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/secrules" ) type ICloudResource interface { @@ -84,6 +85,7 @@ type ICloudStorage interface { GetManagerId() string CreateIDisk(name string, sizeGb int, desc string) (ICloudDisk, error) + GetIDisk(idStr string) (ICloudDisk, error) } type ICloudHost interface { @@ -142,6 +144,7 @@ type ICloudVM interface { GetBios() string GetMachine() string + SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error GetHypervisor() string // GetSecurityGroup() ICloudSecurityGroup @@ -168,6 +171,8 @@ type ICloudEIP interface { type ICloudSecurityGroup interface { ICloudResource + GetDescription() string + GetRules() ([]secrules.SecurityRule, error) } type ICloudDisk interface { @@ -187,6 +192,8 @@ type ICloudDisk interface { GetDriver() string GetCacheMode() string GetMountpoint() string + Delete() error + Resize(int64) error } type ICloudVpc interface { @@ -197,6 +204,7 @@ type ICloudVpc interface { GetCidrBlock() string // GetStatus() string GetIWires() ([]ICloudWire, error) + GetISecurityGroups() ([]ICloudSecurityGroup, error) GetManagerId() string diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 50ebab0f8d..9477c541f6 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -170,6 +170,13 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil, err } + if len(guest.SecgrpId) > 0 { + if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + log.Errorf("SyncSecurityGroup error: %v", err) + return nil, err + } + } + if onfinish == "none" { err = iVM.StartVM() if err != nil { @@ -254,3 +261,19 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } + +func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + if ihost, err := host.GetIHost(); err != nil { + return nil, err + } else if iVM, err := ihost.GetIVMById(guest.ExternalId); err != nil { + return nil, err + } else if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + return nil, err + } + } + return nil, nil + }) + return nil +} diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index d91cb4c49f..2ec935e955 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -2,13 +2,14 @@ package guestdrivers import ( "context" + "fmt" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" ) type SBaremetalGuestDriver struct { @@ -73,7 +74,7 @@ func (self *SBaremetalGuestDriver) RequestGuestCreateInsertIso(ctx context.Conte return nil } -func (self *SBaremetalGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { +func (self *SBaremetalGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { data := jsonutils.NewDict() // TODO return data, nil @@ -154,3 +155,20 @@ func (self *SBaremetalGuestDriver) RequestDeployGuestOnHost(ctx context.Context, return nil } + +func (self *SBaremetalGuestDriver) CanKeepDetachDisk() bool { + return false +} + +func (self *SBaremetalGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SBaremetalGuestDriver) StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + return fmt.Errorf("Cannot detach disk from a baremetal serer") +} + +func (self *SBaremetalGuestDriver) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + return fmt.Errorf("Cannot suspend a baremetal serer") +} diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 6d10f94aa5..a63aa1470c 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -54,16 +54,16 @@ func (self *SBaseGuestDriver) StartGuestCreateTask(guest *models.SGuest, ctx con } func (self *SBaseGuestDriver) OnGuestCreateTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - //if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { - // task.SetStage("on_auto_start_guest", nil) - // return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) - //} else { - task.SetStage("on_sync_status_complete", nil) - return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) - //} + if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { + task.SetStage("on_auto_start_guest", nil) + return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) + } else { + task.SetStage("on_sync_status_complete", nil) + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) + } } -func (self *SBaseGuestDriver) StartDeleteGuestTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { +func (self *SBaseGuestDriver) StartDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { task, err := taskman.TaskManager.NewTask(ctx, "GuestDeleteTask", guest, userCred, params, parentTaskId, "", nil) if err != nil { return err @@ -80,3 +80,52 @@ func (self *SBaseGuestDriver) RequestDetachDisksFromGuestForDelete(ctx context.C func (self *SBaseGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { return guest.DeleteAllDisksInDB(ctx, userCred) } + +func (self *SBaseGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SBaseGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{}, fmt.Errorf("This Guest driver dose not implement GetDetachDiskStatus") +} + +func (self *SBaseGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SBaseGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} diff --git a/pkg/compute/guestdrivers/esxi.go b/pkg/compute/guestdrivers/esxi.go index 4d515c0f84..f1ab950f82 100644 --- a/pkg/compute/guestdrivers/esxi.go +++ b/pkg/compute/guestdrivers/esxi.go @@ -1,6 +1,9 @@ package guestdrivers import ( + "context" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -16,3 +19,30 @@ func init() { func (self *SESXiGuestDriver) GetHypervisor() string { return models.HYPERVISOR_ESXI } + +func (self *SESXiGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SESXiGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{models.VM_READY}, nil +} + +func (self *SESXiGuestDriver) CanKeepDetachDisk() bool { + return false +} + +func (self *SESXiGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + err := disk.RealDelete(ctx, task.GetUserCred()) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SESXiGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 4b43f4f79d..d98ed72b15 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -3,15 +3,17 @@ package guestdrivers import ( "context" "fmt" + "net/http" "regexp" "strconv" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SKVMGuestDriver struct { @@ -36,9 +38,12 @@ func (self *SKVMGuestDriver) RequestDetachDisksFromGuestForDelete(ctx context.Co return nil } -func (self *SKVMGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { - // guest.DeleteAllDisksInDB(ctx, userCred) - // do nothing +func (self *SKVMGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "KVMGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) return nil } @@ -90,40 +95,173 @@ func (self *SKVMGuestDriver) GetGuestVncInfo(userCred mcclient.TokenCredential, return retval, nil } -func (self *SKVMGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - // TODO +func (self *SKVMGuestDriver) RequestStopOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + body := jsonutils.NewDict() + params := task.GetParams() + timeout, err := params.Int("timeout") + if err != nil { + timeout = 30 + } + isForce, err := params.Bool("is_force") + if isForce { + timeout = 0 + } + body.Add(jsonutils.NewInt(timeout), "timeout") + + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + + url := fmt.Sprintf("%s/servers/%s/stop", host.ManagerUri, guest.Id) + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + return err +} + +func (self *SKVMGuestDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + url := fmt.Sprintf("%s/servers/%s", host.ManagerUri, guest.Id) + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "DELETE", url, header, nil, false) + if err != nil { + return err + } + delayClean := jsonutils.QueryBoolean(res, "delay_clean", false) + if res != nil && delayClean { + return nil + } + task.ScheduleRun(nil) return nil } func (self *SKVMGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO - + config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) + log.Debugf("RequestDeployGuestOnHost: %s", config) + if config.Contains("container") { + // ... + } + action, err := config.GetString("action") + if err != nil { + return err + } + url := fmt.Sprintf("%s/servers/%s/%s", host.ManagerUri, guest.Id, action) + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false) + if err != nil { + return err + } return nil } -func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - // TODO +func (self *SKVMGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } -func (self *SKVMGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - // TODO - return data, nil -} +func (self *SKVMGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") -func (self *SKVMGuestDriver) RequestStopOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO - return nil + config := jsonutils.NewDict() + desc := self.GetJsonDescAtHost(ctx, guest, host) + config.Add(desc, "desc") + params := task.GetParams() + if params.Length() > 0 { + config.Add(params, "params") + } + url := fmt.Sprintf("%s/servers/%s/start", host.ManagerUri, guest.Id) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false) + if err != nil { + return nil, err + } + return res, nil } func (self *SKVMGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - // TODO - return data, nil + header := http.Header{} + header.Set("X-Auth-Token", userCred.GetTokenString()) + header.Set("X-Region-Version", "v2") + + url := fmt.Sprintf("%s/servers/%s/status", host.ManagerUri, guest.Id) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false) + if err != nil { + return nil, err + } + return res, nil } -func (self *SKVMGuestDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO +func (self *SKVMGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { + return nil +} + +func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + // pass + return nil +} + +func (self *SKVMGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + _, err := guest.SendMonitorCommand(ctx, task.GetUserCred(), "system_reset") + return err +} + +func (self *SKVMGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) +} + +func (self *SKVMGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING}, nil +} + +func (self *SKVMGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + return disk.StartDiskDeleteTask(ctx, task.GetUserCred(), task.GetTaskId(), isPurge) +} + +func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) + body := jsonutils.NewDict() + body.Add(desc, "desc") + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + body.Add(jsonutils.JSONTrue, "fw_only") + } + url := fmt.Sprintf("/servers/%s/sync", guest.Id) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + host := guest.GetHost() + url := fmt.Sprintf("%s/servers/%s/suspend", host.ManagerUri, guest.Id) + header := http.Header{} + header.Add("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, nil, false) + return err +} + +func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), task.GetParams(), task.GetTaskId()) +} + +func (self *SKVMGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) +} + +func (self *SKVMGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "KVMGuestRebuildRootTask", guest, task.GetUserCred(), nil, task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) return nil } diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 486cca38ed..5976b6ff60 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -46,7 +46,7 @@ func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx co return nil } -func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { +func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(_ context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { ihost, err := host.GetIHost() if err != nil { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index 2ed28aca95..7efdd6d475 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -95,8 +95,22 @@ func (self *SVirtualizedGuestDriver) StartGuestStopTask(guest *models.SGuest, ct return nil } +func (self *SVirtualizedGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + var taskName = "GuestSoftResetTask" + if isHard { + taskName = "GuestHardResetTask" + } + task, err := taskman.TaskManager.NewTask(ctx, taskName, guest, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SVirtualizedGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { if jsonutils.QueryBoolean(task.GetParams(), "restart", false) { + task.SetStage("OnDeployStartGuestComplete", nil) return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) } else { guest.SetStatus(task.GetUserCred(), models.VM_READY, "ready") @@ -157,3 +171,25 @@ func (self *SVirtualizedGuestDriver) CheckDiskTemplateOnStorage(ctx context.Cont } return cache.StartImageCacheTask(ctx, userCred, imageId, false, task.GetTaskId()) } + +func (self *SVirtualizedGuestDriver) CanKeepDetachDisk() bool { + return true +} + +func (self *SVirtualizedGuestDriver) StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestDetachDiskTask", guest, userCred, params, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SVirtualizedGuestDriver) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestSuspendTask", guest, userCred, params, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go new file mode 100644 index 0000000000..0bcfa18005 --- /dev/null +++ b/pkg/compute/hostdrivers/aliyun.go @@ -0,0 +1,104 @@ +package hostdrivers + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SAliyunHostDriver struct { +} + +func init() { + driver := SAliyunHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SAliyunHostDriver) GetHostType() string { + return models.HOST_TYPE_ALIYUN +} + +func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, scimg *models.SStoragecachedimage, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + isForce := jsonutils.QueryBoolean(params, "is_force", false) + userCred := task.GetUserCred() + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iStorageCache, err := storageCache.GetIStorageCache() + if err != nil { + return nil, err + } + + extImgId, err := iStorageCache.UploadImage(userCred, imageId, scimg.ExternalId, isForce) + + if err != nil { + return nil, err + } else { + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(extImgId), "image_id") + return ret, nil + } + }) + return nil +} + +func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else { + if size, err := content.Int("size"); err != nil { + return err + } else { + size = size >> 10 + if iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil { + return err + } else { + if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.ExternalId = iDisk.GetGlobalId() + return nil + }); err != nil { + log.Errorf("Update disk externalId err: %v", err) + return err + } + data := jsonutils.NewDict() + data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") + data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") + task.ScheduleRun(data) + } + } + } + return nil +} + +func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { + return err + } else { + return iDisk.Delete() + } +} + +func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +} + +func (self *SAliyunHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { + return err + } else if err := iDisk.Resize(size >> 10); err != nil { + return err + } else { + task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) + } + return nil +} diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go new file mode 100644 index 0000000000..0a5cbca5b2 --- /dev/null +++ b/pkg/compute/hostdrivers/kvm.go @@ -0,0 +1,126 @@ +package hostdrivers + +import ( + "context" + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SKVMHostDriver struct { +} + +func init() { + driver := SKVMHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SKVMHostDriver) GetHostType() string { + return models.HOST_TYPE_HYPERVISOR +} + +func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, scimg *models.SStoragecachedimage, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + isForce := jsonutils.QueryBoolean(params, "is_force", false) + obj, err := models.CachedimageManager.FetchById(imageId) + if err != nil { + return err + } + cacheImage := obj.(*models.SCachedimage) + srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(models.HOST_TYPE_HYPERVISOR, []string{host.Id}, []*models.SZone{host.GetZone()}) + if err != nil { + return err + } + + content := jsonutils.NewDict() + content.Add(jsonutils.NewString(imageId), "image_id") + if srcHostCacheImage != nil { + err = srcHostCacheImage.AddDownloadRefcount() + if err != nil { + return err + } + srcHost, err := srcHostCacheImage.GetHost() + if err != nil { + return err + } + srcUrl := fmt.Sprintf("%s/download/images/%s", srcHost.ManagerUri, imageId) + content.Add(jsonutils.NewString(srcUrl), "src_url") + } + url := fmt.Sprintf("%s/disks/image_cache", host.ManagerUri) + + if isForce { + content.Add(jsonutils.NewBool(true), "is_force") + } + content.Add(jsonutils.NewString(storageCache.Id), "storagecache_id") + body := jsonutils.NewDict() + body.Add(content, "disk") + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + if err != nil { + return err + } + return nil +} + +func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + body.Add(content, "disk") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + log.Infof("Deallocating disk on host %s", host.GetName()) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/resize/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + content := jsonutils.NewDict() + content.Add(jsonutils.NewInt(size), "size") + body.Add(content, "disk") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + self.RequestResizeDiskOnHost(host, storage, disk, size, task) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + for _, guest := range disk.GetAttachedGuests() { + guestdisk := guest.GetGuestDisk(disk.GetId()) + url := fmt.Sprintf("/servers/%s/monitor", guest.GetId()) + body := jsonutils.NewDict() + cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, size) + body.Add(jsonutils.NewString(cmd), "cmd") + host.Request(task.GetUserCred(), "POST", url, header, body) + } + return nil +} diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index b1a9cbdf56..827fdc5d29 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "math/rand" "time" "yunion.io/x/jsonutils" @@ -170,7 +171,7 @@ func (manager *SCachedimageManager) cacheImageInfo(ctx context.Context, userCred } } -func (manager *SCachedimageManager) getImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { +func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { if !refresh { imgObj, _ := manager.FetchById(imageId) if imgObj != nil { @@ -207,7 +208,7 @@ func (manager *SCachedimageManager) getImageByName(ctx context.Context, userCred } func (manager *SCachedimageManager) getImageInfo(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { - img, err := manager.getImageById(ctx, userCred, imageId, refresh) + img, err := manager.GetImageById(ctx, userCred, imageId, refresh) if err == nil { return img, nil } @@ -236,7 +237,7 @@ func (self *SCachedimage) AllowPerformRefresh(ctx context.Context, userCred mccl } func (self *SCachedimage) PerformRefresh(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - img, err := CachedimageManager.getImageById(ctx, userCred, self.Id, true) + img, err := CachedimageManager.GetImageById(ctx, userCred, self.Id, true) if err != nil { return nil, err } @@ -257,6 +258,52 @@ func (self *SCachedimage) addRefCount() { } } +func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs interface{}) (*SStoragecachedimage, error) { + storageCachedImage := StoragecachedimageManager.Query().SubQuery() + storage := StorageManager.Query().SubQuery() + hostStorage := HoststorageManager.Query().SubQuery() + host := HostManager.Query().SubQuery() + + scimgs := make([]SStoragecachedimage, 0) + q := storageCachedImage.Query(). + Join(storage, sqlchemy.AND(sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id")))). + Join(hostStorage, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id")))). + Join(host, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id")))). + Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)). + Filter(sqlchemy.Equals(storageCachedImage.Field("status"), CACHED_IMAGE_STATUS_READY)). + Filter(sqlchemy.Equals(host.Field("status"), HOST_STATUS_RUNNING)). + Filter(sqlchemy.IsTrue(host.Field("enabled"))). + Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)) + + if len(excludes) > 0 { + q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes)) + } + if len(hostType) > 0 { + q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType)) + } + + switch v := rangeObjs.(type) { + case []*SZone: + for _, obj := range v { + q = q.Filter(sqlchemy.Equals(host.Field("zone_id"), obj.Id)) + } + case []*SVCenter: + for _, obj := range v { + q = q.Filter(sqlchemy.Equals(host.Field("manager_id"), obj.Id)) + } + } + err := q.All(&scimgs) + if err != nil { + return nil, err + } + if len(scimgs) == 0 { + return nil, nil + } + + rand.Seed(time.Now().Unix()) + return &scimgs[rand.Intn(len(scimgs))], nil +} + func (manager *SCachedimageManager) ImageAddRefCount(imageId string) { cachedObj, _ := manager.FetchById(imageId) if cachedObj != nil { diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 3ac4efcade..e3c31efbba 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -220,7 +220,7 @@ func (self *SCloudregion) AllowPerformDefaultVpc(ctx context.Context, userCred m } func (self *SCloudregion) PerformDefaultVpc(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - vpcs, err := VpcManager.getVpcsByRegion(self) + vpcs, err := VpcManager.getVpcsByRegion(self, nil) if err != nil { return nil, err } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 047863499a..a2e827b7b9 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,10 +16,12 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -37,6 +39,7 @@ const ( DISK_DEALLOC = "deallocating" DISK_DEALLOC_FAILED = "dealloc_failed" DISK_UNKNOWN = "unknown" + DISK_DETACHING = "detaching" DISK_START_SAVE = "start_save" DISK_SAVING = "saving" @@ -86,6 +89,10 @@ type SDisk struct { Nonpersistent bool `default:"false" list:"user"` // Column(Boolean, default=False) } +func (manager *SDiskManager) GetContextManager() []db.IModelManager { + return []db.IModelManager{StorageManager} +} + func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { @@ -109,6 +116,14 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu sq := storages.Query(storages.Field("id")).Filter(sqlchemy.In(storages.Field("storage_type"), STORAGE_LOCAL_TYPES)) q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) } + if provier, _ := queryDict.GetString("provider"); len(provier) > 0 { + cloudprovider := CloudproviderManager.Query().SubQuery() + sq := storages.Query(storages.Field("id")).Join(cloudprovider, + sqlchemy.AND( + sqlchemy.Equals(cloudprovider.Field("id"), storages.Field("manager_id")), + sqlchemy.Equals(cloudprovider.Field("provider"), provier))) + q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) + } guestId, _ := queryDict.GetString("guest") if len(guestId) != 0 { guest := GuestManager.FetchGuestById(guestId) @@ -141,6 +156,10 @@ func (self *SDisk) GetGuestDiskCount() int { return guestdisks.Equals("disk_id", self.Id).Count() } +func (self *SDisk) isAttached() bool { + return GuestdiskManager.Query().Equals("disk_id", self.Id).Count() > 0 +} + func (self *SDisk) GetGuestdisks() []SGuestdisk { guestdisks := make([]SGuestdisk, 0) q := GuestdiskManager.Query().Equals("disk_id", self.Id) @@ -151,6 +170,193 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk { } return guestdisks } +func (self *SDisk) GetGuests() []SGuest { + result := make([]SGuest, 0) + query := GuestManager.Query() + guestdisks := GuestdiskManager.Query().SubQuery() + q := query.Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), query.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) + // q.DebugQuery() + err := db.FetchModelObjects(GuestManager, q, &result) + if err != nil { + log.Errorf(err.Error()) + return nil + } + return result +} + +func (self *SDisk) GetGuestsCount() int { + guests := GuestManager.Query().SubQuery() + guestdisks := GuestdiskManager.Query().SubQuery() + return guests.Query().Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)).Count() +} + +func (self *SDisk) GetRuningGuestCount() int { + guests := GuestManager.Query().SubQuery() + guestdisks := GuestdiskManager.Query().SubQuery() + return guests.Query().Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)). + Filter(sqlchemy.Equals(guests.Field("status"), VM_RUNNING)).Count() +} + +func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + diskConfig := SDiskConfig{} + if err := data.Unmarshal(&diskConfig, "disk"); err != nil { + return err + } else { + self.fetchDiskInfo(&diskConfig) + } + return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) +} + +func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if disk, err := data.Get("disk"); err != nil { + return nil, err + } else { + if diskConfig, err := parseDiskInfo(ctx, userCred, disk); err != nil { + return nil, err + } else { + data.Add(jsonutils.Marshal(diskConfig), "disk") + if storageID, err := data.GetString("storage_id"); err != nil { + return nil, err + } else { + storages := StorageManager.Query().SubQuery() + storage := SStorage{} + storage.SetModelManager(StorageManager) + if err := storages.Query().Equals("id", storageID).First(&storage); err != nil { + return nil, err + } + if !storage.Enabled { + return nil, httperrors.NewInputParameterError("Cannot create disk with disabled storage[%s]", storage.Name) + } + if !utils.IsInStringArray(storage.Status, []string{STORAGE_ENABLED, STORAGE_ONLINE}) { + return nil, httperrors.NewInputParameterError("Cannot create disk with offline storage[%s]", storage.Name) + } + if storage.StorageType != diskConfig.Backend { + return nil, httperrors.NewInputParameterError("Storage type[%s] not match backend %s", storage.StorageType, diskConfig.Backend) + } + size := diskConfig.Size >> 10 + if storage.StorageType == STORAGE_RBD { + diskConfig.Format = "raw" + data.Add(jsonutils.Marshal(diskConfig), "disk") + } else if storage.StorageType == STORAGE_CLOUD_EFFICIENCY || storage.StorageType == STORAGE_CLOUD_SSD { + if size < 20 || size > 32768 { + return nil, httperrors.NewInputParameterError("cloud_ssd or cloud_efficiency disk only support 20G ~ 32768G") + } + } else if storage.StorageType == STORAGE_PUBLIC_CLOUD { + if size < 5 || size > 2000 { + return nil, httperrors.NewInputParameterError("cloud disk only support 5G ~ 2000G") + } + } + hoststorages := HoststorageManager.Query().SubQuery() + hoststorage := make([]SHoststorage, 0) + if err := hoststorages.Query().Equals("storage_id", storage.Id).All(&hoststorage); err != nil { + return nil, err + } + if len(hoststorage) == 0 { + return nil, httperrors.NewInputParameterError("Storage[%s] must attach to a host", storage.Name) + } + if diskConfig.Size > storage.GetFreeCapacity() && !storage.IsEmulated { + return nil, httperrors.NewInputParameterError("Not enough free space") + } + if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil { + return nil, err + } + pendingUsage := SQuota{Storage: diskConfig.Size} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { + return nil, err + } + } + } + } + return data, nil +} + +func (disk *SDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + disk.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) + disk.StartDiskCreateTask(ctx, userCred, false, "", "") +} + +func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { + kwargs := jsonutils.NewDict() + if rebuild { + kwargs.Add(jsonutils.JSONTrue, "rebuild") + } + if len(snapshot) > 0 { + kwargs.Add(jsonutils.NewString(snapshot), "snapshot") + } + if task, err := taskman.TaskManager.NewTask(ctx, "DiskCreateTask", self, userCred, kwargs, parentTaskId, "", nil); err != nil { + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + +func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { + log.Infof("Allocating disk on host %s ...", host.GetName()) + + templateId := self.GetTemplateId() + fsFormat := self.GetFsFormat() + + content := jsonutils.NewDict() + content.Add(jsonutils.NewString(self.DiskFormat), "format") + content.Add(jsonutils.NewInt(int64(self.DiskSize)), "size") + if len(snapshot) > 0 { + content.Add(jsonutils.NewString(snapshot), "snapshot") + } else if len(templateId) > 0 { + content.Add(jsonutils.NewString(templateId), "image_id") + } + if len(fsFormat) > 0 { + content.Add(jsonutils.NewString(fsFormat), "fs_format") + if fsFormat == "ext4" { + name := strings.ToLower(self.GetName()) + for _, key := range []string{"encrypt", "secret", "cipher", "private"} { + if strings.Index(key, name) > 0 { + content.Add(jsonutils.JSONTrue, "encryption") + break + } + } + } + } + if rebuild { + content.Add(jsonutils.JSONTrue, "rebuild") + } + return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content) +} + +func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if sizeStr, err := data.GetString("size"); err != nil { + return nil, err + } else if size, err := fileutils.GetSizeMb(sizeStr, 'M', 1024); err != nil { + return nil, err + } else if self.Status != DISK_READY { + return nil, httperrors.NewResourceNotReadyError("Resize disk when disk is READY") + } else if size < self.DiskSize { + return nil, httperrors.NewUnsupportOperationError("Disk cannot be thrink") + } else if size == self.DiskSize { + return nil, nil + } else { + addDisk := size - self.DiskSize + storage := self.GetStorage() + if addDisk > storage.GetFreeCapacity() && !storage.IsEmulated { + return nil, httperrors.NewOutOfResourceError("Not enough free space") + } + pendingUsage := SQuota{Storage: int(addDisk)} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { + return nil, httperrors.NewOutOfQuotaError(err.Error()) + } + return nil, self.StartDiskResizeTask(ctx, userCred, int64(size), "", &pendingUsage) + } +} func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { if self.GetGuestDiskCount() > 0 { @@ -159,10 +365,6 @@ func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx) } -func (self *SDisk) StartAllocate(host *SHost, Storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool) { - -} - func (self *SDisk) GetTemplateId() string { return self.TemplateId } @@ -183,6 +385,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -307,6 +516,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extDisk.Refresh() self.Name = extDisk.GetName() self.Status = extDisk.GetStatus() self.DiskFormat = extDisk.GetDiskFormat() @@ -401,6 +611,7 @@ type SDiskConfig struct { Cache string // Mountpoint string // Backend string // stroageType + Medium string ImageProperties map[string]string } @@ -415,6 +626,11 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info } return &diskConfig, nil } + + // default backend and medium type + diskConfig.Backend = STORAGE_LOCAL + diskConfig.Medium = DISK_TYPE_HYBRID + diskStr, err := info.GetString() if err != nil { log.Errorf("invalid diskinfo format %s", err) @@ -432,6 +648,8 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info diskConfig.Driver = p } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { diskConfig.Cache = p + } else if utils.IsInStringArray(p, DISK_TYPES) { + diskConfig.Medium = p } else if p[0] == '/' { diskConfig.Mountpoint = p } else if p == "autoextend" { @@ -536,6 +754,60 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewInt(size), "size") + if task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage); err != nil { + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + func (self *SDisk) StartDiskDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, isPurge bool) error { params := jsonutils.NewDict() if isPurge { diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index 82c63895df..1f69a83d0d 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -6,9 +6,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SGuestdiskManager struct { @@ -47,6 +49,23 @@ func (self *SGuestdisk) AllowDeleteItem(ctx context.Context, userCred mcclient.T return false } +func (self *SGuestdisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if data.Contains("index") { + if index, err := data.Int("index"); err != nil { + return nil, err + } else { + guestdisk := GuestdiskManager.Query().SubQuery() + count := guestdisk.Query().Filter(sqlchemy.Equals(guestdisk.Field("guest_id"), self.GuestId)). + Filter(sqlchemy.NotEquals(guestdisk.Field("disk_id"), self.DiskId)). + Filter(sqlchemy.Equals(guestdisk.Field("index"), index)).Count() + if count > 0 { + return nil, httperrors.NewInputParameterError("DISK Index %d has been occupied", index) + } + } + } + return self.SGuestJointsBase.ValidateUpdateData(ctx, userCred, query, data) +} + func (joint *SGuestdisk) Master() db.IStandaloneModel { return db.JointMaster(joint) } @@ -111,9 +130,11 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { templateId := disk.GetTemplateId() if len(templateId) > 0 { desc.Add(jsonutils.NewString(templateId), "template_id") - // hostcachedimg = Hostcachedimages.get_host_cachedimage(host, template_id) - // if hostcachedimg is not None: - // desc['image_path'] = hostcachedimg.path + storage := disk.GetStorage() + storagecacheimg := StoragecachedimageManager.GetStoragecachedimage(storage.StoragecacheId, templateId) + if storagecacheimg != nil { + desc.Add(jsonutils.NewString(storagecacheimg.Path), "image_path") + } } if host.HostType == HOST_TYPE_HYPERVISOR && disk.IsLocal() { desc.Add(jsonutils.NewString(disk.StorageId), "storage_id") @@ -129,6 +150,8 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } } desc.Add(jsonutils.NewString(disk.DiskFormat), "format") + desc.Add(jsonutils.NewInt(int64(self.Index)), "index") + tid := disk.GetTemplateId() if len(tid) > 0 { desc.Add(jsonutils.NewString(tid), "template_id") @@ -147,6 +170,25 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { return desc } +func (self *SGuestdisk) GetDetailedJson() *jsonutils.JSONDict { + desc := jsonutils.NewDict() + disk := self.GetDisk() + storage := disk.GetStorage() + if fs := disk.GetFsFormat(); len(fs) > 0 { + desc.Add(jsonutils.NewString(fs), "fs") + } + desc.Add(jsonutils.NewString(disk.DiskType), "disk_type") + desc.Add(jsonutils.NewInt(int64(self.Index)), "index") + desc.Add(jsonutils.NewInt(int64(disk.DiskSize)), "size") + desc.Add(jsonutils.NewString(disk.DiskFormat), "disk_format") + desc.Add(jsonutils.NewString(self.Driver), "driver") + desc.Add(jsonutils.NewString(self.CacheMode), "cache_mode") + desc.Add(jsonutils.NewString(self.AioMode), "aio_mode") + desc.Add(jsonutils.NewString(storage.MediumType), "medium_type") + desc.Add(jsonutils.NewString(storage.StorageType), "storage_type") + return desc +} + func (self *SGuestdisk) GetDetailedString() string { disk := self.GetDisk() var fs string diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index f35b77e915..48406df24f 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -39,6 +39,9 @@ type IGuestDriver interface { RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *SGuest, task taskman.ITask) error StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error + + RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error @@ -48,13 +51,15 @@ type IGuestDriver interface { StartGuestSyncstatusTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error + RequestSyncConfigOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error + RequestSyncstatusOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) - RequestStartOnHost(guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) + RequestStartOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) RequestStopOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error - StartDeleteGuestTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error RequestStopGuestForDelete(ctx context.Context, guest *SGuest, task taskman.ITask) error @@ -69,6 +74,23 @@ type IGuestDriver interface { CheckDiskTemplateOnStorage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, storageId string, task taskman.ITask) error GetGuestVncInfo(userCred mcclient.TokenCredential, guest *SGuest, host *SHost) (*jsonutils.JSONDict, error) + + RequestDetachDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error + GetDetachDiskStatus() ([]string, error) + CanKeepDetachDisk() bool + + RequestDeleteDetachedDisk(ctx context.Context, disk *SDisk, task taskman.ITask, isPurge bool) error + StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + + StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + RqeuestSuspendOnHost(ctx context.Context, guest *SGuest, task taskman.ITask) error + + AllowReconfigGuest() bool + DoGuestCreateDisksTask(ctx context.Context, guest *SGuest, task taskman.ITask) error + RequestChangeVmConfig(ctx context.Context, guest *SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error + + RequestGuestHotAddIso(ctx context.Context, guest *SGuest, path string, task taskman.ITask) error + RequestRebuildRootDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..5ff6910060 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -315,9 +315,11 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest if regutils.MatchIP4Addr(gn.IpAddr) || regutils.MatchIP6Addr(gn.Ip6Addr) { net.updateDnsRecord(&gn, false) if regutils.MatchIP4Addr(gn.IpAddr) { + // ?? // netman.get_manager().netmap_remove_node(gn.ip_addr) } } + // ?? // gn.Delete(ctx, userCred) err = gn.Delete(ctx, userCred) if err != nil { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 401bdbd1b0..c5dec34277 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "fmt" + "net/http" "strconv" "strings" "time" @@ -17,6 +18,7 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" @@ -32,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -535,6 +538,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m hypervisor = HYPERVISOR_BAREMETAL } + // base validate_create_data if data.Contains("prefer_baremetal") || data.Contains("prefer_host") { if !userCred.IsSystemAdmin() { return nil, httperrors.NewNotSufficientPrivilegeError("Only system admin can specify preferred host") @@ -572,7 +576,6 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, err } - } else { schedtags := make(map[string]string) if data.Contains("aggregate_strategy") { @@ -627,12 +630,16 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m } } + // default hypervisor + if len(hypervisor) == 0 { + hypervisor = HYPERVISOR_KVM + } + if !utils.IsInStringArray(hypervisor, HYPERVISORS) { return nil, httperrors.NewInputParameterError("Hypervisor %s not supported", hypervisor) } data.Add(jsonutils.NewString(hypervisor), "hypervisor") - for idx := 1; data.Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { diskJson, err := data.Get(fmt.Sprintf("disk.%d", idx)) if err != nil { @@ -819,13 +826,27 @@ func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCr func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { guest.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - + tags := []string{"cpu_bound", "io_bound", "io_hardlimit"} + appTags := make([]string, 0) + for _, tag := range tags { + if data.Contains(tag) { + appTags = append(appTags, tag) + } + } + guest.setApptags(ctx, appTags, userCred) osProfileJson, _ := data.Get("__os_profile__") if osProfileJson != nil { guest.setOSProfile(ctx, userCred, osProfileJson) } } +func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred mcclient.TokenCredential) { + err := guest.SetMetadata(ctx, "app_tags", strings.Join(appTags, ","), userCred) + if err != nil { + log.Errorln(err) + } +} + func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { pendingUsage := getGuestResourceRequirements(ctx, userCred, data, len(items)) @@ -890,7 +911,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") } - extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") if self.PendingDeleted { pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) @@ -904,6 +925,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token extra := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query) extra.Add(jsonutils.NewString(self.getNetworksDetails()), "networks") extra.Add(jsonutils.NewString(self.getDisksDetails()), "disks") + extra.Add(self.getDisksInfoDetails(), "disks_info") extra.Add(jsonutils.NewInt(int64(self.getDiskSize())), "disk") cdrom := self.getCdrom() if cdrom != nil { @@ -911,7 +933,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token } // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") - extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") extra.Add(jsonutils.NewString(strings.Join(self.getIPs(), ",")), "ips") extra.Add(jsonutils.NewString(self.getSecurityRules()), "security_rules") extra.Add(jsonutils.NewString(self.getIsolatedDeviceDetails()), "isolated_devices") @@ -930,6 +952,8 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token if zone != nil { extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") extra.Add(jsonutils.NewString(zone.GetName()), "zone") + extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") + extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") } return extra } @@ -952,6 +976,14 @@ func (self *SGuest) getDisksDetails() string { return buf.String() } +func (self *SGuest) getDisksInfoDetails() *jsonutils.JSONArray { + details := jsonutils.NewArray() + for _, disk := range self.GetDisks() { + details.Add(disk.GetDetailedJson()) + } + return details +} + func (self *SGuest) getIsolatedDeviceDetails() string { var buf bytes.Buffer for _, dev := range self.GetIsolatedDevices() { @@ -1098,7 +1130,7 @@ func (self *SGuest) getAdminSecgroup() *SSecurityGroup { return SecurityGroupManager.FetchSecgroupById(self.AdminSecgrpId) } -func (self *SGuest) getSecgroupName() string { +func (self *SGuest) GetSecgroupName() string { secgrp := self.getSecgroup() if secgrp != nil { return secgrp.GetName() @@ -1114,6 +1146,22 @@ func (self *SGuest) getAdminSecgroupName() string { return "" } +func (self *SGuest) GetSecRules() []secrules.SecurityRule { + return self.getSecRules() +} + +func (self *SGuest) getSecRules() []secrules.SecurityRule { + if secgrp := self.getSecgroup(); secgrp != nil { + return secgrp.getSecRules() + } + if rule, err := secrules.ParseSecurityRule(options.Options.DefaultSecurityRules); err == nil { + return []secrules.SecurityRule{*rule} + } else { + log.Errorf("Default SecurityRules error: %v", err) + } + return []secrules.SecurityRule{} +} + func (self *SGuest) getSecurityRules() string { secgrp := self.getSecgroup() if secgrp != nil { @@ -1138,7 +1186,7 @@ func (self *SGuest) GetIsolatedDevices() []SIsolatedDevice { func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM) error { diff, err := GuestManager.TableSpec().Update(self, func() error { - + extVM.Refresh() self.Name = extVM.GetName() self.Status = extVM.GetStatus() self.VcpuCount = extVM.GetVcpuCount() @@ -1439,6 +1487,122 @@ func (self *SGuest) attach2Disk(disk *SDisk, userCred mcclient.TokenCredential, return err } +func (self *SGuest) AllowPerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if err := self.StartSyncTask(ctx, userCred, false, ""); err != nil { + return nil, err + } + return nil, nil +} + +func (self *SGuest) AllowPerformDeploy(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + kwargs, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil, fmt.Errorf("Parse query body error") + } + if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { + var kpId string + if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { + keypair, _ := kwargs.GetString("keypair") + iKp, err := KeypairManager.FetchByIdOrName(userCred.GetProjectId(), keypair) + if err != nil { + return nil, err + } + if iKp == nil { + return nil, fmt.Errorf("Fetch keypair error") + } + kp := iKp.(*SKeypair) + kpId = kp.Id + } + if self.KeypairId != kpId { + self.GetModelManager().TableSpec().Update(self, func() error { + self.KeypairId = kpId + return nil + }) + kwargs.Set("reset_password", jsonutils.JSONTrue) + } + } + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) { + if self.Status == VM_RUNNING { + kwargs.Set("restart", jsonutils.JSONTrue) + } + err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") + if err != nil { + return nil, err + } + return nil, nil + } + return nil, httperrors.NewServerStatusError("Cannot deploy in status %s", self.Status) +} + +func (self *SGuest) AllowPerformAttachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) ValidateAttachDisk(ctx context.Context, disk *SDisk) error { + if disk.isAttached() { + return httperrors.NewInputParameterError("Disk %s has been attached", disk.Name) + } else if len(disk.GetPathAtHost(self.GetHost())) == 0 { + return httperrors.NewInputParameterError("Disk %s not belong the guest's host", disk.Name) + } else if disk.Status != DISK_READY { + return httperrors.NewInputParameterError("Disk in %s not able to attach", disk.Status) + } else if !utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY}) { + return httperrors.NewInputParameterError("Server in %s not able to attach disk", self.Status) + } + return nil +} + +func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if diskId, err := data.GetString("disk_id"); err != nil { + return nil, err + } else { + if disk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId); err != nil { + return nil, err + } else if disk == nil { + return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) + } else if err := self.ValidateAttachDisk(ctx, disk.(*SDisk)); err != nil { + return nil, err + } else { + driver, _ := data.GetString("driver") + cache, _ := data.GetString("cache") + mountpoint, _ := data.GetString("mountpoint") + if err := self.attach2Disk(disk.(*SDisk), userCred, driver, cache, mountpoint); err != nil { + return nil, err + } else { + self.StartSyncTask(ctx, userCred, false, "") + } + } + } + return nil, nil +} + +func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, fw_only bool, parentTaskId string) error { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING}) { + return httperrors.NewResourceBusyError("Cannot sync in status %s", self.Status) + } + data := jsonutils.NewDict() + if fw_only { + data.Add(jsonutils.JSONTrue, "fw_only") + } else if err := self.SetStatus(userCred, VM_SYNC_CONFIG, ""); err != nil { + log.Errorf(err.Error()) + return err + } + if task, err := taskman.TaskManager.NewTask(ctx, "GuestSyncConfTask", self, userCred, data, parentTaskId, "", nil); err != nil { + log.Errorf(err.Error()) + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + type sSyncDiskPair struct { disk *SDisk vdisk cloudprovider.ICloudDisk @@ -1905,11 +2069,26 @@ func (self *SGuest) insertIso(imageId string) bool { return cdrom.insertIso(imageId) } -func (self *SGuest) insertIsoSucc(imageId string, path string, size int, name string) bool { +func (self *SGuest) InsertIsoSucc(imageId string, path string, size int, name string) bool { cdrom := self.getCdrom() return cdrom.insertIsoSucc(imageId, path, size, name) } +func (self *SGuest) GetDetailsIso(userCred mcclient.TokenCredential) jsonutils.JSONObject { + cdrom := self.getCdrom() + desc := jsonutils.NewDict() + if len(cdrom.ImageId) > 0 { + desc.Set("image_id", jsonutils.NewString(cdrom.ImageId)) + desc.Set("status", jsonutils.NewString("inserting")) + } + if len(cdrom.Path) > 0 { + desc.Set("name", jsonutils.NewString(cdrom.Name)) + desc.Set("size", jsonutils.NewInt(int64(cdrom.Size))) + desc.Set("status", jsonutils.NewString("ready")) + } + return desc +} + func (self *SGuest) StartInsertIsoTask(ctx context.Context, imageId string, hostId string, userCred mcclient.TokenCredential, parentTaskId string) error { self.insertIso(imageId) @@ -1917,7 +2096,7 @@ func (self *SGuest) StartInsertIsoTask(ctx context.Context, imageId string, host data.Add(jsonutils.NewString(imageId), "image_id") data.Add(jsonutils.NewString(hostId), "host_id") - task, err := taskman.TaskManager.NewTask(ctx, "GuestInsertISOTask", self, userCred, data, parentTaskId, "", nil) + task, err := taskman.TaskManager.NewTask(ctx, "GuestInsertIsoTask", self, userCred, data, parentTaskId, "", nil) if err != nil { return err } @@ -1935,6 +2114,15 @@ func (self *SGuest) StartGueststartTask(ctx context.Context, userCred mcclient.T return nil } +func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateDiskTask", self, userCred, data, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SGuest) StartSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { return self.GetDriver().StartGuestSyncstatusTask(self, ctx, userCred, parentTaskId) } @@ -1954,7 +2142,55 @@ func (self *SGuest) StartDeleteGuestTask(ctx context.Context, userCred mcclient. params.Add(jsonutils.JSONTrue, "override_pending_delete") } self.SetStatus(userCred, VM_START_DELETE, "") - return self.GetDriver().StartDeleteGuestTask(self, ctx, userCred, params, parentTaskId) + return self.GetDriver().StartDeleteGuestTask(ctx, userCred, self, params, parentTaskId) +} + +func (self *SGuest) AllowPerformAssignSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) AllowPerformRevokeSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformRevokeSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_SUSPEND}) { + return nil, httperrors.NewInputParameterError("Cannot revoke security rules in status %s", self.Status) + } else { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.SecgrpId = "" + return nil + }); err != nil { + return nil, err + } + if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil { + return nil, err + } + } + return nil, nil +} + +func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_SUSPEND}) { + return nil, httperrors.NewInputParameterError("Cannot assign security rules in status %s", self.Status) + } else { + if secgrp, err := data.GetString("secgrp"); err != nil { + return nil, err + } else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secgrp); err != nil { + return nil, httperrors.NewNotFoundError("SecurityGroup %s not found", secgrp) + } else { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.SecgrpId = sg.GetId() + return nil + }); err != nil { + return nil, err + } + if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil { + return nil, err + } + } + } + return nil, nil } func (self *SGuest) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -1974,13 +2210,265 @@ func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCre return nil, err } -func (self *SGuest) detachDisk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential) { +func (self *SGuest) AllowPerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + imageId, _ := data.GetString("image_id") + if utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_ADMIN}) { + if !data.Contains("image_id") { + gdc := self.CategorizeDisks() + imageId = gdc.Root.GetTemplateId() + if len(imageId) == 0 { + return nil, httperrors.NewBadRequestError("No template for root disk") + } + img, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) + if err != nil { + return nil, httperrors.NewBadRequestError("Template %s not accessible: %s", imageId, err.Error()) + } + osType, _ := img.Properties["os_type"] + osName := self.GetMetadata("os_name", userCred) + if len(osName) == 0 && len(osType) == 0 && strings.ToLower(osType) != strings.ToLower(osName) { + return nil, httperrors.NewBadRequestError("Cannot switch OS between %s-%s", osName, osType) + } + } + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) + var needStop = false + if self.Status == VM_RUNNING { + needStop = true + } + err := self.StartRebuildRootTask(ctx, userCred, imageId, needStop, autoStart) + return nil, err + } + return nil, httperrors.NewInvalidStatusError("Cannot reset root in status %s", self.Status) +} + +func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, needStop, autoStart bool) error { + data := jsonutils.NewDict() + data.Set("image_id", jsonutils.NewString(imageId)) + if needStop { + data.Set("need_stop", jsonutils.JSONTrue) + } + if autoStart { + data.Set("auto_start", jsonutils.JSONTrue) + } + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + task, err := taskman.TaskManager.NewTask(ctx, "BaremetalServerRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } else { + task, err := taskman.TaskManager.NewTask(ctx, "GuestRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } + return nil +} + +func (self *SGuest) DetachDisk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential) { guestdisk := self.GetGuestDisk(disk.Id) if guestdisk != nil { guestdisk.Detach(ctx, userCred) } } +func (self *SGuest) AllowPerformDetachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + diskId, err := data.GetString("disk_id") + if err != nil { + return nil, err + } + keepDisk := jsonutils.QueryBoolean(data, "keep_disk", false) + iDisk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId) + if err != nil { + return nil, err + } + disk := iDisk.(*SDisk) + if disk != nil { + if self.isAttach2Disk(disk) { + detachDiskStatus, err := self.GetDriver().GetDetachDiskStatus() + if err != nil { + return nil, err + } + if keepDisk && !self.GetDriver().CanKeepDetachDisk() { + return nil, httperrors.NewInputParameterError("Cannot keep detached disk") + } + if utils.IsInStringArray(self.Status, detachDiskStatus) { + if disk.Status == DISK_INIT { + disk.SetStatus(userCred, DISK_DETACHING, "") + } + taskData := jsonutils.NewDict() + taskData.Add(jsonutils.NewString(diskId), "disk_id") + taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk") + self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "") + return nil, nil + } else { + return nil, httperrors.NewInvalidStatusError("Server in %s not able to detach disk", self.Status) + } + } else { + return nil, httperrors.NewInvalidStatusError("Disk %s not attached", diskId) + } + } + return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) +} + +func (self *SGuest) AllowPerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) || self.IsAdmin(userCred) +} + +func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY}) { + return nil, httperrors.NewInvalidStatusError("Cannot change config in %s", self.Status) + } + if !self.GetDriver().AllowReconfigGuest() { + return nil, httperrors.NewInvalidStatusError("Not allow to change config") + } + host := self.GetHost() + if host == nil { + return nil, httperrors.NewInvalidStatusError("No valid host") + } + var addCpu, addMem int + confs := jsonutils.NewDict() + vcpuCount, err := data.GetString("vcpu_count") + if err == nil { + nVcpu, err := strconv.ParseInt(vcpuCount, 10, 0) + if err != nil { + return nil, httperrors.NewBadRequestError("Params vcpu_count parse error") + } + err = confs.Add(jsonutils.NewInt(nVcpu), "vcpu_count") + if err != nil { + return nil, httperrors.NewBadRequestError("Params vcpu_count parse error") + } + addCpu = int(nVcpu - int64(self.VcpuCount)) + } + vmemSize, err := data.GetString("vmem_size") + if err == nil { + if !regutils.MatchSize(vmemSize) { + return nil, httperrors.NewBadRequestError("Memory size must be number[+unit], like 256M, 1G or 256") + } + nVmem, err := fileutils.GetSizeMb(vmemSize, 'M', 1024) + if err != nil { + httperrors.NewBadRequestError("Params vmem_size parse error") + } + err = confs.Add(jsonutils.NewInt(int64(nVmem)), "vmem_size") + if err != nil { + return nil, httperrors.NewBadRequestError("Params vmem_size parse error") + } + addMem = nVmem - self.VmemSize + } + disks := self.GetDisks() + var addDisk int + var diskIdx = 1 + var newDiskIdx = 0 + var diskSizes = make(map[string]int, 0) + var newDisks = jsonutils.NewDict() + var resizeDisks = jsonutils.NewArray() + for { + diskNum := fmt.Sprintf("disk.%d", diskIdx) + diskDesc, err := data.Get(diskNum) + if err != nil { + break + } + diskConf, err := parseDiskInfo(ctx, userCred, diskDesc) + if err != nil { + return nil, httperrors.NewBadRequestError("Parse disk info error: %s", err) + } + if diskConf.Size > 0 { + if diskIdx >= len(disks) { + newDisks.Add(jsonutils.Marshal(diskConf), fmt.Sprintf("disk.%d", newDiskIdx)) + newDiskIdx += 1 + addDisk += diskConf.Size + storage := host.GetLeastUsedStorage(diskConf.Backend) + _, ok := diskSizes[storage.Id] + if !ok { + diskSizes[storage.Id] = 0 + } + diskSizes[storage.Id] = diskSizes[storage.Id] + diskConf.Size + } else { + disk := disks[diskIdx].GetDisk() + oldSize := disk.DiskSize + if diskConf.Size < oldSize { + return nil, httperrors.NewInputParameterError("Cannot reduce disk size") + } else if diskConf.Size > oldSize { + arr := jsonutils.NewArray(jsonutils.NewString(disks[diskIdx].DiskId), jsonutils.NewInt(int64(diskConf.Size))) + resizeDisks.Add(arr) + addDisk += diskConf.Size - oldSize + storage := disks[diskIdx].GetDisk().GetStorage() + _, ok := diskSizes[storage.Id] + if !ok { + diskSizes[storage.Id] = 0 + } + diskSizes[storage.Id] = diskSizes[storage.Id] + diskConf.Size - oldSize + } + } + } + diskIdx += 1 + } + + for storageId, needSize := range diskSizes { + iStorage, err := StorageManager.FetchById(storageId) + if err != nil { + return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) + } + storage := iStorage.(*SStorage) + if storage.GetFreeCapacity() < needSize { + return nil, httperrors.NewInsufficientResourceError("Not enough free space") + } + } + if newDisks.Length() > 0 { + confs.Add(newDisks, "create") + } + if resizeDisks.Length() > 0 { + confs.Add(resizeDisks, "resize") + } + if jsonutils.QueryBoolean(data, "auto_start", false) { + confs.Add(jsonutils.NewBool(true), "auto_start") + } + pendingUsage := &SQuota{} + if addCpu > 0 { + pendingUsage.Cpu = addCpu + } + if addMem > 0 { + pendingUsage.Memory = addMem + } + if addDisk > 0 { + pendingUsage.Storage = addDisk + } + if !pendingUsage.IsEmpty() { + err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), pendingUsage) + if err != nil { + return nil, httperrors.NewBadRequestError("Check set pending quota error %s", err) + } + } + if newDisks.Length() > 0 { + err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage) + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) + return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err) + } + } + self.StartChangeConfigTask(ctx, userCred, confs, "", pendingUsage) + return nil, nil +} + +func (self *SGuest) StartChangeConfigTask(ctx context.Context, userCred mcclient.TokenCredential, + data *jsonutils.JSONDict, parentTaskId string, pendingUsage quotas.IQuota) error { + self.SetStatus(userCred, VM_CHANGE_FLAVOR, "") + task, err := taskman.TaskManager.NewTask(ctx, "GuestChangeConfigTask", self, userCred, data, parentTaskId, "", pendingUsage) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) { for _, guestdisk := range self.GetDisks() { disk := guestdisk.GetDisk() @@ -1988,7 +2476,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token if utils.IsInStringArray(storage.StorageType, sysutils.LOCAL_STORAGE_TYPES) || disk.DiskType == DISK_TYPE_SYS || disk.DiskType == DISK_TYPE_SWAP || self.Hypervisor == HYPERVISOR_ALIYUN { disk.DoPendingDelete(ctx, userCred) } else { - self.detachDisk(ctx, disk, userCred) + self.DetachDisk(ctx, disk, userCred) } } self.SVirtualResourceBase.DoPendingDelete(ctx, userCred) @@ -2028,7 +2516,24 @@ func (self *SGuest) StartUndeployGuestTask(ctx context.Context, userCred mcclien } func (self *SGuest) LeaveAllGroups(userCred mcclient.TokenCredential) { - // TODO + groupGuests := make([]SGroupguest, 0) + q := GroupguestManager.Query() + err := q.Filter(sqlchemy.Equals(q.Field("guest_id"), self.Id)).All(&groupGuests) + if err != nil { + log.Errorln(err.Error()) + return + } + for _, gg := range groupGuests { + gg.Delete(context.Background(), userCred) + var group SGroup + gq := GroupManager.Query() + err := gq.Filter(sqlchemy.Equals(gq.Field("id"), gg.SrvtagId)).First(&group) + if err != nil { + log.Errorln(err.Error()) + return + } + db.OpsLog.LogDetachEvent(self, &group, userCred, nil) + } } func (self *SGuest) DetachAllNetworks(ctx context.Context, userCred mcclient.TokenCredential) error { @@ -2076,9 +2581,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } @@ -2110,6 +2615,29 @@ func (self *SGuest) PerformSyncstatus(ctx context.Context, userCred mcclient.Tok return nil, err } +func (self *SGuest) isNotRunningStatus(status string) bool { + if status == VM_READY || status == VM_SUSPEND { + return true + } + return false +} + +func (self *SGuest) PerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + preStatus := self.Status + _, err := self.SVirtualResourceBase.PerformStatus(ctx, userCred, query, data) + if err != nil { + return nil, err + } + if preStatus != self.Status && !self.isNotRunningStatus(preStatus) && self.isNotRunningStatus(self.Status) { + db.OpsLog.LogEvent(self, db.ACT_STOP, "", userCred) + if self.Status == VM_READY && !self.DisableDelete.Bool() && self.ShutdownBehavior == SHUTDOWN_TERMINATE { + err = self.StartAutoDeleteGuestTask(ctx, userCred, "") + return nil, err + } + } + return nil, nil +} + type SDeployConfig struct { Path string Action string @@ -2202,6 +2730,16 @@ func (self *SGuest) getExtraOptions() jsonutils.JSONObject { return self.GetMetadataJson("extra_options", nil) } +/* +func (self *SGuest) GetFlavor() *SFlav { + +} + +func (self *SGuest) getFlavorName() string { + f := self.GetFlavor() +} +*/ + func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *jsonutils.JSONDict { desc := jsonutils.NewDict() @@ -2218,13 +2756,14 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j desc.Add(jsonutils.NewString(self.getBios()), "bios") desc.Add(jsonutils.NewString(self.BootOrder), "boot_order") + // isolated devices isolatedDevs := IsolatedDeviceManager.generateJsonDescForGuest(self) - desc.Add(jsonutils.NewArray(isolatedDevs...), "solated_devices") + desc.Add(jsonutils.NewArray(isolatedDevs...), "isolated_devices") + // nics, domain jsonNics := make([]jsonutils.JSONObject, 0) nics := self.GetNetworks() domain := options.Options.DNSDomain - if nics != nil && len(nics) > 0 { for _, nic := range nics { nicDesc := nic.getJsonDescAtHost(host) @@ -2238,6 +2777,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j desc.Add(jsonutils.NewArray(jsonNics...), "nics") desc.Add(jsonutils.NewString(domain), "domain") + // disks jsonDisks := make([]jsonutils.JSONObject, 0) disks := self.GetDisks() if disks != nil && len(disks) > 0 { @@ -2248,17 +2788,19 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j } desc.Add(jsonutils.NewArray(jsonDisks...), "disks") + // cdrom cdDesc := self.getCdrom().getJsonDesc() if cdDesc != nil { desc.Add(cdDesc, "cdrom") } + // tenant tc, _ := self.GetTenantCache(ctx) if tc != nil { desc.Add(jsonutils.NewString(tc.GetName()), "tenant") } - desc.Add(jsonutils.NewString(self.ProjectId), "tenant_id") + // flavor // desc.Add(jsonuitls.NewString(self.getFlavorName()), "flavor") @@ -2516,6 +3058,26 @@ func (self *SGuest) isAllDisksReady() bool { return ready } +func (self *SGuest) AllowPerformSuspend(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformSuspend(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status == VM_RUNNING { + err := self.StartSuspendTask(ctx, userCred) + return nil, err + } + return nil, httperrors.NewInvalidStatusError("Cannot suspend VM in status %s", self.Status) +} + +func (self *SGuest) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential) error { + err := self.SetStatus(userCred, VM_SUSPEND, "do suspend") + if err != nil { + return err + } + return self.GetDriver().StartSuspendTask(ctx, userCred, self, nil, "") +} + func (self *SGuest) AllowPerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, @@ -2528,20 +3090,36 @@ func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCre if utils.IsInStringArray(self.Status, []string{VM_READY, VM_START_FAILED, VM_SAVE_DISK_FAILED, VM_SUSPEND}) { if self.isAllDisksReady() { var kwargs *jsonutils.JSONDict - if data == nil { + if data != nil { kwargs = data.(*jsonutils.JSONDict) } err := self.GetDriver().PerformStart(ctx, userCred, self, kwargs) return nil, err } else { - msg := "Some disk not ready" - return nil, httperrors.NewResourceNotReadyError(msg) + return nil, httperrors.NewInvalidStatusError("Some disk not ready") } } else { return nil, httperrors.NewInvalidStatusError("Cannot do start server in status %s", self.Status) } } +func (self *SGuest) AllowPerformReset(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + isHard := jsonutils.QueryBoolean(data, "is_hard", false) + if self.Status == VM_RUNNING || self.Status == VM_STOP_FAILED { + self.GetDriver().StartGuestResetTask(self, ctx, userCred, isHard, "") + return nil, nil + } + return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status) +} + func (self *SGuest) AllowPerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, @@ -2600,6 +3178,36 @@ func (self *SGuest) GetDetailsVnc(ctx context.Context, userCred mcclient.TokenCr } } +func (self *SGuest) AllowGetDetailsMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) GetDetailsMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_SNAPSHOT_STREAM}) { + cmd, err := query.GetString("command") + if err != nil { + return nil, err + } + return self.SendMonitorCommand(ctx, userCred, cmd) + } + return nil, httperrors.NewInvalidStatusError("Cannot send command in status %s", self.Status) +} + +func (self *SGuest) SendMonitorCommand(ctx context.Context, userCred mcclient.TokenCredential, cmd string) (jsonutils.JSONObject, error) { + host := self.GetHost() + url := fmt.Sprintf("%s/servers/%s/monitor", host.ManagerUri, self.Id) + header := http.Header{} + header.Add("X-Auth-Token", userCred.GetTokenString()) + body := jsonutils.NewDict() + body.Add(jsonutils.NewString(cmd), "cmd") + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + if err != nil { + return nil, err + } + ret := res.(*jsonutils.JSONDict) + return ret, nil +} + func (self *SGuest) GetKeypairPublicKey() string { keypair := self.getKeypair() if keypair != nil { diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go new file mode 100644 index 0000000000..3f162d75da --- /dev/null +++ b/pkg/compute/models/hostdrivers.go @@ -0,0 +1,39 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +) + +type IHostDriver interface { + GetHostType() string + CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error + RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error + RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error + RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error + RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error +} + +var hostDrivers map[string]IHostDriver + +func init() { + hostDrivers = make(map[string]IHostDriver) +} + +func RegisterHostDriver(driver IHostDriver) { + hostDrivers[driver.GetHostType()] = driver +} + +func GetHostDriver(hostType string) IHostDriver { + driver, ok := hostDrivers[hostType] + if ok { + return driver + } else { + log.Fatalf("Unsupported hostType %s", hostType) + return nil + } +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index f4de4e2228..6b47c4e370 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" @@ -65,7 +66,7 @@ const ( HOST_STATUS_CONVERTING = "converting" ) -var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN} +var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN} var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN} type SHostManager struct { @@ -751,9 +752,19 @@ func (self *SHost) DeleteBaremetalnetwork(ctx context.Context, userCred mcclient } } -func (manager *SHostManager) getHostsByZone(zone *SZone) ([]SHost, error) { +func (self *SHost) GetHostDriver() IHostDriver { + if !utils.IsInStringArray(self.HostType, HOST_TYPES) { + log.Fatalf("Unsupported host type %s", self.HostType) + } + return GetHostDriver(self.HostType) +} + +func (manager *SHostManager) getHostsByZone(zone *SZone, provider *SCloudprovider) ([]SHost, error) { hosts := make([]SHost, 0) q := manager.Query().Equals("zone_id", zone.Id) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &hosts) if err != nil { log.Errorf("%s", err) @@ -762,12 +773,12 @@ func (manager *SHostManager) getHostsByZone(zone *SZone) ([]SHost, error) { return hosts, nil } -func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { +func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { localHosts := make([]SHost, 0) remoteHosts := make([]cloudprovider.ICloudHost, 0) syncResult := compare.SyncResult{} - dbHosts, err := manager.getHostsByZone(zone) + dbHosts, err := manager.getHostsByZone(zone, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -1457,6 +1468,8 @@ func (self *SHost) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict if zone != nil { extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") + extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") } server := self.getBaremetalServer() if server != nil { @@ -1533,6 +1546,22 @@ func (manager *SHostManager) GetHostsByManagerAndRegion(managerId string, region return ret } +func (self *SHost) StartImageCacheTask(ctx context.Context, userCred mcclient.TokenCredential, imageId, parentTaskId string, isForce bool) error { + //Todo + // HostcachedimagesManager.Register(userCred, self, imageId) + data := jsonutils.NewDict() + data.Set("image_id", jsonutils.NewString(imageId)) + if isForce { + data.Set("is_force", jsonutils.JSONTrue) + } + task, err := taskman.TaskManager.NewTask(ctx, "StorageCacheImageTask", self, userCred, data, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SHost) Request(userCred mcclient.TokenCredential, method string, url string, headers http.Header, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { s := auth.GetSession(userCred, "", "") _, ret, err := s.JSONRequest(self.ManagerUri, "", method, url, headers, body) diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index e4df6a1ea0..a2d86c4f5c 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -458,6 +458,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extNet.Refresh() self.Name = extNet.GetName() self.Status = extNet.GetStatus() self.GuestIpStart = extNet.GetIpStart() diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 38171617d4..4347ea48c9 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -3,6 +3,7 @@ package models import ( "context" "fmt" + "sort" "strconv" "strings" @@ -11,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/sqlchemy" @@ -39,6 +41,25 @@ type SSecurityGroupRule struct { SecgroupID string `width:"128" charset:"ascii" create:"required"` } +type SecurityGroupRuleSet []SSecurityGroupRule + +func (v SecurityGroupRuleSet) Len() int { + return len(v) +} + +func (v SecurityGroupRuleSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurityGroupRuleSet) Less(i, j int) bool { + if v[i].Priority < v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + func (manager *SSecurityGroupRuleManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return true } @@ -131,6 +152,8 @@ func (manager *SSecurityGroupRuleManager) ValidateCreateData( key += ":" } fields = append(fields, key) + } else if field == "cidr" { + data.Add(jsonutils.NewString("0.0.0.0/0"), "cidr") } } if _, err := secrules.ParseSecurityRule(strings.Join(fields, " ")); err != nil { @@ -167,7 +190,11 @@ func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred fields = append(fields, self.CIDR) } case "protocol": - fields = append(fields, self.Protocol) + protocol := self.Protocol + if protocol == "" { + protocol = secrules.PROTO_ANY + } + fields = append(fields, protocol) case "ports": if len(self.Ports) > 0 { fields = append(fields, self.Ports) @@ -181,7 +208,7 @@ func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred return self.SResourceBase.ValidateUpdateData(ctx, userCred, query, data) } -func (self *SSecurityGroupRule) GetRule() string { +func (self *SSecurityGroupRule) String() string { var fields []string for _, field := range []string{"direction", "action", "cidr", "protocol", "ports"} { switch field { @@ -193,11 +220,15 @@ func (self *SSecurityGroupRule) GetRule() string { case "action": fields = append(fields, self.Action) case "cidr": - if len(self.CIDR) > 0 { + if len(self.CIDR) > 0 && self.CIDR != "0.0.0.0/0" { fields = append(fields, self.CIDR) } case "protocol": - fields = append(fields, self.Protocol) + protocol := self.Protocol + if protocol == "" { + protocol = secrules.PROTO_ANY + } + fields = append(fields, protocol) case "ports": if len(self.Ports) > 0 { fields = append(fields, self.Ports) @@ -207,12 +238,49 @@ func (self *SSecurityGroupRule) GetRule() string { return fields[0] + strings.Join(fields[1:], " ") } +func (self *SSecurityGroupRule) SingleRules() ([]secrules.SecurityRule, error) { + rules := make([]secrules.SecurityRule, 0) + ruleStr := self.String() + if rule, err := secrules.ParseSecurityRule(ruleStr); err != nil { + return nil, err + } else if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + _rule := secrules.SecurityRule{ + Priority: int(self.Priority), + Action: rule.Action, + IPNet: rule.IPNet, + Protocol: rule.Protocol, + Direction: rule.Direction, + PortStart: -1, + PortEnd: -1, + Ports: []int{port}, + Description: self.Description, + } + rules = append(rules, _rule) + } + } else { + _rule := secrules.SecurityRule{ + Priority: int(self.Priority), + Action: rule.Action, + IPNet: rule.IPNet, + Protocol: rule.Protocol, + Direction: rule.Direction, + PortStart: rule.PortStart, + PortEnd: rule.PortEnd, + Ports: []int{}, + Description: self.Description, + } + rules = append(rules, _rule) + } + return rules, nil +} + func (self *SSecurityGroupRule) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) log.Debugf("POST Create %s", data) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } @@ -220,7 +288,7 @@ func (self *SSecurityGroupRule) PreDelete(ctx context.Context, userCred mcclient self.SResourceBase.PreDelete(ctx, userCred) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } @@ -229,6 +297,111 @@ func (self *SSecurityGroupRule) PostUpdate(ctx context.Context, userCred mcclien log.Debugf("POST Update %s", data) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } + +func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSecurityGroup) ([]SSecurityGroupRule, error) { + rules := make([]SSecurityGroupRule, 0) + q := manager.Query().Equals("secgroup_id", secgroup.Id) + if err := db.FetchModelObjects(manager, q, &rules); err != nil { + return nil, err + } + return rules, nil +} + +func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules []secrules.SecurityRule) ([]SSecurityGroupRule, []SSecurityGroupRule, compare.SyncResult) { + syncResult := compare.SyncResult{} + + if dbRules, err := manager.getRulesBySecurityGroup(secgroup); err != nil { + return nil, nil, syncResult + } else { + + sort.Sort(SecurityGroupRuleSet(dbRules)) + sort.Sort(secrules.SecurityRuleSet(rules)) + + i, j := 0, 0 + for i < len(rules) || j < len(dbRules) { + if i < len(rules) && j < len(dbRules) { + dbStr := dbRules[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(dbStr, ruleStr) + if cmp == 0 { + if dbRules[j].Description != rules[i].Description { + if _, err := manager.TableSpec().Update(dbRules[j], func() error { + dbRules[j].Description = rules[i].Description + return nil + }); err != nil { + log.Errorf("Update SecurityGroupRule failed: %v", err) + } + } + i += 1 + j += 1 + } else if cmp > 0 { + if err := dbRules[j].Delete(ctx, userCred); err != nil { + syncResult.AddError(err) + } else { + syncResult.Delete() + } + j += 1 + } else { + if _, err := manager.newFromCloudSecurityGroup(rules[i], secgroup); err != nil { + syncResult.AddError(err) + } else { + syncResult.Add() + } + i += 1 + } + } else if i >= len(rules) { + if err := dbRules[j].Delete(ctx, userCred); err != nil { + syncResult.AddError(err) + } else { + syncResult.Delete() + } + j += 1 + } else if j >= len(dbRules) { + if _, err := manager.newFromCloudSecurityGroup(rules[i], secgroup); err != nil { + syncResult.AddError(err) + } else { + syncResult.Add() + } + i += 1 + } + } + } + return nil, nil, syncResult +} + +func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(rule secrules.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) { + protocol := rule.Protocol + if rule.Protocol == "any" { + protocol = "" + } + ports, _ports := "", make([]string, len(rule.Ports)) + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + _ports = append(_ports, fmt.Sprintf("%d", port)) + } + ports = strings.Join(_ports, ",") + } else if rule.PortStart != 0 || rule.PortEnd != 0 { + if rule.PortStart == rule.PortEnd { + ports = fmt.Sprintf("%d", rule.PortStart) + } else { + ports = fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd) + } + } + secrule := &SSecurityGroupRule{ + Priority: int64(rule.Priority), + Protocol: protocol, + Ports: ports, + Direction: string(rule.Direction), + CIDR: rule.IPNet.String(), + Action: string(rule.Action), + Description: rule.Description, + SecgroupID: secgroup.Id, + } + if err := manager.TableSpec().Insert(secrule); err != nil { + return nil, err + } + return secrule, nil +} diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index 1fc2ca24bd..e2418d3ed1 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -3,12 +3,17 @@ package models import ( "context" "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/sqlchemy" ) @@ -94,16 +99,28 @@ func (self *SSecurityGroup) getSecurityRules() (rules []SSecurityGroupRule) { sql := secgrouprules.Query().Filter(sqlchemy.Equals(secgrouprules.Field("secgroup_id"), self.Id)) if err := db.FetchModelObjects(SecurityGroupRuleManager, sql, &rules); err != nil { log.Errorf("GetGuests fail %s", err) - return nil + return } return } +func (self *SSecurityGroup) getSecRules() []secrules.SecurityRule { + rules := make([]secrules.SecurityRule, 0) + for _, _rule := range self.getSecurityRules() { + singleRules, err := _rule.SingleRules() + if err != nil { + log.Errorf(err.Error()) + } + rules = append(rules, singleRules...) + } + return rules +} + func (self *SSecurityGroup) getSecurityRuleString() string { secgrouprules := self.getSecurityRules() var rules []string for _, rule := range secgrouprules { - rules = append(rules, rule.GetRule()) + rules = append(rules, rule.String()) } return strings.Join(rules, SECURITY_GROUP_SEPARATOR) } @@ -158,11 +175,125 @@ func (self *SSecurityGroup) PerformClone(ctx context.Context, userCred mcclient. return nil, nil } -func (self *SSecurityGroup) DoSync() { +func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, error) { + secgroups := make([]SSecurityGroup, 0) + q := manager.Query() + if err := db.FetchModelObjects(manager, q, &secgroups); err != nil { + return nil, err + } else { + return secgroups, nil + } +} + +func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { + localSecgroups := make([]SSecurityGroup, 0) + remoteSecgroups := make([]cloudprovider.ICloudSecurityGroup, 0) + syncResult := compare.SyncResult{} + + if dbSecgroups, err := manager.getSecurityGroups(); err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } else { + removed := make([]SSecurityGroup, 0) + commondb := make([]SSecurityGroup, 0) + commonext := make([]cloudprovider.ICloudSecurityGroup, 0) + added := make([]cloudprovider.ICloudSecurityGroup, 0) + if err := compare.CompareSets(dbSecgroups, secgroups, &removed, &commondb, &commonext, &added); err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } + + for i := 0; i < len(commondb); i += 1 { + if rules, err := commonext[i].GetRules(); err != nil { + syncResult.Error(err) + } else if len(rules) > 0 { + if err = commondb[i].SyncWithCloudSecurityGroup(userCred, commonext[i]); err != nil { + syncResult.UpdateError(err) + } else { + localSecgroups = append(localSecgroups, commondb[i]) + remoteSecgroups = append(remoteSecgroups, commonext[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, &commondb[i], rules) + syncResult.Update() + } + } + } + + for i := 0; i < len(added); i += 1 { + if rules, err := added[i].GetRules(); err != nil { + syncResult.AddError(err) + } else if len(rules) > 0 { + if new, err := manager.newFromCloudVpc(added[i]); err != nil { + syncResult.AddError(err) + } else { + localSecgroups = append(localSecgroups, *new) + remoteSecgroups = append(remoteSecgroups, added[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules) + syncResult.Add() + } + } + } + } + return localSecgroups, remoteSecgroups, syncResult +} + +func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup) error { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extSec.Refresh() + self.Name = extSec.GetName() + self.Description = extSec.GetDescription() + return nil + }); err != nil { + log.Errorf("syncWithCloudSecurityGroup error %s", err) + return err + } + return nil +} + +func (manager *SSecurityGroupManager) newFromCloudVpc(extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) { + secgroup := SSecurityGroup{} + secgroup.SetModelManager(manager) + secgroup.Name = extSec.GetName() + secgroup.ExternalId = extSec.GetGlobalId() + secgroup.Description = extSec.GetDescription() + + if err := manager.TableSpec().Insert(&secgroup); err != nil { + return nil, err + } + return &secgroup, nil +} + +func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mcclient.TokenCredential, idStr string) { + if secgrp := manager.FetchSecgroupById(idStr); secgrp == nil { + log.Errorf("DelaySync secgroup failed") + } else { + needSync := false + lockman.LockObject(ctx, secgrp) + defer lockman.ReleaseObject(ctx, secgrp) + if secgrp.IsDirty { + if _, err := secgrp.GetModelManager().TableSpec().Update(secgrp, func() error { + secgrp.IsDirty = false + return nil + }); err != nil { + log.Errorf("Update Security Group error: %s", err.Error()) + } + needSync = true + } + if needSync { + for _, guest := range secgrp.GetGuests() { + guest.StartSyncTask(ctx, userCred, true, "") + } + } + } +} + +func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenCredential) { if _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.IsDirty = true return nil }); err != nil { log.Errorf("Update Security Group error: %s", err.Error()) } + time.AfterFunc(10*time.Second, func() { + SecurityGroupManager.DelaySync(ctx, userCred, self.Id) + }) } diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index eabf5a2a82..6e4d1db026 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -5,14 +5,16 @@ import ( "fmt" "time" + "github.com/serialx/hashring" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" ) const ( @@ -71,6 +73,47 @@ func (joint *SStoragecachedimage) Slave() db.IStandaloneModel { return db.JointSlave(joint) } +func (self *SStoragecachedimage) getStorageHostId() (string, error) { + var s SStorage + storage := StorageManager.Query() + err := storage.Filter(sqlchemy.Equals(storage.Field("storagecache_id"), self.StoragecacheId)).First(&s) + if err != nil { + return "", err + } + + hosts := s.GetAllAttachingHosts() + var hostIds = make([]string, 0) + for _, host := range hosts { + hostIds = append(hostIds, host.Id) + } + if len(hostIds) == 0 { + return "", nil + } + + ring := hashring.New(hostIds) + ret, _ := ring.GetNode(self.StoragecacheId) + return ret, nil +} + +func (self *SStoragecachedimage) GetHost() (*SHost, error) { + hostId, err := self.getStorageHostId() + if err != nil { + return nil, err + } else if len(hostId) == 0 { + return nil, nil + } + + host, err := HostManager.FetchById(hostId) + if err != nil { + return nil, err + } else if host == nil { + return nil, nil + } + h, _ := host.(*SHost) + return h, nil + +} + func (self *SStoragecachedimage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := self.SJointResourceBase.GetCustomizeColumns(ctx, userCred, query) extra = db.JointModelExtra(self, extra) @@ -252,6 +295,15 @@ func (self *SStoragecachedimage) SetStatus(userCred mcclient.TokenCredential, st return nil } +func (self *SStoragecachedimage) AddDownloadRefcount() error { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.DownloadRefcnt += 1 + self.LastDownload = time.Now() + return nil + }) + return err +} + func (self *SStoragecachedimage) SetExternalId(externalId string) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.ExternalId = externalId diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 643c5efec8..e0d71d9712 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -4,14 +4,16 @@ import ( "context" "database/sql" + "github.com/serialx/hashring" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/sqlchemy" ) type SStoragecacheManager struct { @@ -55,6 +57,56 @@ func (self *SStoragecache) getStorageNames() []string { return names } +func (self *SStoragecache) GetHost() (*SHost, error) { + hostId, err := self.getHostId() + if err != nil { + return nil, err + } + if len(hostId) == 0 { + return nil, nil + } + + host, err := HostManager.FetchById(hostId) + if err != nil { + return nil, err + } else if host == nil { + return nil, nil + } + h, _ := host.(*SHost) + return h, nil +} + +func (self *SStoragecache) getHostId() (string, error) { + hoststorages := HoststorageManager.Query().SubQuery() + storages := StorageManager.Query().SubQuery() + + hosts := make([]SHost, 0) + host := HostManager.Query().SubQuery() + q := host.Query(host.Field("id")) + err := q.Join(hoststorages, sqlchemy.AND(sqlchemy.Equals(hoststorages.Field("host_id"), host.Field("id")), + sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE), + sqlchemy.IsTrue(host.Field("enabled")))). + Join(storages, sqlchemy.AND(sqlchemy.Equals(storages.Field("storagecache_id"), self.Id), + sqlchemy.Equals(storages.Field("status"), STORAGE_ONLINE), + sqlchemy.IsTrue(storages.Field("enabled")))). + Filter(sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id"))).All(&hosts) + if err != nil { + return "", err + } + + hostIds := make([]string, 0) + for _, h := range hosts { + hostIds = append(hostIds, h.Id) + } + + if len(hostIds) == 0 { + return "", nil + } + ring := hashring.New(hostIds) + ret, _ := ring.GetNode(self.Id) + return ret, nil +} + func (manager *SStoragecacheManager) SyncWithCloudStoragecache(cloudCache cloudprovider.ICloudStoragecache) (*SStoragecache, error) { localCacheObj, err := manager.FetchByExternalId(cloudCache.GetGlobalId()) if err != nil { diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 28650d29f1..4ac8ea7587 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -16,19 +16,22 @@ import ( ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + STORAGE_CLOUD_SSD = "cloud_ssd" + STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" STORAGE_ENABLED = "enabled" STORAGE_DISABLED = "disabled" STORAGE_OFFLINE = "offline" - STORAGE_ONLINE = "offline" + STORAGE_ONLINE = "online" DISK_TYPE_ROTATE = "rotate" DISK_TYPE_SSD = "ssd" @@ -235,9 +238,12 @@ func (self *SStorage) SyncStatusWithHosts() { } } -func (manager *SStorageManager) getStoragesByZoneId(zoneId string) ([]SStorage, error) { +func (manager *SStorageManager) getStoragesByZoneId(zoneId string, provider *SCloudprovider) ([]SStorage, error) { storages := make([]SStorage, 0) q := manager.Query().Equals("zone_id", zoneId) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &storages) if err != nil { log.Errorf("getStoragesByZoneId fail %s", err) @@ -261,7 +267,7 @@ func (manager *SStorageManager) scanLegacyStorages() error { return nil } -func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, zone *SZone, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { +func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { localStorages := make([]SStorage, 0) remoteStorages := make([]cloudprovider.ICloudStorage, 0) syncResult := compare.SyncResult{} @@ -272,7 +278,7 @@ func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mccli return nil, nil, syncResult } - dbStorages, err := manager.getStoragesByZoneId(zone.Id) + dbStorages, err := manager.getStoragesByZoneId(zone.Id, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index bba2aed865..368a4a963b 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -146,9 +146,12 @@ func (self *SVpc) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCr return self.getMoreDetails(extra) } -func (manager *SVpcManager) getVpcsByRegion(region *SCloudregion) ([]SVpc, error) { +func (manager *SVpcManager) getVpcsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SVpc, error) { vpcs := make([]SVpc, 0) q := manager.Query().Equals("cloudregion_id", region.Id) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &vpcs) if err != nil { return nil, err @@ -167,12 +170,12 @@ func (self *SVpc) setDefault(def bool) error { return err } -func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpcs []cloudprovider.ICloudVpc) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { +func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, vpcs []cloudprovider.ICloudVpc) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { localVPCs := make([]SVpc, 0) remoteVPCs := make([]cloudprovider.ICloudVpc, 0) syncResult := compare.SyncResult{} - dbVPCs, err := manager.getVpcsByRegion(region) + dbVPCs, err := manager.getVpcsByRegion(region, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -237,6 +240,7 @@ func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.Toke func (self *SVpc) SyncWithCloudVpc(extVPC cloudprovider.ICloudVpc) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extVPC.Refresh() self.Name = extVPC.GetName() self.Status = extVPC.GetStatus() self.CidrBlock = extVPC.GetCidrBlock() diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index 227874f9d6..f8cac5dbae 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -4,9 +4,15 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + "yunion.io/x/sqlchemy" + + _ "github.com/go-sql-driver/mysql" + _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" + _ "yunion.io/x/onecloud/pkg/compute/hostdrivers" + _ "yunion.io/x/onecloud/pkg/compute/tasks" + _ "yunion.io/x/onecloud/pkg/util/aliyun/provider" + _ "yunion.io/x/onecloud/pkg/util/esxi/provider" "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" @@ -14,16 +20,15 @@ import ( "yunion.io/x/onecloud/pkg/compute" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - - _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" - _ "yunion.io/x/onecloud/pkg/compute/tasks" - _ "yunion.io/x/onecloud/pkg/util/aliyun/provider" - _ "yunion.io/x/onecloud/pkg/util/esxi/provider" ) func StartService() { cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "region.conf") + if options.Options.DebugSqlchemy { + sqlchemy.DEBUG_SQLCHEMY = true + } + if options.Options.PortV2 > 0 { log.Infof("Port V2 %d is specified, use v2 port", options.Options.PortV2) options.Options.Port = options.Options.PortV2 diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index 84af4378a6..997f0792f3 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -130,7 +130,7 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task * return } - localVpcs, remoteVpcs, result := models.VpcManager.SyncVPCs(ctx, task.UserCred, localRegion, vpcs) + localVpcs, remoteVpcs, result := models.VpcManager.SyncVPCs(ctx, task.UserCred, provider, localRegion, vpcs) msg := result.Result() log.Infof("SyncVPCs for region %s result: %s", localRegion.Name, msg) if result.IsError() { @@ -141,13 +141,31 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task * for j := 0; j < len(localVpcs); j += 1 { syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) + syncVpcSecGroup(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) + } +} + +func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc) { + if secgroups, err := remoteVpc.GetISecurityGroups(); err != nil { + msg := fmt.Sprintf("GetISecurityGroups for vpc %s failed %s", remoteVpc.GetId(), err) + log.Errorf(msg) + logSyncFailed(provider, task, msg) + return + } else { + _, _, result := models.SecurityGroupManager.SyncSecgroups(ctx, task.UserCred, secgroups) + msg := result.Result() + log.Infof("SyncSecurityGroup for VPC %s result: %s", localVpc.Name, msg) + if result.IsError() { + logSyncFailed(provider, task, msg) + return + } } } func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc) { wires, err := remoteVpc.GetIWires() if err != nil { - msg := fmt.Sprintf("GetIWires for vps %s failed %s", remoteVpc.GetId(), err) + msg := fmt.Sprintf("GetIWires for vpc %s failed %s", remoteVpc.GetId(), err) log.Errorf(msg) logSyncFailed(provider, task, msg) return @@ -191,7 +209,7 @@ func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } - localStorages, remoteStorages, result := models.StorageManager.SyncStorages(ctx, task.UserCred, localZone, storages) + localStorages, remoteStorages, result := models.StorageManager.SyncStorages(ctx, task.UserCred, provider, localZone, storages) msg := result.Result() log.Infof("SyncZones for region %s result: %s", localZone.Name, msg) if result.IsError() { @@ -249,7 +267,7 @@ func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *C logSyncFailed(provider, task, msg) return } - localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, localZone, hosts) + localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, localZone, hosts) msg := result.Result() log.Infof("SyncHosts for zone %s result: %s", localZone.Name, msg) if result.IsError() { diff --git a/pkg/compute/tasks/disk_base_task.go b/pkg/compute/tasks/disk_base_task.go index ddd4445b20..083f78d85d 100644 --- a/pkg/compute/tasks/disk_base_task.go +++ b/pkg/compute/tasks/disk_base_task.go @@ -3,6 +3,8 @@ package tasks import ( "context" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -31,3 +33,16 @@ func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) { } } } + +func (self *SDiskBaseTask) CleanHostSchedCache(disk *models.SDisk) { + storage := disk.GetStorage() + if hosts := storage.GetAllAttachingHosts(); hosts == nil { + log.Errorf("get attaching host error") + } else { + for _, h := range hosts { + if err := h.ClearSchedDescCache(); err != nil { + log.Errorf("host CleanHostSchedCache error: %v", err) + } + } + } +} diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go new file mode 100644 index 0000000000..1c47d60d5a --- /dev/null +++ b/pkg/compute/tasks/disk_create_task.go @@ -0,0 +1,76 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type DiskCreateTask struct { + SDiskBaseTask +} + +func init() { + taskman.RegisterTask(DiskCreateTask{}) +} + +func (self *DiskCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + disk := obj.(*models.SDisk) + + storagecache := disk.GetStorage().GetStoragecache() + imageId := disk.GetTemplateId() + if len(imageId) > 0 { + self.SetStage("on_storage_cache_image_complete", nil) + storagecache.StartImageCacheTask(ctx, self.UserCred, imageId, false, self.GetTaskId()) + } else { + self.OnStorageCacheImageComplete(ctx, disk, nil) + } +} + +func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + rebuild, _ := self.GetParams().Bool("rebuild") + snapshot, _ := self.GetParams().GetString("snapshot") + if rebuild { + db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.GetUserCred()) + } + storage := disk.GetStorage() + host := storage.GetMasterHost() + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred()) + disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") + self.SetStage("on_disk_ready", nil) + if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { + self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) + } +} + +func (self *DiskCreateTask) OnStartAllocateFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) +} + +func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + diskSize, _ := data.Int("disk_size") + if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.DiskSize = int(diskSize) + disk.DiskFormat, _ = data.GetString("disk_format") + disk.AccessPath, _ = data.GetString("disk_path") + return nil + }); err != nil { + log.Errorf("update disk info error: %v", err) + } + + disk.SetStatus(self.UserCred, models.DISK_READY, "") + self.CleanHostSchedCache(disk) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.UserCred) + self.SetStageComplete(ctx, nil) +} + +func (self *DiskCreateTask) OnDiskReadyFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) +} diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 2ac629ecf9..ff6c4f76c1 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" ) type DiskDeleteTask struct { @@ -50,7 +51,12 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD if isPurge { self.OnGuestDiskDeleteSucc(ctx, disk, nil) } else { - // TODO + self.SetStage("on_guest_disk_delete_succ", nil) + if host == nil { + self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host")) + } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { + self.OnGuestDiskDeleteFailed(ctx, disk, err) + } } } @@ -65,8 +71,14 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteSucc(ctx context.Context, obj db.IS return } disk := obj.(*models.SDisk) - // self.clean_host_sched_cache(disk) + self.CleanHostSchedCache(disk) db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.UserCred) disk.RealDelete(ctx, self.UserCred) self.SetStageComplete(ctx, nil) } + +func (self *DiskDeleteTask) OnGuestDiskDeleteFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.GetUserCred(), models.DISK_DEALLOC_FAILED, resion.Error()) + self.SetStageFailed(ctx, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_DELOCATE_FAIL, disk.GetShortDesc(), self.GetUserCred()) +} diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go new file mode 100644 index 0000000000..584c43ab94 --- /dev/null +++ b/pkg/compute/tasks/disk_resize_task.go @@ -0,0 +1,108 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type DiskResizeTask struct { + SDiskBaseTask +} + +func init() { + taskman.RegisterTask(DiskResizeTask{}) +} + +func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + disk := obj.(*models.SDisk) + storage := disk.GetStorage() + host := storage.GetMasterHost() + online := disk.GetRuningGuestCount() > 0 + if online { + for _, guest := range disk.GetGuests() { + host = guest.GetHost() + } + } + resion := "Cannot find host for disk" + if host == nil || host.HostStatus != models.HOST_ONLINE { + disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion) + self.SetStageFailed(ctx, resion) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred()) + } else { + disk.SetStatus(self.GetUserCred(), models.DISK_START_RESIZE, "") + for _, guest := range disk.GetGuests() { + guest.SetStatus(self.GetUserCred(), models.VM_RESIZE_DISK, "") + } + self.StartResizeDisk(ctx, host, storage, disk, online) + } +} + +func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, online bool) { + log.Infof("Resizing disk on host %s ...", host.GetName()) + self.SetStage("on_disk_resize_complete", nil) + size, _ := self.GetParams().Int("size") + proc := host.GetHostDriver().RequestResizeDiskOnHost + if online { + proc = host.GetHostDriver().RequestResizeDiskOnHostOnline + } + if err := proc(host, storage, disk, size, self); err != nil { + log.Errorf("request_resize_disk_on_host: %v", err) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + self.OnStartResizeDiskSucc(ctx, disk) +} + +func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *models.SDisk) { + disk.SetStatus(self.GetUserCred(), models.DISK_RESIZING, "") +} + +func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error()) + self.SetStageFailed(ctx, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred()) +} + +func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + jSize, err := data.Get("disk_size") + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + size, err := jSize.Int() + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + oldStatus := disk.Status + _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.Status = models.DISK_READY + disk.DiskSize = int(size) + return nil + }) + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status) + db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred) + self.CleanHostSchedCache(disk) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE, disk.GetShortDesc(), self.UserCred) + self.SetStageComplete(ctx, disk.GetShortDesc()) + self.finalReleasePendingUsage(ctx) +} + +func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) +} diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go new file mode 100644 index 0000000000..44ba532ae1 --- /dev/null +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -0,0 +1,208 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestChangeConfigTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestChangeConfigTask{}) +} + +func (self *GuestChangeConfigTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + _, err := self.Params.Get("resize") + if err == nil { + self.SetStage("on_disks_resize_complete", nil) + self.OnDisksResizeComplete(ctx, obj, data) + } else { + guest := obj.(*models.SGuest) + self.DoCreateDisksTask(ctx, guest) + } +} + +func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + iResizeDisks, err := self.Params.Get("resize") + if iResizeDisks == nil || err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + resizeDisks := iResizeDisks.(*jsonutils.JSONArray) + for i := 0; i < resizeDisks.Length(); i++ { + iResizeSet, err := resizeDisks.GetAt(i) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + resizeSet := iResizeSet.(*jsonutils.JSONArray) + diskId, err := resizeSet.GetAt(0) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + idStr, err := diskId.GetString() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + jSize, err := resizeSet.GetAt(1) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + size, err := jSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + iDisk, err := models.DiskManager.FetchById(idStr) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + disk := iDisk.(*models.SDisk) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if disk.DiskSize < int(size) { + var pendingUsage models.SQuota + err = self.GetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage) + return + } + } + guest := obj.(*models.SGuest) + self.DoCreateDisksTask(ctx, guest) +} + +func (self *GuestChangeConfigTask) DoCreateDisksTask(ctx context.Context, guest *models.SGuest) { + iCreateData, err := self.Params.Get("create") + if err != nil || iCreateData == nil { + self.OnCreateDisksComplete(ctx, guest, nil) + return + } + data := (iCreateData).(*jsonutils.JSONDict) + self.SetStage("on_create_disks_complete", nil) + guest.StartGuestCreateDiskTask(ctx, self.UserCred, data, self.GetTaskId()) + +} + +func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + iVcpuCount, errCpu := self.Params.Get("vcpu_count") + iVmemSize, errMem := self.Params.Get("vmem_size") + var vcpuCount, vmemSize int64 + var err error + guest := obj.(*models.SGuest) + if errCpu == nil || errMem == nil { + if iVcpuCount != nil { + vcpuCount, err = iVcpuCount.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + if iVmemSize != nil { + vmemSize, err = iVmemSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + err = guest.GetDriver().RequestChangeVmConfig(ctx, guest, self, vcpuCount, vmemSize) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + var addCpu, addMem = 0, 0 + if vcpuCount > 0 { + addCpu = int(vcpuCount - int64(guest.VcpuCount)) + if addCpu < 0 { + addCpu = 0 + } + } + if vmemSize > 0 { + addMem = int(vmemSize - int64(guest.VmemSize)) + if addMem < 0 { + addMem = 0 + } + } + _, err = guest.GetModelManager().TableSpec().Update(guest, func() error { + if vcpuCount > 0 { + guest.VcpuCount = int8(vcpuCount) + } + if vmemSize > 0 { + guest.VmemSize = int(vmemSize) + } + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + var pendingUsage models.SQuota + err = self.GetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + // ownerCred := guest.GetOwnerUserCred() + var cancelUsage models.SQuota + if addCpu > 0 { + cancelUsage.Cpu = addCpu + } + if addMem > 0 { + cancelUsage.Memory = addMem + } + lockman.LockClass(ctx, guest.GetModelManager(), guest.ProjectId) + defer lockman.ReleaseClass(ctx, guest.GetModelManager(), guest.ProjectId) + err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, guest.ProjectId, &pendingUsage, &cancelUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + err = self.SetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + self.SetStage("on_sync_status_complete", nil) + err = guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } +} + +func (self *GuestChangeConfigTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) { + self.SetStage("on_guest_start_complete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + dt := jsonutils.NewDict() + dt.Add(jsonutils.NewString(guest.Id), "id") + self.SetStageComplete(ctx, dt) + } +} + +func (self *GuestChangeConfigTask) OnGuestStartComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + dt := jsonutils.NewDict() + dt.Add(jsonutils.NewString(guest.Id), "id") + self.SetStageComplete(ctx, dt) +} diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go new file mode 100644 index 0000000000..c7034bd735 --- /dev/null +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -0,0 +1,124 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *GuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_disk_prepared", nil) + guest := obj.(*models.SGuest) + err := guest.GetDriver().DoGuestCreateDisksTask(ctx, guest, self) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } +} + +func (self *GuestCreateDiskTask) OnDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *GuestCreateDiskTask) OnDiskPreparedFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data.String()) +} + +/* --------------------------------------------- */ +/* -----------KVMGuestCreateDiskTask------------ */ +/* --------------------------------------------- */ + +type KVMGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_kvm_disk_prepared", nil) + self.OnKvmDiskPrepared(ctx, obj, data) +} + +func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + var diskIndex = 0 + var diskReady = true + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status == models.DISK_INIT { + snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + if err != nil { + snapInfo = "" + } + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + diskReady = false + break + } + diskIndex += 1 + } + diskIndex = 0 + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status != models.DISK_READY { + diskReady = false + break + } + diskIndex += 1 + } + if diskReady { + guest := obj.(*models.SGuest) + if guest.Status == models.VM_RUNNING { + self.SetStage("on_config_sync_complete", nil) + err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + } else { + self.SetStageComplete(ctx, nil) + } + } +} + +func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func init() { + taskman.RegisterTask(GuestCreateDiskTask{}) + taskman.RegisterTask(KVMGuestCreateDiskTask{}) +} diff --git a/pkg/compute/tasks/guest_delete_task.go b/pkg/compute/tasks/guest_delete_task.go index 09666d5e52..b86a63d2ec 100644 --- a/pkg/compute/tasks/guest_delete_task.go +++ b/pkg/compute/tasks/guest_delete_task.go @@ -28,15 +28,14 @@ func (self *GuestDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) + guestStatus, _ := self.Params.GetString("guest_status") if options.Options.EnablePendingDelete && !guest.PendingDeleted && !jsonutils.QueryBoolean(self.Params, "purge", false) && - !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { - guestStatus, _ := self.Params.GetString("guest_status") - if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, + !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) && + !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) { - self.StartPendingDeleteGuest(ctx, guest) - return - } + self.StartPendingDeleteGuest(ctx, guest) + return } self.OnGuestStopCompleteFailed(ctx, guest, data) } @@ -64,6 +63,7 @@ func (self *GuestDeleteTask) OnPendingDeleteComplete(ctx context.Context, obj db } func (self *GuestDeleteTask) StartDeleteGuest(ctx context.Context, guest *models.SGuest) { + // No snapshot self.SetStage("on_guest_detach_disks_complete", nil) guest.GetDriver().RequestDetachDisksFromGuestForDelete(ctx, guest, self) } @@ -100,7 +100,6 @@ func (self *GuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.I } func (self *GuestDeleteTask) DeleteGuest(ctx context.Context, guest *models.SGuest) { - // host := guest.GetHost() guest.RealDelete(ctx, self.UserCred) guest.RemoveAllMetadata(ctx, self.UserCred) db.OpsLog.LogEvent(guest, db.ACT_DELOCATE, nil, self.UserCred) diff --git a/pkg/compute/tasks/guest_detach_all_disks_task.go b/pkg/compute/tasks/guest_detach_all_disks_task.go index ed2bc2529a..6be7da967f 100644 --- a/pkg/compute/tasks/guest_detach_all_disks_task.go +++ b/pkg/compute/tasks/guest_detach_all_disks_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/tasks/guest_detach_disk_task.go b/pkg/compute/tasks/guest_detach_disk_task.go index b8af7ce35c..0a6653750f 100644 --- a/pkg/compute/tasks/guest_detach_disk_task.go +++ b/pkg/compute/tasks/guest_detach_disk_task.go @@ -2,10 +2,15 @@ package tasks import ( "context" + "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" ) type GuestDetachDiskTask struct { @@ -17,5 +22,83 @@ func init() { } func (self *GuestDetachDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + diskId, _ := self.Params.GetString("disk_id") + objDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + disk := objDisk.(*models.SDisk) + if disk == nil { + self.OnTaskFail(ctx, guest, fmt.Errorf("Connot find disk %s", diskId)) + return + } + guest.DetachDisk(ctx, disk, self.UserCred) + if disk.Status == models.DISK_INIT { + self.OnSyncConfigComplete(ctx, guest, nil) + return + } + host := guest.GetHost() + purge := false + if host != nil && host.Status == models.HOST_DISABLED && jsonutils.QueryBoolean(self.Params, "purge", false) { + purge = true + } + detachStatus, err := guest.GetDriver().GetDetachDiskStatus() + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + if utils.IsInStringArray(guest.Status, detachStatus) && !purge { + self.SetStage("on_sync_config_complete", nil) + guest.GetDriver().RequestDetachDisk(ctx, guest, self) + disk.SetStatus(self.UserCred, models.DISK_READY, "Disk detach") + } else { + self.OnSyncConfigComplete(ctx, guest, nil) + } +} + +func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + diskId, _ := self.Params.GetString("disk_id") + objDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + disk := objDisk.(*models.SDisk) + if disk == nil { + self.OnTaskFail(ctx, guest, fmt.Errorf("Connot find disk %s", diskId)) + return + } + keepDisk := jsonutils.QueryBoolean(self.Params, "keep_disk", true) + host := guest.GetHost() + purge := false + if host != nil && host.Status == models.HOST_DISABLED && jsonutils.QueryBoolean(self.Params, "purge", false) { + purge = true + } + if disk.Status == models.DISK_INIT { + db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) + disk.RealDelete(ctx, self.UserCred) + self.SetStageComplete(ctx, nil) + } else if (disk.Status == models.DISK_READY || !keepDisk) && disk.GetGuestDiskCount() == 0 && disk.AutoDelete { + self.SetStage("on_disk_delete_complete", nil) + db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) + err := guest.GetDriver().RequestDeleteDetachedDisk(ctx, disk, self, purge) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestDetachDiskTask) OnTaskFail(ctx context.Context, guest *models.SGuest, err error) { + self.SetStageFailed(ctx, err.Error()) + log.Errorf("Guest %s GuestDetachDiskTask failed %s", guest.Id, err.Error()) +} + +func (self *GuestDetachDiskTask) OnDiskDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_insert_iso_task.go b/pkg/compute/tasks/guest_insert_iso_task.go index 21bb54b31d..e0b9816f52 100644 --- a/pkg/compute/tasks/guest_insert_iso_task.go +++ b/pkg/compute/tasks/guest_insert_iso_task.go @@ -4,8 +4,10 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" ) type GuestInsertIsoTask struct { @@ -17,5 +19,59 @@ func init() { } func (self *GuestInsertIsoTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - // TODO + self.prepareIsoImage(ctx, obj) +} + +func (self *GuestInsertIsoTask) prepareIsoImage(ctx context.Context, obj db.IStandaloneModel) { + imageId, _ := self.Params.GetString("image_id") + db.OpsLog.LogEvent(obj, db.ACT_ISO_PREPARING, imageId, self.UserCred) + var host *models.SHost + if self.Params.Contains("host_id") { + hostId, _ := self.Params.GetString("host_id") + iHost, _ := models.HostManager.FetchById(hostId) + host = iHost.(*models.SHost) + } else { + guest := obj.(*models.SGuest) + host = guest.GetHost() + } + self.SetStage("OnIsoPrepareComplete", nil) + host.StartImageCacheTask(ctx, self.UserCred, imageId, self.GetTaskId(), false) +} + +func (self *GuestInsertIsoTask) OnIsoPrepareCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + imageId, _ := self.Params.GetString("image_id") + db.OpsLog.LogEvent(obj, db.ACT_ISO_PREPARE_FAIL, imageId, self.UserCred) + guest := obj.(*models.SGuest) + guest.EjectIso(self.UserCred) + self.SetStageFailed(ctx, "OnIsoPrepareCompleteFailed") +} + +func (self *GuestInsertIsoTask) OnIsoPrepareComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + imageId, _ := data.GetString("image_id") + jSize, err := data.Get("size") + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + size, err := jSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + name, _ := data.GetString("name") + path, _ := data.GetString("path") + guest := obj.(*models.SGuest) + if guest.InsertIsoSucc(imageId, path, int(size), name) { + db.OpsLog.LogEvent(guest, db.ACT_ISO_ATTACH, guest.GetDetailsIso(self.UserCred), self.UserCred) + if guest.Status == models.VM_RUNNING { + self.SetStage("OnConfigSyncComplete", nil) + guest.GetDriver().RequestGuestHotAddIso(ctx, guest, path, self) + } else { + self.SetStageComplete(ctx, nil) + } + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestInsertIsoTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go new file mode 100644 index 0000000000..03345b85ac --- /dev/null +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -0,0 +1,144 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/osprofile" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestRebuildRootTask{}) + taskman.RegisterTask(KVMGuestRebuildRootTask{}) +} + +type GuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if jsonutils.QueryBoolean(self.Params, "need_stop", false) { + self.SetStage("OnStopServerComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) + } else { + self.StartRebuildRootDisk(ctx, guest) + } +} + +func (self *GuestRebuildRootTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartRebuildRootDisk(ctx, guest) +} + +func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred) + gds := guest.CategorizeDisks() + imageId, _ := self.Params.GetString("image_id") + oldStatus := gds.Root.Status + _, err := gds.Root.GetModelManager().TableSpec().Update(gds.Root, func() error { + gds.Root.TemplateId = imageId + gds.Root.Status = models.DISK_REBUILD + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } else { + db.OpsLog.LogEvent(gds.Root, db.ACT_UPDATE_STATUS, + fmt.Sprintf("%s=>%s", oldStatus, models.DISK_REBUILD), self.UserCred) + } + + self.SetStage("OnRebuildRootDiskComplete", nil) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT, "") + guest.GetDriver().RequestRebuildRootDisk(ctx, guest, self) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + imgId, _ := self.Params.GetString("image_id") + imginfo, err := models.CachedimageManager.GetImageById(ctx, self.UserCred, imgId, false) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + osprof, err := osprofile.GetOSProfileFromImageProperties(imginfo.Properties, guest.Hypervisor) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + err = guest.SetMetadata(ctx, "__os_profile__", osprof, self.UserCred) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if guest.OsType != osprof.OSType { + _, err := guest.GetModelManager().TableSpec().Update(guest, func() error { + guest.OsType = osprof.OSType + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT, "", self.UserCred) + // TODO: logclient + guest.NotifyServerEvent(notifyclient.SERVER_REBUILD_ROOT, notifyclient.PRIORITY_IMPORTANT, true) + self.SetStage("OnSyncStatusComplete", nil) + guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT_FAIL, data.String(), self.UserCred) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT_FAIL, "") + // TODO: logclient +} + +func (self *GuestRebuildRootTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) { + self.SetStage("OnGuestStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestRebuildRootTask) OnGuestStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +/* -------------------------------------------------- */ +/* ------------ KVMGuestRebuildRootTask ------------- */ +/* -------------------------------------------------- */ + +type KVMGuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *KVMGuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + gds := guest.CategorizeDisks() + self.SetStage("OnRebuildRootDiskComplete", nil) + gds.Root.StartDiskCreateTask(ctx, self.UserCred, true, "", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStage("OnGuestDeployComplete", nil) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") + params := jsonutils.NewDict() + params.Set("reset_password", jsonutils.JSONTrue) + guest.StartGuestDeployTask(ctx, self.UserCred, params, "deploy", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data.String()) +} + +func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_reset_task.go b/pkg/compute/tasks/guest_reset_task.go new file mode 100644 index 0000000000..72a5273e2d --- /dev/null +++ b/pkg/compute/tasks/guest_reset_task.go @@ -0,0 +1,58 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestSoftResetTask{}) + taskman.RegisterTask(GuestHardResetTask{}) +} + +type GuestSoftResetTask struct { + SGuestBaseTask +} + +func (self *GuestSoftResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + err := guest.GetDriver().RequestSoftReset(ctx, guest, self) + if err == nil { + self.SetStageComplete(ctx, nil) + } else { + self.SetStageFailed(ctx, err.Error()) + } +} + +type GuestHardResetTask struct { + SGuestBaseTask +} + +func (self *GuestHardResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.StopServer(ctx, guest) +} + +func (self *GuestHardResetTask) StopServer(ctx context.Context, guest *models.SGuest) { + guest.SetStatus(self.UserCred, models.VM_STOPPING, "") + self.SetStage("OnServerStopComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartServer(ctx, guest) +} + +func (self *GuestHardResetTask) StartServer(ctx context.Context, guest *models.SGuest) { + self.SetStage("OnServerStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 3d910543a6..7bbb2ca2dc 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -42,7 +43,7 @@ func (self *GuestStartTask) startStart(ctx context.Context, guest *models.SGuest self.SetStage("on_start_complete", nil) host := guest.GetHost() guest.SetStatus(self.UserCred, models.VM_STARTING, "") - result, err := guest.GetDriver().RequestStartOnHost(guest, host, self.UserCred, self) + result, err := guest.GetDriver().RequestStartOnHost(ctx, guest, host, self.UserCred, self) if err != nil { self.onStartGuestFailed(ctx, guest, err) } else { diff --git a/pkg/compute/tasks/guest_stop_task.go b/pkg/compute/tasks/guest_stop_task.go index ddffa32032..2ed0f4aa6c 100644 --- a/pkg/compute/tasks/guest_stop_task.go +++ b/pkg/compute/tasks/guest_stop_task.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/tasks/guest_suspend_task.go b/pkg/compute/tasks/guest_suspend_task.go new file mode 100644 index 0000000000..13023f12f3 --- /dev/null +++ b/pkg/compute/tasks/guest_suspend_task.go @@ -0,0 +1,47 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestSuspendTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestSuspendTask{}) +} + +func (self *GuestSuspendTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_STOPPING, "", self.UserCred) + guest.SetStatus(self.UserCred, models.VM_SUSPENDING, "GuestSusPendTask") + self.SetStage("on_suspend_complete", nil) + err := guest.GetDriver().RqeuestSuspendOnHost(ctx, guest, self) + if err != nil { + self.OnSuspendGuestFail(guest, err.Error()) + } +} + +func (self *GuestSuspendTask) OnSuspendComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.UserCred, models.VM_SUSPEND, "") + db.OpsLog.LogEvent(guest, db.ACT_STOP, "", self.UserCred) + self.SetStageComplete(ctx, nil) +} + +func (self *GuestSuspendTask) OnSuspendCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.UserCred, models.VM_RUNNING, "") + db.OpsLog.LogEvent(guest, db.ACT_STOP_FAIL, err.String(), self.UserCred) +} + +func (self *GuestSuspendTask) OnSuspendGuestFail(guest *models.SGuest, reason string) { + guest.SetStatus(self.UserCred, models.VM_SUSPEND_FAILED, reason) +} diff --git a/pkg/compute/tasks/guest_sync_task.go b/pkg/compute/tasks/guest_sync_task.go new file mode 100644 index 0000000000..1daeb9e058 --- /dev/null +++ b/pkg/compute/tasks/guest_sync_task.go @@ -0,0 +1,69 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestSyncConfTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestSyncConfTask{}) +} + +func (self *GuestSyncConfTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred) + if host := guest.GetHost(); host == nil { + self.SetStageFailed(ctx, "No host for sync") + return + } else { + self.SetStage("on_sync_complete", nil) + if err := guest.GetDriver().RequestSyncConfigOnHost(ctx, guest, host, self); err != nil { + self.SetStageFailed(ctx, err.Error()) + log.Errorf("SyncConfTask faled %v", err) + } + } +} + +func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if fw_only, _ := self.GetParams().Bool("fw_only"); fw_only { + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred) + self.SetStageComplete(ctx, guest.GetShortDesc()) + } else if data.Contains("task") { + self.SetStage("on_disk_sync_complete", nil) + } else { + self.OnDiskSyncComplete(ctx, guest, data) + } +} + +func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStage("on_sync_status_complete", nil) + guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId()) +} + +func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) + log.Errorf("Guest sync config failed: %v", data.String()) +} + +func (self *GuestSyncConfTask) OnSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.GetUserCred(), models.VM_SYNC_FAIL, data.String()) + log.Errorf("Guest sync config failed: %v", data.String()) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) +} + +func (self *GuestSyncConfTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_syncstatus_task.go b/pkg/compute/tasks/guest_syncstatus_task.go index 6c4b5f5af4..478f4afe6d 100644 --- a/pkg/compute/tasks/guest_syncstatus_task.go +++ b/pkg/compute/tasks/guest_syncstatus_task.go @@ -49,7 +49,9 @@ func (self *GuestSyncstatusTask) OnGetStatusSucc(ctx context.Context, guest *mod default: statusStr = models.VM_UNKNOWN } - guest.SetStatus(self.UserCred, statusStr, "syncstatus") + statusData := jsonutils.NewDict() + statusData.Add(jsonutils.NewString(statusStr), "status") + guest.PerformStatus(ctx, self.UserCred, nil, statusData) self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_undeploy_task.go b/pkg/compute/tasks/guest_undeploy_task.go index 1cb128ec53..eb3f789e8a 100644 --- a/pkg/compute/tasks/guest_undeploy_task.go +++ b/pkg/compute/tasks/guest_undeploy_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -33,8 +34,6 @@ func (self *GuestUndeployTask) OnInit(ctx context.Context, obj db.IStandaloneMod err := guest.GetDriver().RequestUndeployGuestOnHost(ctx, guest, host, self) if err != nil { self.OnStartDeleteGuestFail(ctx, err) - } else { - // do nothing } } else { self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/storage_cache_image_task.go b/pkg/compute/tasks/storage_cache_image_task.go index ffd44c6c62..fc24e13ea0 100644 --- a/pkg/compute/tasks/storage_cache_image_task.go +++ b/pkg/compute/tasks/storage_cache_image_task.go @@ -5,6 +5,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -20,7 +21,7 @@ func init() { func (self *StorageCacheImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { imageId, _ := self.Params.GetString("image_id") - isForce := jsonutils.QueryBoolean(self.Params, "is_force", false) + // isForce := jsonutils.QueryBoolean(self.Params, "is_force", false) storageCache := obj.(*models.SStoragecache) scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) @@ -32,22 +33,13 @@ func (self *StorageCacheImageTask) OnInit(ctx context.Context, obj db.IStandalon self.SetStage("on_image_cache_complete", nil) - taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) { - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } + host, _ := storageCache.GetHost() + err := host.GetHostDriver().CheckAndSetCacheImage(ctx, host, storageCache, scimg, self) + if err != nil { + errData := taskman.Error2TaskData(err) + self.OnImageCacheCompleteFailed(ctx, storageCache, errData) + } - extImgId, err := iStorageCache.UploadImage(self.UserCred, imageId, scimg.ExternalId, isForce) - - if err != nil { - return nil, err - } else { - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - } - }) } func (self *StorageCacheImageTask) OnImageCacheComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -80,7 +72,7 @@ func (self *StorageCacheImageTask) OnCacheFailed(ctx context.Context, cache *mod func (self *StorageCacheImageTask) OnCacheSucc(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage, extImgId string) { scimg.SetStatus(self.UserCred, models.CACHED_IMAGE_STATUS_READY, "cached") - if len(extImgId) > 0 && scimg.ExternalId != extImgId { + if len(cache.ExternalId) > 0 && len(extImgId) > 0 && scimg.ExternalId != extImgId { scimg.SetExternalId(extImgId) } models.CachedimageManager.ImageAddRefCount(imageId) diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index 77d0e62d67..d57bf8ff79 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -39,6 +39,13 @@ func NewResourceNotReadyError(msg string, params ...interface{}) *httputils.JSON return NewJsonClientError(500, "ResourceNotReadyError", msg) } +func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError { + if len(params) > 0 { + msg = fmt.Sprintf(msg, params...) + } + return NewJsonClientError(500, "NewOutOfResourceError", msg) +} + func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError { if len(params) > 0 { msg = fmt.Sprintf(msg, params...) diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 92bf5b4bb9..4df0561160 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -53,6 +53,10 @@ func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client { return &client } +func (this *Client) SetDebug(debug bool) { + this.debug = debug +} + func (this *Client) AuthVersion() string { pos := strings.LastIndexByte(this.authUrl, '/') if pos > 0 { @@ -62,6 +66,13 @@ func (this *Client) AuthVersion() string { } } +func (this *Client) NewAuthTokenCredential() TokenCredential { + if this.AuthVersion() == "v3" { + return &TokenCredentialV3{} + } + return &TokenCredentialV2{} +} + func getDefaultHeader(header http.Header, token string) http.Header { if len(token) > 0 { if header == nil { diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index d458d65038..3526f6eea4 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -63,6 +63,10 @@ func SplitVersionedURL(url string) (string, string) { return base }*/ +func (this *ClientSession) GetClient() *Client { + return this.client +} + func (this *ClientSession) GetServiceURL(service, endpointType string) (string, error) { if len(this.endpointType) > 0 { // session specific endpoint type should override the input endpointType, which is supplied by manager diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 4d3a039793..aeeca8096c 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -96,6 +96,14 @@ func (self *SDisk) GetId() string { return self.DiskId } +func (self *SDisk) Delete() error { + return self.storage.zone.region.deleteDisk(self.DiskId) +} + +func (self *SDisk) Resize(size int64) error { + return self.storage.zone.region.resizeDisk(self.DiskId, size) +} + func (self *SDisk) GetName() string { return self.DiskId } @@ -214,3 +222,12 @@ func (self *SRegion) deleteDisk(diskId string) error { _, err := self.ecsRequest("DeleteDisk", params) return err } + +func (self *SRegion) resizeDisk(diskId string, size int64) error { + params := make(map[string]string) + params["DiskId"] = diskId + params["NewSize"] = fmt.Sprintf("%d", size) + + _, err := self.ecsRequest("ResizeDisk", params) + return err +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index c5b61dde3b..3ee4a40236 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -222,7 +222,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int return "", fmt.Errorf("vsiwtch's wire's vpc is empty") } - secgroups, err := net.wire.vpc.GetSecurityGroups() + secgroups, err := net.wire.vpc.GetISecurityGroups() if err != nil { return "", fmt.Errorf("get security group error %s", err) } @@ -236,7 +236,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int secgroupId = secId } } else { - secgroupId = secgroups[0].SecurityGroupId + secgroupId = secgroups[0].GetId() } keypair := "" diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 292a231193..80a2e9c267 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/seclib" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -185,6 +186,10 @@ func (self *SInstance) IsEmulated() bool { return false } +func (self *SInstance) getVpc() (*SVpc, error) { + return self.host.zone.region.getVpc(self.VpcAttributes.VpcId) +} + func (self *SInstance) fetchDisks() error { disks, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, 0, 50) if err != nil { @@ -508,3 +513,29 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") return ret, nil } + +func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { + if vpc, err := self.getVpc(); err != nil { + return err + } else if len(secgroupId) == 0 { + for index, secgrpId := range self.SecurityGroupIds.SecurityGroupId { + if err := vpc.revokeSecurityGroup(secgrpId, self.InstanceId, index == 0); err != nil { + return err + } + } + } else if secgrpId, err := vpc.syncSecurityGroup(secgroupId, name, rules); err != nil { + return err + } else if err := vpc.assignSecurityGroup(secgrpId, self.InstanceId); err != nil { + return err + } else { + for _, secgroupId := range self.SecurityGroupIds.SecurityGroupId { + if secgroupId != secgrpId { + if err := vpc.revokeSecurityGroup(secgroupId, self.InstanceId, false); err != nil { + return err + } + } + } + self.SecurityGroupIds.SecurityGroupId = []string{secgrpId} + } + return nil +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index cdea189749..78efb6a653 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -292,6 +292,7 @@ func (self *SRegion) getVpc(vpcId string) (*SVpc, error) { if total != 1 { return nil, cloudprovider.ErrNotFound } + vpcs[0].region = self return &vpcs[0], nil } diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 85d9518b32..69515ae79d 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -2,9 +2,13 @@ package aliyun import ( "fmt" + "sort" + "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" ) @@ -43,6 +47,7 @@ type SPermissions struct { } type SSecurityGroup struct { + vpc *SVpc CreationTime time.Time Description string SecurityGroupId string @@ -53,6 +58,82 @@ type SSecurityGroup struct { RegionId string } +type PermissionSet []SPermission + +func (v PermissionSet) Len() int { + return len(v) +} + +func (v PermissionSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v PermissionSet) Less(i, j int) bool { + if v[i].Priority < v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + +func (self *SSecurityGroup) GetId() string { + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetGlobalId() string { + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetDescription() string { + return self.Description +} + +func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { + rules := make([]secrules.SecurityRule, 0) + if secgrp, err := self.vpc.region.GetSecurityGroupDetails(self.SecurityGroupId); err != nil { + return rules, err + } else { + for _, permission := range secgrp.Permissions.Permission { + if rule, err := secrules.ParseSecurityRule(permission.String()); err != nil { + return rules, err + } else { + priority := permission.Priority + if priority > 100 { + priority = 100 + } + rule.Priority = 101 - priority + rule.Description = permission.Description + rules = append(rules, *rule) + } + } + } + return rules, nil +} + +func (self *SSecurityGroup) GetName() string { + if len(self.SecurityGroupName) > 0 { + return self.SecurityGroupName + } + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetStatus() string { + return "" +} + +func (self *SSecurityGroup) IsEmulated() bool { + return false +} + +func (self *SSecurityGroup) Refresh() error { + if new, err := self.vpc.region.GetSecurityGroupDetails(self.SecurityGroupId); err != nil { + return err + } else { + return jsonutils.Update(self, new) + } +} + func (self *SRegion) GetSecurityGroups(vpcId string, offset int, limit int) ([]SSecurityGroup, int, error) { if limit > 50 || limit <= 0 { limit = 50 @@ -122,15 +203,25 @@ func (self *SRegion) createSecurityGroup(vpcId string, name string, desc string) return body.GetString("SecurityGroupId") } -func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { +func (self *SRegion) modifySecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { params := make(map[string]string) + params["RegionId"] = self.RegionId params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["Description"] = rule.Description + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) protocol := rule.Protocol - if len(rule.Protocol) == 0 { + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { protocol = "all" } params["IpProtocol"] = protocol - params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } if rule.Action == secrules.SecurityRuleAllow { params["Policy"] = "accept" } else { @@ -143,7 +234,78 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi } else { params["SourceCidrIp"] = "0.0.0.0/0" } - params["DestCidrIp"] = "0.0.0.0/0" + _, err := self.ecsRequest("ModifySecurityGroupRule", params) + return err + } else { // rule.Direction == secrules.SecurityRuleEgress { + //阿里云不支持出方向API接口调用 + return nil + // if rule.IPNet != nil { + // params["DestCidrIp"] = rule.IPNet.String() + // } else { + // params["DestCidrIp"] = "0.0.0.0/0" + // } + // _, err := self.ecsRequest("ModifySecurityGroupRule", params) + // return err + } +} + +func (self *SRegion) modifySecurityGroup(secGrpId string, name string, desc string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["SecurityGroupName"] = name + if len(desc) > 0 { + params["Description"] = desc + } + _, err := self.ecsRequest("ModifySecurityGroupAttribute", params) + return err +} + +func (self *SRegion) addSecurityGroupRules(secGrpId string, rule *secrules.SecurityRule) error { + if len(rule.Ports) != 0 { + for _, port := range rule.Ports { + rule.PortStart, rule.PortEnd = port, port + if err := self.addSecurityGroupRule(secGrpId, rule); err != nil { + return err + } + } + } else { + return self.addSecurityGroupRule(secGrpId, rule) + } + return nil +} + +func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["Description"] = rule.Description + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + protocol := rule.Protocol + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = "all" + } + params["IpProtocol"] = protocol + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } + if rule.Action == secrules.SecurityRuleAllow { + params["Policy"] = "accept" + } else { + params["Policy"] = "drop" + } + params["Priority"] = fmt.Sprintf("%d", 101-rule.Priority) + if rule.Direction == secrules.SecurityRuleIngress { + if rule.IPNet != nil { + params["SourceCidrIp"] = rule.IPNet.String() + } else { + params["SourceCidrIp"] = "0.0.0.0/0" + } _, err := self.ecsRequest("AuthorizeSecurityGroup", params) return err } else { // rule.Direction == secrules.SecurityRuleEgress { @@ -152,12 +314,54 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi } else { params["DestCidrIp"] = "0.0.0.0/0" } - params["SourceCidrIp"] = "0.0.0.0/0" _, err := self.ecsRequest("AuthorizeSecurityGroupEgress", params) return err } } +func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + protocol := rule.Protocol + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = "all" + } + params["IpProtocol"] = protocol + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } + if rule.Action == secrules.SecurityRuleAllow { + params["Policy"] = "accept" + } else { + params["Policy"] = "drop" + } + params["Priority"] = fmt.Sprintf("%d", rule.Priority) + if rule.Direction == secrules.SecurityRuleIngress { + if rule.IPNet != nil { + params["SourceCidrIp"] = rule.IPNet.String() + } else { + params["SourceCidrIp"] = "0.0.0.0/0" + } + _, err := self.ecsRequest("RevokeSecurityGroup", params) + return err + } else { // rule.Direction == secrules.SecurityRuleEgress { + if rule.IPNet != nil { + params["DestCidrIp"] = rule.IPNet.String() + } else { + params["DestCidrIp"] = "0.0.0.0/0" + } + _, err := self.ecsRequest("RevokeSecurityGroupEgress", params) + return err + } +} + func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { secId, err := self.createSecurityGroup(vpcId, "", "") if err != nil { @@ -171,7 +375,7 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { PortStart: -1, PortEnd: -1, } - err = self.addSecurityGroupRule(secId, &inRule) + err = self.addSecurityGroupRules(secId, &inRule) if err != nil { return "", err } @@ -183,9 +387,190 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { PortStart: -1, PortEnd: -1, } - err = self.addSecurityGroupRule(secId, &outRule) + err = self.addSecurityGroupRules(secId, &outRule) if err != nil { return "", err } return secId, nil } + +func (self *SRegion) getSecurityGroupByTag(vpcId, secgroupId string) (*SSecurityGroup, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + params["Tag.1.Key"] = "id" + params["Tag.1.Value"] = secgroupId + + secgrps := make([]SSecurityGroup, 0) + if body, err := self.ecsRequest("DescribeSecurityGroups", params); err != nil { + return nil, err + } else if err := body.Unmarshal(&secgrps, "SecurityGroups", "SecurityGroup"); err != nil { + return nil, err + } else if len(secgrps) != 1 { + return nil, httperrors.NewNotFoundError("failed to find SecurityGroup %s", secgroupId) + } + return &secgrps[0], nil +} + +func (self *SPermission) String() string { + action := secrules.SecurityRuleDeny + if strings.ToLower(self.Policy) == "accept" { + action = secrules.SecurityRuleAllow + } + direction := "in" + if self.Direction == "egress" { + direction = "out" + } + cidr := self.SourceCidrIp + if direction == "out" { + cidr = self.DestCidrIp + } + if cidr == "0.0.0.0/0" { + cidr = "" + } + protocol := strings.ToLower(self.IpProtocol) + if protocol == "all" { + protocol = "any" + } + port, ports := "", strings.Split(self.PortRange, "/") + if ports[0] == ports[1] { + if ports[0] != "-1" { + port = ports[0] + } + } else if ports[0] != "1" && ports[1] != "65535" { + port = fmt.Sprintf("%s-%s", ports[0], ports[1]) + } + result := fmt.Sprintf("%s:%s", direction, string(action)) + if len(cidr) > 0 { + result += fmt.Sprintf(" %s", cidr) + } + result += fmt.Sprintf(" %s", protocol) + if len(port) > 0 { + result += fmt.Sprintf(" %s", port) + } + return result +} + +func (self *SRegion) addTagToSecurityGroup(secgroupId, key, value string, index int) error { + if index > 5 || index < 1 { + index = 1 + } + params := map[string]string{"ResourceType": "securitygroup", "ResourceId": secgroupId} + params[fmt.Sprintf("Tag.%d.Key", index)] = key + params[fmt.Sprintf("Tag.%d.Value", index)] = value + _, err := self.ecsRequest("AddTags", params) + return err +} + +func (self *SRegion) revokeSecurityGroup(secgroupId, instanceId string, keep bool) error { + if !keep { + return self.leaveSecurityGroup(secgroupId, instanceId) + } + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return err + } else { + for _, permission := range secgroup.Permissions.Permission { + if rule, err := secrules.ParseSecurityRule(permission.String()); err != nil { + return err + } else { + rule.Priority = permission.Priority + if err := self.delSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + } + if rule, err := secrules.ParseSecurityRule("in:allow any"); err != nil { + rule.Priority = 100 + if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + if rule, err := secrules.ParseSecurityRule("out:allow any"); err != nil { + rule.Priority = 100 + if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + } + return nil +} + +func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error { + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return err + } else { + + sort.Sort(secrules.SecurityRuleSet(rules)) + sort.Sort(PermissionSet(secgroup.Permissions.Permission)) + + i, j := 0, 0 + for i < len(rules) || j < len(secgroup.Permissions.Permission) { + if i < len(rules) && j < len(secgroup.Permissions.Permission) { + permissionStr := secgroup.Permissions.Permission[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(permissionStr, ruleStr) + if cmp == 0 { + if secgroup.Permissions.Permission[j].Description != rules[i].Description { + rules[i].Priority = secgroup.Permissions.Permission[j].Priority + if err := self.modifySecurityGroupRule(secgroupId, &rules[i]); err != nil { + log.Errorf("modifySecurityGroupRule error %v", rules[i]) + return err + } + } + i += 1 + j += 1 + } else if cmp > 0 { + if rule, err := secrules.ParseSecurityRule(permissionStr); err != nil { + return err + } else { + rule.Priority = secgroup.Permissions.Permission[j].Priority + if err := self.delSecurityGroupRule(secgroupId, rule); err != nil { + log.Errorf("delSecurityGroupRule error %v", rule) + return err + } + } + j += 1 + } else { + if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil { + log.Errorf("addSecurityGroupRule error %v", rules[i]) + return err + } + i += 1 + } + } else if i >= len(rules) { + permissionStr := secgroup.Permissions.Permission[j].String() + if rule, err := secrules.ParseSecurityRule(permissionStr); err != nil { + return err + } else { + rule.Priority = secgroup.Permissions.Permission[j].Priority + if err := self.delSecurityGroupRule(secgroupId, rule); err != nil { + log.Errorf("delSecurityGroupRule error %v", rule) + return err + } + } + j += 1 + } else if j >= len(secgroup.Permissions.Permission) { + if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil { + log.Errorf("addSecurityGroupRule error %v", rules[i]) + return err + } + i += 1 + } + } + } + return nil +} + +func (self *SRegion) assignSecurityGroup(secgroupId, instanceId string) error { + params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId} + _, err := self.ecsRequest("JoinSecurityGroup", params) + return err +} + +func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error { + params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId} + _, err := self.ecsRequest("LeaveSecurityGroup", params) + return err +} diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index 0376b3b5f7..b54700b1e6 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -113,3 +113,12 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr disk.storage = self return disk, nil } + +func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { + if disk, err := self.zone.region.getDisk(idStr); err != nil { + return nil, err + } else { + disk.storage = self + return disk, nil + } +} diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index d61028a555..9751039c39 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -5,7 +5,9 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/pkg/util/secrules" ) const ( @@ -28,7 +30,7 @@ type SVpc struct { iwires []cloudprovider.ICloudWire - secgroups []SSecurityGroup + secgroups []cloudprovider.ICloudSecurityGroup CidrBlock string CreationTime time.Time @@ -159,11 +161,15 @@ func (self *SVpc) fetchSecurityGroups() error { break } } - self.secgroups = secgroups + self.secgroups = make([]cloudprovider.ICloudSecurityGroup, len(secgroups)) + for index, secgroup := range secgroups { + secgroup.vpc = self + self.secgroups[index] = &secgroup + } return nil } -func (self *SVpc) GetSecurityGroups() ([]SSecurityGroup, error) { +func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { if self.secgroups == nil { err := self.fetchSecurityGroups() if err != nil { @@ -180,3 +186,39 @@ func (self *SVpc) GetManagerId() string { func (self *SVpc) Delete() error { return self.region.DeleteVpc(self.VpcId) } + +func (self *SVpc) syncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) (string, error) { + secgrpId := "" + if secgroup, err := self.region.getSecurityGroupByTag(self.VpcId, secgroupId); err != nil { + if secgrpId, err = self.region.createSecurityGroup(self.VpcId, name, ""); err != nil { + return "", err + } else if err := self.region.addTagToSecurityGroup(secgrpId, "id", secgroupId, 1); err != nil { + return "", err + } + //addRules + log.Debugf("Add Rules for %s", secgrpId) + for _, rule := range rules { + if err := self.region.addSecurityGroupRule(secgrpId, &rule); err != nil { + return "", err + } + } + } else { + //syncRules + secgrpId = secgroup.SecurityGroupId + log.Debugf("Sync Rules for %s", secgroup.GetName()) + if secgroup.GetName() != name { + if err := self.region.modifySecurityGroup(secgrpId, name, ""); err != nil { + log.Errorf("Change SecurityGroup name to %s failed: %v", name, err) + } + } + self.region.syncSecgroupRules(secgrpId, rules) + } + return secgrpId, nil +} +func (self *SVpc) assignSecurityGroup(secgroupId string, instanceId string) error { + return self.region.assignSecurityGroup(secgroupId, instanceId) +} + +func (self *SVpc) revokeSecurityGroup(secgroupId string, instanceId string, keep bool) error { + return self.region.revokeSecurityGroup(secgroupId, instanceId, keep) +} diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 1c7fdd3f7e..90846ed5a9 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -54,6 +54,10 @@ func (self *SDatastore) GetIZone() cloudprovider.ICloudZone { return nil } +func (self *SDatastore) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { + return nil, cloudprovider.ErrNotImplemented +} + func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 11b1f7231a..2005bbc9bc 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -1,18 +1,18 @@ package esxi import ( + "fmt" "time" + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - - "fmt" - "github.com/vmware/govmomi/object" ) var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary"} @@ -35,6 +35,10 @@ func (self *SVirtualMachine) GetGlobalId() string { return self.getUuid() } +func (self *SVirtualMachine) SyncSecurityGroup(secgroupId, name string, rules []secrules.SecurityRule) error { + return nil +} + func (self *SVirtualMachine) GetStatus() string { vm := object.NewVirtualMachine(self.manager.client.Client, self.getVirtualMachine().Self) state, err := vm.PowerState(self.manager.context) diff --git a/vendor/github.com/serialx/hashring/LICENSE b/vendor/github.com/serialx/hashring/LICENSE new file mode 100644 index 0000000000..cbc54295ad --- /dev/null +++ b/vendor/github.com/serialx/hashring/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sung-jin Hong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/serialx/hashring/README.md b/vendor/github.com/serialx/hashring/README.md new file mode 100644 index 0000000000..0077c4d703 --- /dev/null +++ b/vendor/github.com/serialx/hashring/README.md @@ -0,0 +1,70 @@ +hashring +============================ + +Implements consistent hashing that can be used when +the number of server nodes can increase or decrease (like in memcached). +The hashing ring is built using the same algorithm as libketama. + +This is a port of Python hash_ring library +in Go with the extra methods to add and remove nodes. + + +Using +============================ + +Importing :: + +```go +import "github.com/serialx/hashring" +``` + +Basic example usage :: + +```go +memcacheServers := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.249:11212"} + +ring := hashring.New(memcacheServers) +server, _ := ring.GetNode("my_key") +``` + +To fulfill replication requirements, you can also get a list of servers that should store your key. +```go +serversInRing := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.248:11212", + "192.168.0.249:11212", + "192.168.0.250:11212", + "192.168.0.251:11212", + "192.168.0.252:11212"} + +replicaCount := 3 +ring := hashring.New(serversInRing) +server, _ := ring.GetNodes("my_key", replicaCount) +``` + +Using weights example :: + +```go +weights := make(map[string]int) +weights["192.168.0.246:11212"] = 1 +weights["192.168.0.247:11212"] = 2 +weights["192.168.0.249:11212"] = 1 + +ring := hashring.NewWithWeights(weights) +server, _ := ring.GetNode("my_key") +``` + +Adding and removing nodes example :: + +```go +memcacheServers := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.249:11212"} + +ring := hashring.New(memcacheServers) +ring = ring.RemoveNode("192.168.0.246:11212") +ring = ring.AddNode("192.168.0.250:11212") +server, _ := ring.GetNode("my_key") +``` diff --git a/vendor/github.com/serialx/hashring/hashring.go b/vendor/github.com/serialx/hashring/hashring.go new file mode 100644 index 0000000000..4f4f1d59dd --- /dev/null +++ b/vendor/github.com/serialx/hashring/hashring.go @@ -0,0 +1,268 @@ +package hashring + +import ( + "crypto/md5" + "fmt" + "math" + "sort" +) + +type HashKey uint32 +type HashKeyOrder []HashKey + +func (h HashKeyOrder) Len() int { return len(h) } +func (h HashKeyOrder) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] } + +type HashRing struct { + ring map[HashKey]string + sortedKeys []HashKey + nodes []string + weights map[string]int +} + +func New(nodes []string) *HashRing { + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: make(map[string]int), + } + hashRing.generateCircle() + return hashRing +} + +func NewWithWeights(weights map[string]int) *HashRing { + nodes := make([]string, 0, len(weights)) + for node, _ := range weights { + nodes = append(nodes, node) + } + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func (h *HashRing) Size() int { + return len(h.nodes) +} + +func (h *HashRing) UpdateWithWeights(weights map[string]int) { + nodesChgFlg := false + if len(weights) != len(h.weights) { + nodesChgFlg = true + } else { + for node, newWeight := range weights { + oldWeight, ok := h.weights[node] + if !ok || oldWeight != newWeight { + nodesChgFlg = true + break + } + } + } + + if nodesChgFlg { + newhring := NewWithWeights(weights) + h.weights = newhring.weights + h.nodes = newhring.nodes + h.ring = newhring.ring + h.sortedKeys = newhring.sortedKeys + } +} + +func (h *HashRing) generateCircle() { + totalWeight := 0 + for _, node := range h.nodes { + if weight, ok := h.weights[node]; ok { + totalWeight += weight + } else { + totalWeight += 1 + h.weights[node] = 1 + } + } + + for _, node := range h.nodes { + weight := h.weights[node] + + factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight)) + + for j := 0; j < int(factor); j++ { + nodeKey := fmt.Sprintf("%s-%d", node, j) + bKey := hashDigest(nodeKey) + + for i := 0; i < 3; i++ { + key := hashVal(bKey[i*4 : i*4+4]) + h.ring[key] = node + h.sortedKeys = append(h.sortedKeys, key) + } + } + } + + sort.Sort(HashKeyOrder(h.sortedKeys)) +} + +func (h *HashRing) GetNode(stringKey string) (node string, ok bool) { + pos, ok := h.GetNodePos(stringKey) + if !ok { + return "", false + } + return h.ring[h.sortedKeys[pos]], true +} + +func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) { + if len(h.ring) == 0 { + return 0, false + } + + key := h.GenKey(stringKey) + + nodes := h.sortedKeys + pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key }) + + if pos == len(nodes) { + // Wrap the search, should return first node + return 0, true + } else { + return pos, true + } +} + +func (h *HashRing) GenKey(key string) HashKey { + bKey := hashDigest(key) + return hashVal(bKey[0:4]) +} + +func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) { + pos, ok := h.GetNodePos(stringKey) + if !ok { + return nil, false + } + + if size > len(h.nodes) { + return nil, false + } + + returnedValues := make(map[string]bool, size) + //mergedSortedKeys := append(h.sortedKeys[pos:], h.sortedKeys[:pos]...) + resultSlice := make([]string, 0, size) + + for i := pos; i < pos+len(h.sortedKeys); i++ { + key := h.sortedKeys[i%len(h.sortedKeys)] + val := h.ring[key] + if !returnedValues[val] { + returnedValues[val] = true + resultSlice = append(resultSlice, val) + } + if len(returnedValues) == size { + break + } + } + + return resultSlice, len(resultSlice) == size +} + +func (h *HashRing) AddNode(node string) *HashRing { + return h.AddWeightedNode(node, 1) +} + +func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing { + if weight <= 0 { + return h + } + + if _, ok := h.weights[node]; ok { + return h + } + + nodes := make([]string, len(h.nodes), len(h.nodes)+1) + copy(nodes, h.nodes) + nodes = append(nodes, node) + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + weights[eNode] = eWeight + } + weights[node] = weight + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing { + if weight <= 0 { + return h + } + + /* node is not need to update for node is not existed or weight is not changed */ + if oldWeight, ok := h.weights[node]; (!ok) || (ok && oldWeight == weight) { + return h + } + + nodes := make([]string, len(h.nodes), len(h.nodes)) + copy(nodes, h.nodes) + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + weights[eNode] = eWeight + } + weights[node] = weight + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} +func (h *HashRing) RemoveNode(node string) *HashRing { + /* if node isn't exist in hashring, don't refresh hashring */ + if _, ok := h.weights[node]; !ok { + return h + } + + nodes := make([]string, 0) + for _, eNode := range h.nodes { + if eNode != node { + nodes = append(nodes, eNode) + } + } + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + if eNode != node { + weights[eNode] = eWeight + } + } + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func hashVal(bKey []byte) HashKey { + return ((HashKey(bKey[3]) << 24) | + (HashKey(bKey[2]) << 16) | + (HashKey(bKey[1]) << 8) | + (HashKey(bKey[0]))) +} + +func hashDigest(key string) [md5.Size]byte { + return md5.Sum([]byte(key)) +}