From ea609cdbd1a7b677bbfe58a61a08fbfb7121d277 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 15 Aug 2018 10:54:48 +0800 Subject: [PATCH 01/71] auth token add list endpoints method --- pkg/mcclient/token.go | 10 ++++++++++ pkg/mcclient/token2.go | 4 ++++ pkg/mcclient/token3.go | 25 +++++++++++++++++++++++++ pkg/mcclient/tokensimple.go | 4 ++++ 4 files changed, 43 insertions(+) diff --git a/pkg/mcclient/token.go b/pkg/mcclient/token.go index 794df976ad..7a9ba2751b 100644 --- a/pkg/mcclient/token.go +++ b/pkg/mcclient/token.go @@ -11,6 +11,15 @@ type ExternalService struct { Url string } +type Endpoint struct { + Id string + RegionId string + ServiceId string + ServiceName string + Url string + Interface string +} + type TokenCredential interface { gotypes.ISerializable @@ -33,4 +42,5 @@ type TokenCredential interface { GetServiceURL(service, region, zone, endpointType string) (string, error) GetInternalServices(region string) []string GetExternalServices(region string) []ExternalService + GetEndpoints(region string, endpointType string) []Endpoint } diff --git a/pkg/mcclient/token2.go b/pkg/mcclient/token2.go index 9afd072292..d38bd052b9 100644 --- a/pkg/mcclient/token2.go +++ b/pkg/mcclient/token2.go @@ -144,6 +144,10 @@ func (this *TokenCredentialV2) GetExternalServices(region string) []ExternalServ return nil } +func (this *TokenCredentialV2) GetEndpoints(region string, endpointType string) []Endpoint { + return nil +} + func stringArrayContains(arr []string, needle string) bool { for i := 0; i < len(arr); i++ { if arr[i] == needle { diff --git a/pkg/mcclient/token3.go b/pkg/mcclient/token3.go index 69333f8308..f7265da44e 100644 --- a/pkg/mcclient/token3.go +++ b/pkg/mcclient/token3.go @@ -152,6 +152,10 @@ func (this *TokenCredentialV3) GetExternalServices(region string) []ExternalServ return this.Token.Catalog.getExternalServices(region) } +func (this *TokenCredentialV3) GetEndpoints(region string, endpointType string) []Endpoint { + return this.Token.Catalog.getEndpoints(region, endpointType) +} + func (catalog KeystoneServiceCatalogV3) getInternalServices(region string) []string { services := make([]string, 0) for i := 0; i < len(catalog); i++ { @@ -200,6 +204,27 @@ func (catalog KeystoneServiceCatalogV3) getRegions() []string { return regions } +func (catalog KeystoneServiceCatalogV3) getEndpoints(region string, endpointType string) []Endpoint { + endpoints := make([]Endpoint, 0) + for i := 0; i < len(catalog); i++ { + for j := 0; j < len(catalog[i].Endpoints); j++ { + endpoint := catalog[i].Endpoints[j] + if (endpoint.Region_id == region || strings.HasPrefix(endpoint.Region_id, region+"-")) && endpoint.Interface == endpointType { + endpoints = append(endpoints, Endpoint{ + endpoint.Id, + endpoint.Region_id, + catalog[i].Id, + catalog[i].Name, + endpoint.Url, + endpoint.Interface, + }) + } + } + } + + return endpoints +} + func RegionID(region, zone string) string { if len(zone) > 0 { return fmt.Sprintf("%s%c%s", region, REGION_ZONE_SEP, zone) diff --git a/pkg/mcclient/tokensimple.go b/pkg/mcclient/tokensimple.go index 8fbea3a328..2054560440 100644 --- a/pkg/mcclient/tokensimple.go +++ b/pkg/mcclient/tokensimple.go @@ -104,6 +104,10 @@ func (self *SSimpleToken) GetExternalServices(region string) []ExternalService { return nil } +func (this *SSimpleToken) GetEndpoints(region string, endpointType string) []Endpoint { + return nil +} + func SimplifyToken(token TokenCredential) TokenCredential { simToken, ok := token.(*SSimpleToken) if ok { From f991bd67f15f73efa54cd765e19848879027ddbc Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 15 Aug 2018 11:35:50 +0800 Subject: [PATCH 02/71] region: kvm server reset, fix climc parmas --- cmd/climc/climc.go | 2 +- cmd/climc/shell/servers.go | 12 ++++- pkg/compute/guestdrivers/base.go | 8 +++ pkg/compute/guestdrivers/kvm.go | 5 ++ pkg/compute/guestdrivers/virtualization.go | 13 +++++ pkg/compute/models/guestdrivers.go | 3 ++ pkg/compute/models/guests.go | 17 +++++++ pkg/compute/tasks/guest_reset_task.go | 58 ++++++++++++++++++++++ 8 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 pkg/compute/tasks/guest_reset_task.go diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index e5983068a6..d7a8da1372 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -181,7 +181,7 @@ func main() { fmt.Print(parser.HelpString()) } else if options.Version { fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString()) - } else if len(options.SUBCOMMAND) == 0 { + } else if len(os.Args) <= 1 || (options.ApiVersion == "v2" && len(os.Args) <= 3) { session, e := newClientSession(options) if e != nil { showErrorAndExit(e) diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index 7845391197..6381d8d0f1 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -339,8 +339,16 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { - ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", nil) + type ServerResetOptions struct { + ServerOpsOptions + Hard bool `help:"Hard reset or not; default soft"` + } + R(&ServerResetOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerResetOptions) error { + params := jsonutils.NewDict() + if args.Hard { + params.Add(jsonutils.JSONTrue, "is_hard") + } + ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", params) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil }) diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index b73a757637..a63aa1470c 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -102,6 +102,14 @@ func (self *SBaseGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *m return fmt.Errorf("Not Implement") } +func (self *SBaseGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + func (self *SBaseGuestDriver) AllowReconfigGuest() bool { return true } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 2a04d43eda..d98ed72b15 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -206,6 +206,11 @@ func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *m return nil } +func (self *SKVMGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + _, err := guest.SendMonitorCommand(ctx, task.GetUserCred(), "system_reset") + return err +} + func (self *SKVMGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) } diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index c6311bd5f3..7efdd6d475 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -95,6 +95,19 @@ func (self *SVirtualizedGuestDriver) StartGuestStopTask(guest *models.SGuest, ct return nil } +func (self *SVirtualizedGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + var taskName = "GuestSoftResetTask" + if isHard { + taskName = "GuestHardResetTask" + } + task, err := taskman.TaskManager.NewTask(ctx, taskName, guest, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SVirtualizedGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { if jsonutils.QueryBoolean(task.GetParams(), "restart", false) { task.SetStage("OnDeployStartGuestComplete", nil) diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 9e9ce6c70c..48406df24f 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -39,6 +39,9 @@ type IGuestDriver interface { RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *SGuest, task taskman.ITask) error StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error + + RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 444791a76b..833b967a1c 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3038,6 +3038,23 @@ func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCre } } +func (self *SGuest) AllowPerformReset(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + isHard := jsonutils.QueryBoolean(data, "is_hard", false) + if self.Status == VM_RUNNING || self.Status == VM_STOP_FAILED { + self.GetDriver().StartGuestResetTask(self, ctx, userCred, isHard, "") + return nil, nil + } + return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status) +} + func (self *SGuest) AllowPerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, diff --git a/pkg/compute/tasks/guest_reset_task.go b/pkg/compute/tasks/guest_reset_task.go new file mode 100644 index 0000000000..72a5273e2d --- /dev/null +++ b/pkg/compute/tasks/guest_reset_task.go @@ -0,0 +1,58 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestSoftResetTask{}) + taskman.RegisterTask(GuestHardResetTask{}) +} + +type GuestSoftResetTask struct { + SGuestBaseTask +} + +func (self *GuestSoftResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + err := guest.GetDriver().RequestSoftReset(ctx, guest, self) + if err == nil { + self.SetStageComplete(ctx, nil) + } else { + self.SetStageFailed(ctx, err.Error()) + } +} + +type GuestHardResetTask struct { + SGuestBaseTask +} + +func (self *GuestHardResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.StopServer(ctx, guest) +} + +func (self *GuestHardResetTask) StopServer(ctx context.Context, guest *models.SGuest) { + guest.SetStatus(self.UserCred, models.VM_STOPPING, "") + self.SetStage("OnServerStopComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartServer(ctx, guest) +} + +func (self *GuestHardResetTask) StartServer(ctx context.Context, guest *models.SGuest) { + self.SetStage("OnServerStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} From 2634b31ea5f7dad29dcabe5686ea1d498e7a1756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 15 Aug 2018 15:05:40 +0800 Subject: [PATCH 03/71] =?UTF-8?q?=E6=B7=BB=E5=8A=A0disk=20=E8=AF=A6?= =?UTF-8?q?=E6=83=85=E4=BF=A1=E6=81=AF=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/disks.go | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 2214102b4e..47442001da 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -384,6 +385,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -745,6 +753,49 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error { params := jsonutils.NewDict() params.Add(jsonutils.NewInt(size), "size") From 7d1d1d1d3759474e104758ab29cb433f961ce2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 15 Aug 2018 21:38:49 +0800 Subject: [PATCH 04/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcheck=20create=20data?= =?UTF-8?q?=20=E4=BD=BF=E7=94=A8keyword=E8=80=8C=E4=B8=8D=E6=98=AFkeywordp?= =?UTF-8?q?lural?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/db_dispatcher.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 4a661bbd77..8b4179f50d 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -841,7 +841,12 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act managerValue := reflect.ValueOf(dispatcher.modelManager) if action == "check-create-data" { - return dispatcher.modelManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data.(*jsonutils.JSONDict)) + manager := dispatcher.modelManager + if body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword()); err != nil { + return nil, httperrors.NewGeneralError(err) + } else { + return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, body.(*jsonutils.JSONDict)) + } } return objectPerformAction(dispatcher, managerValue, ctx, userCred, action, query, data) } From 231e94fbd802ae00549b3a0c704b9fdcfcd259a1 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 15 Aug 2018 21:52:24 +0800 Subject: [PATCH 05/71] 1.fix corn run task with empyt context --- pkg/cloudcommon/cronman/cronman.go | 23 ++++++++++---------- pkg/cloudcommon/db/taskman/tasks.go | 18 ++++++++++----- pkg/compute/models/guests.go | 6 ++--- pkg/compute/tasks/guest_rebuild_root_task.go | 4 ++++ 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index b2e443d983..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -1,33 +1,34 @@ package cronman import ( - "time" - "runtime/debug" "context" + "reflect" + "runtime" + "runtime/debug" + "time" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" - "reflect" - "runtime" ) const ( - DEFAULT_CRON_INTERVAL = 60*time.Second // default resolution is 1 monutes + DEFAULT_CRON_INTERVAL = 60 * time.Second // default resolution is 1 monutes ) type SCronJobManager struct { checkInterval time.Duration - timer *time.Timer - jobs []SCronJob + timer *time.Timer + jobs []SCronJob } type SCronJob struct { - name string + name string runInterval time.Duration - job func(ctx context.Context, userCred mcclient.TokenCredential) - lastRun time.Time + job func(ctx context.Context, userCred mcclient.TokenCredential) + lastRun time.Time } func NewCronJobManager(interval time.Duration) *SCronJobManager { @@ -89,7 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } - diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index bd8578ca36..8e4f45acf6 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -412,9 +416,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index c12f0e4221..20f695e286 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2581,9 +2581,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go index 0d0ee7ee6a..03345b85ac 100644 --- a/pkg/compute/tasks/guest_rebuild_root_task.go +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -32,6 +32,10 @@ func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandalone } } +func (self *GuestRebuildRootTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartRebuildRootDisk(ctx, guest) +} + func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) { db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred) gds := guest.CategorizeDisks() From e9d437d1736f2b5289ee571199d9c6cf3d3723e9 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 16 Aug 2018 11:47:06 +0800 Subject: [PATCH 06/71] =?UTF-8?q?zone=20capability=E7=9A=84hypervisor?= =?UTF-8?q?=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/zones.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 05fc01a0c9..fe4379b0de 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -515,7 +515,7 @@ func (self *SZone) GetDetailsCapabilities(ctx context.Context, userCred mcclient } func (self *SZone) getHypervisors() []string { - q := HostManager.Query("host_type").Equals("zone_id", self.Id) + q := HostManager.Query("host_type").Equals("zone_id", self.Id).Distinct() rows, err := q.Rows() if err != nil { return nil From bb5beccf47ba342e0ec5caa816e9b7f019881b23 Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 16 Aug 2018 14:19:01 +0800 Subject: [PATCH 07/71] add more resources interfaces --- pkg/cloudprovider/resources.go | 8 +++++++- pkg/util/aliyun/instance.go | 20 ++++++++++++++++++++ pkg/util/esxi/virtualmachine.go | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f388d66feb..bee54d3cd9 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -153,7 +153,12 @@ type ICloudVM interface { StopVM(isForce bool) error DeleteVM() error + UpdateVM(name string) error + RebuildRoot(imageId string) error + DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error + ChangeConfig(instanceId string,ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) + AttachDisk(diskId string) error } type ICloudNic interface { @@ -193,7 +198,8 @@ type ICloudDisk interface { GetCacheMode() string GetMountpoint() string Delete() error - Resize(int64) error + + Resize(newSize int64) error } type ICloudVpc interface { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 80a2e9c267..e6dbbf5610 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -539,3 +539,23 @@ func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules [ } return nil } + +func (self *SInstance) UpdateVM(name string) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) RebuildRoot(imageId string) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) AttachDisk(diskId string) error { + return fmt.Errorf("not implement") +} \ No newline at end of file diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2005bbc9bc..d4aed7891f 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -209,3 +209,23 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(url), "url") return ret, nil } + +func (self *SVirtualMachine) UpdateVM(name string) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) RebuildRoot(imageId string) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) AttachDisk(diskId string) error { + return fmt.Errorf("not implement") +} \ No newline at end of file From 7cad0d54f39b6bfa87474f80e5136f21e1f8d954 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 16 Aug 2018 16:19:52 +0800 Subject: [PATCH 08/71] scheduler: network container server_type --- cmd/climc/shell/networks.go | 4 ++-- .../algorithm/predicates/guest/network_predicate.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/climc/shell/networks.go b/cmd/climc/shell/networks.go index fdf011d56f..52d6900a28 100644 --- a/cmd/climc/shell/networks.go +++ b/cmd/climc/shell/networks.go @@ -45,7 +45,7 @@ func init() { ID string `help:"ID or Name of zone to update"` Name string `help:"Name of zone"` Desc string `metavar:"" help:"Description"` - ServerType string `help:"server type," choices:"baremetal|guest"` + ServerType string `help:"server type," choices:"baremetal|guest|container"` StartIp string `help:"Start ip"` EndIp string `help:"end ip"` NetMask int64 `help:"Netmask"` @@ -178,7 +178,7 @@ func init() { Gateway string `help:"Default gateway"` VlanId int64 `help:"Vlan ID" default:"1"` AllocPolicy string `help:"Address allocation policy" choices:"none|stepdown|stepup|random"` - ServerType string `help:"Server type" choices:"baremetal|guest"` + ServerType string `help:"Server type" choices:"baremetal|guest|container"` Desc string `help:"Description" metavar:"DESCRIPTION"` } R(&NetworkCreateOptions{}, "network-create", "Create a virtual network", func(s *mcclient.ClientSession, args *NetworkCreateOptions) error { diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index 8f0c13aa50..e78ba0d99c 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/core" networks "yunion.io/x/onecloud/pkg/scheduler/db/models" "yunion.io/x/pkg/utils" + "yunion.io/yunioncloud/pkg/util/sets" ) // NetworkPredicate will filter the current network information with @@ -51,9 +52,9 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return len(d.HostID) > 0 } - // ServerType's value is 'guest' or ''(support all type) will return true. + // ServerType's value is 'guest', 'container' or ''(support all type) will return true. isMatchServerType := func(network *networks.NetworkSchedResult) bool { - return network.ServerType == "guest" || network.ServerType == "" + return sets.NewString("guest", "", "container").Has(network.ServerType) } counterOfNetwork := func(u *core.Unit, n *networks.NetworkSchedResult, r int) core.Counter { From cf39f5ed4482912405381c1a4347e82f5ac21d8f Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 16 Aug 2018 18:47:10 +0800 Subject: [PATCH 09/71] climc: server-deploy password --- cmd/climc/shell/servers.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index e6bccf2e57..bad4870e80 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -139,6 +139,7 @@ func init() { Net []string `help:"Network descriptions" metavar:"NETWORK"` IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` Keypair string `help:"SSH Keypair"` + Password string `help:"Default user password"` Iso string `help:"ISO image ID" metavar:"IMAGE_ID"` Ncpu int64 `help:"#CPU cores of VM server, default 1" default:"1" metavar:""` Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl"` @@ -192,6 +193,9 @@ func init() { if args.NoAccountInit { params.Add(jsonutils.JSONFalse, "reset_password") } + if args.Password != "" { + params.Add(jsonutils.NewString(args.Password), "password") + } if len(args.Vdi) > 0 { params.Add(jsonutils.NewString(args.Vdi), "vdi") } @@ -484,6 +488,7 @@ func init() { DeleteKeypair bool `help:"Remove ssh Keypairs"` Deploy []string `help:"Specify deploy files in virtual server file system"` ResetPassword bool `help:"Force reset password"` + Password string `help:"Default user password"` } R(&ServerDeployOptions{}, "server-deploy", "Deploy hostname and keypair to a stopped virtual server", func(s *mcclient.ClientSession, args *ServerDeployOptions) error { params := jsonutils.NewDict() @@ -501,6 +506,9 @@ func init() { if args.ResetPassword { params.Add(jsonutils.JSONTrue, "reset_password") } + if args.Password != "" { + params.Add(jsonutils.NewString(args.Password), "password") + } srv, e := modules.Servers.PerformAction(s, args.ID, "deploy", params) if e != nil { return e From 7f0d75cfa1f1e1267391a09d818c13bc06f9036d Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 16 Aug 2018 12:50:11 +0000 Subject: [PATCH 10/71] scheduler: fix wrong import path --- pkg/scheduler/algorithm/predicates/guest/network_predicate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index e78ba0d99c..bd6ece34bf 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -9,8 +9,8 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" networks "yunion.io/x/onecloud/pkg/scheduler/db/models" + "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" - "yunion.io/yunioncloud/pkg/util/sets" ) // NetworkPredicate will filter the current network information with From f65acc402b999fcbbca8c53d6f17080f0a0b5583 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 16 Aug 2018 13:06:31 +0000 Subject: [PATCH 11/71] dep: update for recent changes - Gopkg.toml: restore constraint on yunion.io/x/pkg to master branch - treewide: convert optional:bool to positional:!bool --- Gopkg.lock | 11 +- Gopkg.toml | 4 +- cmd/climc/shell/dnsrecords.go | 16 +-- cmd/climc/shell/servers.go | 2 +- vendor/yunion.io/x/jsonutils/marshal.go | 5 +- vendor/yunion.io/x/jsonutils/unmarshal.go | 59 +++++++-- vendor/yunion.io/x/pkg/gotypes/serialize.go | 30 ++--- .../x/pkg/util/reflectutils/reflectutils.go | 20 ++- .../yunion.io/x/pkg/util/secrules/secrules.go | 36 ++++-- vendor/yunion.io/x/sqlchemy/column.go | 7 +- vendor/yunion.io/x/sqlchemy/insert.go | 33 +++-- vendor/yunion.io/x/sqlchemy/parser.go | 15 +-- vendor/yunion.io/x/sqlchemy/query.go | 1 - vendor/yunion.io/x/sqlchemy/reflect.go | 31 +++-- vendor/yunion.io/x/sqlchemy/sync.go | 4 +- vendor/yunion.io/x/sqlchemy/table.go | 23 ++-- vendor/yunion.io/x/sqlchemy/tablealias.go | 8 +- vendor/yunion.io/x/sqlchemy/update.go | 6 +- vendor/yunion.io/x/structarg/structarg.go | 119 +++++++++++------- 19 files changed, 265 insertions(+), 165 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index eb4115111f..8323a741ba 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -844,7 +844,7 @@ branch = "master" name = "yunion.io/x/jsonutils" packages = ["."] - revision = "a86e16793be4a6f4c5ea9203e40c2a0a9361efb8" + revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" [[projects]] branch = "master" @@ -856,6 +856,7 @@ revision = "a10b94c5038480920262e0f344656328434955a2" [[projects]] + branch = "master" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -889,23 +890,23 @@ "util/workqueue", "utils" ] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" + revision = "30cdcc2f0c2ca0f3eef573c27eb7350bdd95a7dd" [[projects]] branch = "master" name = "yunion.io/x/sqlchemy" packages = ["."] - revision = "f6a47145f9b94ebff1517017e790f068698b5b5a" + revision = "8d8043b1fc6f8a9192f1d086520bb167a8423475" [[projects]] branch = "master" name = "yunion.io/x/structarg" packages = ["."] - revision = "5f84a68c40a2d8eb923f0607f809cbda8d5b944a" + revision = "adf929ce0f8bd62335ebe57d9b51f023c84e4d7a" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cff7836c426e8d0c9f6d7220994a2fbcd8b3fa2f4604e1aba3e9a0f50a0da98f" + inputs-digest = "8b3cd1b8309273551ebd55dfb0cadeaaad0aecc1b3a94d680a4a1769d2f42512" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 3343180803..007f3aec01 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -34,7 +34,6 @@ name = "yunion.io/x/jsonutils" [[constraint]] - #revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53" branch = "master" name = "yunion.io/x/sqlchemy" @@ -43,8 +42,7 @@ name = "yunion.io/x/log" [[constraint]] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" - #branch = "master" + branch = "master" name = "yunion.io/x/pkg" [[constraint]] diff --git a/cmd/climc/shell/dnsrecords.go b/cmd/climc/shell/dnsrecords.go index 45b7c923ff..05185894ef 100644 --- a/cmd/climc/shell/dnsrecords.go +++ b/cmd/climc/shell/dnsrecords.go @@ -10,12 +10,12 @@ import ( ) type DNSRecordOptions struct { - A []string `help:"DNS A record" metavar:"A_RECORD" optional:"true"` - AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" optional:"true"` - CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" optional:"true"` - SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" optional:"true"` - SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" optional:"true"` - PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" optional:"true"` + A []string `help:"DNS A record" metavar:"A_RECORD" positional:"false"` + AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" positional:"false"` + CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" positional:"false"` + SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" positional:"false"` + SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" positional:"false"` + PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" positional:"false"` } func parseDNSRecords(args *DNSRecordOptions, params *jsonutils.JSONDict) { @@ -52,7 +52,7 @@ func init() { type DNSCreateOptions struct { NAME string `help:"DNS name to create"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description" metavar:"DESCRIPTION"` DNSRecordOptions } @@ -92,7 +92,7 @@ func init() { type DNSUpdateOptions struct { ID string `help:"ID of DNS record to update"` Name string `help:"Domain name"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description"` } R(&DNSUpdateOptions{}, "dns-update", "Update details of a dns records", func(s *mcclient.ClientSession, args *DNSUpdateOptions) error { diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index e6bccf2e57..999afdfd5a 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -511,7 +511,7 @@ func init() { type ServerSecGroupOptions struct { ID string `help:"ID or Name of server" metavar:"Guest"` - SecGrp string `help:"ID of Security Group" metavar:"Security Group" optional:"false"` + SecGrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"` } R(&ServerSecGroupOptions{}, "server-assign-secgroup", "Assign security group to a VM", func(s *mcclient.ClientSession, args *ServerSecGroupOptions) error { diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index bedce745e9..aff79cf1d2 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -12,8 +12,8 @@ import ( "reflect" "time" - "yunion.io/x/pkg/gotypes" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/timeutils" @@ -58,6 +58,9 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { objPairs = append(objPairs, newPairs...) } else { key := reflectutils.GetStructFieldName(&fieldType) // utils.CamelSplit(fieldType.Name, "_") + if key == "" { + continue + } val := marshalValue(val.Field(i)) if val != JSONNull { objPair := JSONPair{key: key, val: val} diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index 3a98a40490..e1f3b62144 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -47,8 +47,6 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error { } func (this *JSONValue) unmarshalValue(val reflect.Value) error { - // return fmt.Errorf("JSONValue: type mismatch") - // null value if val.CanSet() { zeroVal := reflect.New(val.Type()).Elem() val.Set(zeroVal) @@ -102,6 +100,11 @@ func (this *JSONInt) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%d", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONInt type mismatch: %s", val.Type()) } @@ -162,6 +165,11 @@ func (this *JSONBool) unmarshalValue(val reflect.Value) error { } else { val.SetString("false") } + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONBool type mismatch: %s", val.Type()) } @@ -231,6 +239,11 @@ func (this *JSONFloat) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%f", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONFloat type mismatch: %s", val.Type()) } @@ -314,6 +327,11 @@ func (this *JSONString) unmarshalValue(val reflect.Value) error { val.SetBool(utils.ToBool(this.data)) case reflect.String: val.SetString(this.data) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONString type mismatch: %s", val.Type()) } @@ -339,15 +357,32 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error { case reflect.String: val.SetString(this.String()) return nil + case reflect.Ptr: + if val.IsNil() { + kind := val.Type().Elem().Kind() + if kind == reflect.Array || kind == reflect.Slice { + val.Set(reflect.New(val.Type().Elem())) + return this.unmarshalValue(val.Elem()) + } + } + return fmt.Errorf("JSONArray type mismatch %s", val.Type()) case reflect.Slice, reflect.Array: - for _, json := range this.data { - newEle := reflect.New(val.Type().Elem()).Elem() - err := json.unmarshalValue(newEle) + if val.Kind() == reflect.Array { + if val.Len() != len(this.data) { + return fmt.Errorf("JSONArray length unmatch %s: %d != %d", + val.Type(), val.Len(), len(this.data)) + } + } else if val.Kind() == reflect.Slice { + if val.Len() < len(this.data) { + newVal := reflect.MakeSlice(val.Type(), len(this.data), len(this.data)) + val.Set(newVal) + } + } + for i, json := range this.data { + err := json.unmarshalValue(val.Index(i)) if err != nil { return err } - newVal := reflect.Append(val, newEle) - val.Set(newVal) } default: return fmt.Errorf("JSONArray type mismatch: %s", val.Type()) @@ -376,6 +411,16 @@ func (this *JSONDict) unmarshalValue(val reflect.Value) error { return this.unmarshalMap(val) case reflect.Struct: return this.unmarshalStruct(val) + case reflect.Ptr: + if val.IsNil() { + kind := val.Type().Elem().Kind() + if kind != reflect.Struct && kind != reflect.Map { + return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) + } + newVal := reflect.New(val.Type().Elem()) + val.Set(newVal) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) } diff --git a/vendor/yunion.io/x/pkg/gotypes/serialize.go b/vendor/yunion.io/x/pkg/gotypes/serialize.go index 8cb8c41d07..fdbc271497 100644 --- a/vendor/yunion.io/x/pkg/gotypes/serialize.go +++ b/vendor/yunion.io/x/pkg/gotypes/serialize.go @@ -14,30 +14,32 @@ type ISerializable interface { type FuncSerializableAllocator func() ISerializable -var ISerializableType reflect.Type -var serializableAllocators map[reflect.Type]FuncSerializableAllocator -var ErrTypeNotSerializable error - -func init() { - ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() - serializableAllocators = make(map[reflect.Type]FuncSerializableAllocator) +var ( + ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() + serializableAllocators = map[reflect.Type]FuncSerializableAllocator{} ErrTypeNotSerializable = errors.New("Type not serializable") -} +) +// RegisterSerializable registers an allocator func for the specified serializable type. +// +// This is intended to be used when you have multiple implmenetations of an +// interface and you want to use only one of them to cover them all. +// TokenCredential and SSimpleToken is such a case. func RegisterSerializable(valType reflect.Type, alloc FuncSerializableAllocator) { + if !IsSerializable(valType) { + panic(valType.String() + " does not implement ISerializable") + } serializableAllocators[valType] = alloc } -func IsSerializable(objType reflect.Type) bool { - _, ok := serializableAllocators[objType] - return ok +func IsSerializable(valType reflect.Type) bool { + return valType.Implements(ISerializableType) } func NewSerializable(objType reflect.Type) (ISerializable, error) { deserFunc, ok := serializableAllocators[objType] - if !ok { - return nil, ErrTypeNotSerializable - } else { + if ok { return deserFunc(), nil } + return nil, ErrTypeNotSerializable } diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go index 6b58992180..55d9019d8e 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go @@ -2,6 +2,7 @@ package reflectutils import ( "reflect" + "strings" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" @@ -15,14 +16,23 @@ func GetStructFieldName(field *reflect.StructField) string { return nameStr } else { jsonStr, _ := tagMap["json"] - if len(jsonStr) > 0 { - return jsonStr - } else { - return utils.CamelSplit(field.Name, "_") - } + return toJsonKey(field.Name, jsonStr) } } +func toJsonKey(fieldName, jsonTag string) string { + jsonTag = strings.Replace(jsonTag, "omitempty", "", -1) + words := utils.FindWords([]byte(jsonTag), 0) + if len(words) == 0 { + return utils.CamelSplit(fieldName, "_") + } + name := words[0] + if name == "-" { + return "" + } + return name +} + func FetchStructFieldNameValueInterfaces(dataValue reflect.Value) map[string]interface{} { fields := make(map[string]interface{}) fetchStructFieldNameValueInterfaces(dataValue.Type(), dataValue, fields) diff --git a/vendor/yunion.io/x/pkg/util/secrules/secrules.go b/vendor/yunion.io/x/pkg/util/secrules/secrules.go index 663ee8632f..4f249fdcd6 100644 --- a/vendor/yunion.io/x/pkg/util/secrules/secrules.go +++ b/vendor/yunion.io/x/pkg/util/secrules/secrules.go @@ -25,14 +25,15 @@ const ( ) type SecurityRule struct { - Priority int // [1, 100] - Action TSecurityRuleAction - IPNet *net.IPNet - Protocol string - Direction TSecurityRuleDirection - PortStart int - PortEnd int - Ports []int + Priority int // [1, 100] + Action TSecurityRuleAction + IPNet *net.IPNet + Protocol string + Direction TSecurityRuleDirection + PortStart int + PortEnd int + Ports []int + Description string } const ( @@ -62,6 +63,25 @@ var ( ErrInvalidPort = errors.New("invalid port") ) +type SecurityRuleSet []SecurityRule + +func (v SecurityRuleSet) Len() int { + return len(v) +} + +func (v SecurityRuleSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurityRuleSet) Less(i, j int) bool { + if v[i].Priority > v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + func parsePortString(ps string) (int, error) { p, err := strconv.ParseUint(ps, 10, 16) if err != nil || p == 0 { diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 90ff609a11..8f65bdaf74 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/regutils" @@ -465,11 +464,11 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool { func NewDecimalColumn(name string, tagmap map[string]string) SDecimalColumn { tagmap, v, ok := utils.TagPop(tagmap, TAG_PRECISION) if !ok { - log.Fatalf("Field %s of float should have precision tag", name) + panic(fmt.Sprintf("Field %q of float misses precision tag", name)) } prec, err := strconv.Atoi(v) if err != nil { - log.Fatalf("Field %s of float precision %s shoud be integer!", name, v) + panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) } return SDecimalColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "DECIMAL", tagmap), Precision: prec} @@ -544,7 +543,7 @@ func NewTextColumn(name string, tagmap map[string]string) STextColumn { if len(charset) == 0 { charset = "utf8" } else if charset != "utf8" && charset != "ascii" { - log.Fatalf("Unsupported charset %s for %s", charset, name) + panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name)) } return STextColumn{SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap), Charset: charset} diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index 341ce91343..06e3518d81 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -42,7 +42,6 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { dtc, ok := c.(*SDateTimeColumn) ov := fields[k] - // log.Debugf("field %s value %s %s", k, ov, ov==nil) if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) @@ -54,15 +53,13 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { format = append(format, "?") } else if c.IsPrimary() { if isAutoInc { - if len(autoIncField) == 0 { - autoIncField = k - } else { - log.Fatalf("multiple auto_increment columns???") + if len(autoIncField) > 0 { + panic(fmt.Sprintf("multiple auto_increment columns: %q, %q", autoIncField, k)) } - } else { - return fmt.Errorf("fail to insert for null primary key `%s`", k) + autoIncField = k } - } else if ! c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value + return fmt.Errorf("cannot insert for null primary key %q", k) + } else if !c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value val := c.ConvertFromString(c.Default()) values = append(values, val) names = append(names, fmt.Sprintf("`%s`", k)) @@ -92,15 +89,15 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { } /* - if len(autoIncField) > 0 { - lastId, err := results.LastInsertId() - if err == nil { - val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) - if ok { - gotypes.SetValue(val, fmt.Sprint(lastId)) + if len(autoIncField) > 0 { + lastId, err := results.LastInsertId() + if err == nil { + val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) + if ok { + gotypes.SetValue(val, fmt.Sprint(lastId)) + } } } - } */ // query the value, so default value can be feedback into the object @@ -112,7 +109,7 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { if ok && nc.IsAutoIncrement { lastId, err := results.LastInsertId() if err != nil { - log.Errorf("Fail to fetch lastInsertId %s", err) + err := fmt.Errorf("fetching lastInsertId failed: %v", err) return err } else { q = q.Equals(c.Name(), lastId) @@ -124,9 +121,9 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { } err = q.First(dt) if err != nil { - log.Errorf("query after insert failed %s", err) + err := fmt.Errorf("query after insert failed: %v", err) return err } - + return nil } diff --git a/vendor/yunion.io/x/sqlchemy/parser.go b/vendor/yunion.io/x/sqlchemy/parser.go index 2464f9c82c..c66f38bf7b 100644 --- a/vendor/yunion.io/x/sqlchemy/parser.go +++ b/vendor/yunion.io/x/sqlchemy/parser.go @@ -3,14 +3,13 @@ package sqlchemy import ( "reflect" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/utils" ) -func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { +func structField2ColumnSpec(field *reflect.StructField) IColumnSpec { fieldname := reflectutils.GetStructFieldName(field) tagmap := utils.TagMap(field.Tag) if _, ok := tagmap[TAG_IGNORE]; ok { @@ -85,13 +84,11 @@ func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { col := NewJSONColumn(fieldname, tagmap) return &col*/ default: - if gotypes.IsSerializable(field.Type) { + if field.Type.Implements(gotypes.ISerializableType) { col := NewCompoundColumn(fieldname, tagmap) return &col - } else { - log.Fatalf("Unsupported type! %s", field.Type) - return nil } + panic("not supported type %s" + field.Type.Name()) } } @@ -101,9 +98,9 @@ func struct2TableSpec(st reflect.Type, table *STableSpec) { if f.Type.Kind() == reflect.Struct && f.Type != gotypes.TimeType { struct2TableSpec(f.Type, table) } else { - coldef := fieldToColumnSpec(&f) - if coldef != nil { - table.columns = append(table.columns, coldef) + column := structField2ColumnSpec(&f) + if column != nil { + table.columns = append(table.columns, column) } } } diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 7b0f332e5b..619240006e 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -126,7 +126,6 @@ func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { return &sqf } } - log.Errorf("field %s not found for subquery", id) return nil } diff --git a/vendor/yunion.io/x/sqlchemy/reflect.go b/vendor/yunion.io/x/sqlchemy/reflect.go index 7c7753a2ad..c85cb63a0e 100644 --- a/vendor/yunion.io/x/sqlchemy/reflect.go +++ b/vendor/yunion.io/x/sqlchemy/reflect.go @@ -25,7 +25,7 @@ func getStringValue(dat interface{}) string { case gotypes.IntType, gotypes.Int8Type, gotypes.Int16Type, gotypes.Int32Type, gotypes.Int64Type: return fmt.Sprintf("%d", value.Int()) case gotypes.UintType, gotypes.Uint8Type, gotypes.Uint16Type, gotypes.Uint32Type, gotypes.Uint64Type: - return fmt.Sprintf("%u", value.Uint()) + return fmt.Sprintf("%d", value.Uint()) case gotypes.Float32Type, gotypes.Float64Type: return fmt.Sprintf("%f", value.Float()) case gotypes.StringType: @@ -53,15 +53,12 @@ func getStringValue(dat interface{}) string { return string(rawBytes) } default: - if gotypes.IsSerializable(value.Type()) { - is, ok := value.Interface().(gotypes.ISerializable) - if !ok { - log.Errorf("%s fail to convert to JSONObject", value) - } else { - return is.String() - } + serializable, ok := value.Interface().(gotypes.ISerializable) + if !ok { + log.Errorf("cannot convert %v to string", value) + return "" } - log.Errorf("Cannot convert to string %s", value) + return serializable.String() } return "" } @@ -127,15 +124,15 @@ func setValueBySQLString(value reflect.Value, val string) error { gotypes.Float32SliceType, gotypes.Float64SliceType, gotypes.StringSliceType: reflect.Append(value, reflect.ValueOf(val)) default: - if gotypes.IsSerializable(value.Type()) { - is, err := jsonutils.JSONDeserialize(value.Type(), val) - if err != nil { - return err - } - value.Set(reflect.ValueOf(is)) - } else { - return fmt.Errorf("!!!Unsupported type: %s", value.Type) + valueType := value.Type() + if !valueType.Implements(gotypes.ISerializableType) { + return fmt.Errorf("not supported type: %s", valueType) } + serializable, err := jsonutils.JSONDeserialize(valueType, val) + if err != nil { + return err + } + value.Set(reflect.ValueOf(serializable)) } return nil } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 07b3df8a81..d6343129f1 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -212,10 +212,10 @@ func (ts *STableSpec) Sync() error { func (ts *STableSpec) CheckSync() { sqls := ts.SyncSQL() - if sqls != nil && len(sqls) > 0 { + if len(sqls) > 0 { for _, sql := range sqls { fmt.Println(sql) } - log.Fatalf("DB not in sync!!!") + log.Fatalf("DB table %q not in sync", ts.name) } } diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 3f9813f5de..74eedc5a51 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -5,7 +5,6 @@ import ( "reflect" "strings" - "yunion.io/x/log" "yunion.io/x/pkg/utils" ) @@ -29,12 +28,15 @@ type STableField struct { func NewTableSpecFromStruct(s interface{}, name string) *STableSpec { st := reflect.TypeOf(s) if st.Kind() != reflect.Struct { - log.Fatalf("Invalid table struct, NOT a STRUCT!!!") - return nil + panic("expect Struct kind") } - table := STableSpec{columns: make([]IColumnSpec, 0), name: name, structType: st} - struct2TableSpec(st, &table) - return &table + table := &STableSpec{ + columns: []IColumnSpec{}, + name: name, + structType: st, + } + struct2TableSpec(st, table) + return table } func (ts *STableSpec) Name() string { @@ -88,12 +90,11 @@ func (ts *STableSpec) ColumnSpec(name string) IColumnSpec { func (tbl *STable) Field(name string, alias ...string) IQueryField { // name = reflectutils.StructFieldName(name) name = utils.CamelSplit(name, "_") - cSpec := tbl.spec.ColumnSpec(name) - if cSpec == nil { - log.Fatalf("Column %s not found", name) - return nil + spec := tbl.spec.ColumnSpec(name) + if spec == nil { + panic("column not found: " + name) } - col := STableField{table: tbl, spec: cSpec} + col := STableField{table: tbl, spec: spec} if len(alias) > 0 { col.Label(alias[0]) } diff --git a/vendor/yunion.io/x/sqlchemy/tablealias.go b/vendor/yunion.io/x/sqlchemy/tablealias.go index 47898e4ba6..eec3c89d78 100644 --- a/vendor/yunion.io/x/sqlchemy/tablealias.go +++ b/vendor/yunion.io/x/sqlchemy/tablealias.go @@ -6,14 +6,10 @@ import ( ) var ( - tableID = 0 - tableIDLock *sync.Mutex + tableID = 0 + tableIDLock *sync.Mutex = &sync.Mutex{} ) -func init() { - tableIDLock = &sync.Mutex{} -} - func getTableAliasName() string { tableIDLock.Lock() defer tableIDLock.Unlock() diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index b651f338aa..73cd0b8dfc 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -64,9 +64,9 @@ func UpdateDiffString(diff map[string]SUpdateDiff) string { } func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, error) { - beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") - if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { - beforeInsertFunc.Call([]reflect.Value{}) + beforeUpdateFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") + if beforeUpdateFunc.IsValid() && !beforeUpdateFunc.IsNil() { + beforeUpdateFunc.Call([]reflect.Value{}) } // dataType := reflect.TypeOf(dt).Elem() diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 2f64914771..7e4c9193d5 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -27,8 +27,8 @@ type Argument interface { AliasToken() string ShortToken() string MetaVar() string - IsOptional() bool IsPositional() bool + IsRequired() bool IsMulti() bool IsSubcommand() bool HelpString(indent string) string @@ -45,8 +45,8 @@ type SingleArgument struct { aliasToken string shortToken string metavar string - optional bool positional bool + required bool help string choices []string useDefault bool @@ -136,7 +136,13 @@ const ( A boolean value explicitly declare whether the argument is optional, the tag is optional */ - TAG_OPTIONAL = "optional" + TAG_POSITIONAL = "positional" + /* + A boolean value explicitly declare whether the argument is required. + The tag is optional. This is for optional arguments. Positional + arguments must be "required" + */ + TAG_REQUIRED = "required" /* A boolean value explicitly decalre whther the argument is an subcommand A subcommand argument must be the last positional argument. @@ -263,51 +269,64 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } } } - var positional, optional bool + // heuristic guessing "positional" + var positional bool if f.Name == strings.ToUpper(f.Name) { positional = true - optional = false } else { positional = false - optional = true } - opt_val := tagMap[TAG_OPTIONAL] - if len(opt_val) > 0 { - if opt_val == "true" { - optional = true - positional = false - } else if opt_val == "false" { - optional = false + if positionalTag := tagMap[TAG_POSITIONAL]; len(positionalTag) > 0 { + switch positionalTag { + case "true": positional = true - } else { - return fmt.Errorf("Invalid optional value %s, neither true nor false", opt_val) + case "false": + positional = false + default: + return fmt.Errorf("Invalid positional tag %q, neither true nor false", positionalTag) } - // fmt.Println(token, "optional", opt_val, optional, positional) } - if positional && !optional && use_default { - return fmt.Errorf("A positional non-optional argument should not set default value") + required := positional + if requiredTag := tagMap[TAG_REQUIRED]; len(requiredTag) > 0 { + switch requiredTag { + case "true": + required = true + case "false": + required = false + default: + return fmt.Errorf("Invalid required tag %q, neither true nor false", requiredTag) + } } - subcommand, e := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) - if e != nil { + if positional { + if !required { + return fmt.Errorf("positional %s must not have required:false", token) + } + if use_default { + return fmt.Errorf("positional %s must not have default value", token) + } + } + subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) + if err != nil { subcommand = false } var defval_t reflect.Value if use_default { - defval_t, e = gotypes.ParseValue(defval, f.Type) - if e != nil { - return e + defval_t, err = gotypes.ParseValue(defval, f.Type) + if err != nil { + return err } } if subcommand { positional = true - optional = false } var arg Argument = nil ovalue := reflect.New(v.Type()).Elem() ovalue.Set(v) sarg := SingleArgument{token: token, shortToken: shorttoken, - optional: optional, positional: positional, - metavar: metavar, help: help, + positional: positional, + required: required, + metavar: metavar, + help: help, choices: choices, useDefault: use_default, aliasToken: alias, @@ -321,7 +340,7 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) subcommands: make(map[string]SubcommandArgumentData)} } else if f.Type.Kind() == reflect.Array || f.Type.Kind() == reflect.Slice { var min, max int64 - var e error + var err error nargs := tagMap[TAG_NARGS] if nargs == "*" { min = 0 @@ -333,13 +352,13 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) min = 1 max = -1 } else { - min, e = strconv.ParseInt(nargs, 10, 64) - if e == nil { + min, err = strconv.ParseInt(nargs, 10, 64) + if err == nil { max = min } else if positional { min = 1 max = -1 - } else if optional { + } else if !required { min = 0 max = -1 } @@ -349,7 +368,11 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } else { arg = &sarg } - return this.AddArgument(arg) + err = this.AddArgument(arg) + if err != nil { + return fmt.Errorf("AddArgument %s: %v", arg, err) + } + return nil } func (this *ArgumentParser) AddArgument(arg Argument) error { @@ -361,13 +384,25 @@ func (this *ArgumentParser) AddArgument(arg Argument) error { return fmt.Errorf("Cannot append positional argument after an array positional argument") case last_arg.IsSubcommand(): return fmt.Errorf("Cannot append positional argument after a subcommand argument") - case last_arg.IsOptional() && !arg.IsOptional(): - return fmt.Errorf("Cannot append positional argument after an optional positional argument") } } this.posArgs = append(this.posArgs, arg) } else { - this.optArgs = append(this.optArgs, arg) + // Put required at the end and try to be stable + if arg.IsRequired() { + this.optArgs = append(this.optArgs, arg) + } else { + var i int + var opt Argument + for i, opt = range this.optArgs { + if opt.IsRequired() { + break + } + } + this.optArgs = append(this.optArgs, nil) + copy(this.optArgs[i+1:], this.optArgs[i:]) + this.optArgs[i] = arg + } } return nil } @@ -432,12 +467,12 @@ func (this *SingleArgument) ShortToken() string { func (this *SingleArgument) String() string { var start, end byte - if this.IsOptional() { - start = '[' - end = ']' - } else { + if this.IsRequired() { start = '<' end = '>' + } else { + start = '[' + end = ']' } if this.IsPositional() { return fmt.Sprintf("%c%s%c", start, this.MetaVar(), end) @@ -450,8 +485,8 @@ func (this *SingleArgument) String() string { } } -func (this *SingleArgument) IsOptional() bool { - return this.optional +func (this *SingleArgument) IsRequired() bool { + return this.required } func (this *SingleArgument) IsPositional() bool { @@ -519,7 +554,7 @@ func (this *SingleArgument) SetDefault() { } func (this *SingleArgument) Validate() error { - if !this.optional && !this.isSet && !this.useDefault { + if this.required && !this.isSet && !this.useDefault { return fmt.Errorf("Non-optional argument %s not set", this.token) } return nil @@ -818,7 +853,7 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error } } } - if err == nil && pos_idx < len(this.posArgs) && !this.posArgs[pos_idx].IsOptional() { + if err == nil && pos_idx < len(this.posArgs) { err = fmt.Errorf("Not enough arguments, missing %s", this.posArgs[pos_idx]) } if err == nil { From cbc0c234f28fa943a257ba9311d2f37b8b77b06c Mon Sep 17 00:00:00 2001 From: Zexi Date: Fri, 17 Aug 2018 02:27:39 +0800 Subject: [PATCH 12/71] =?UTF-8?q?-=20climc:=20=E8=AE=BE=E7=BD=AE=20option?= =?UTF-8?q?=20=E6=B2=A1=E6=9C=89=20subcommand=20=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5=E4=B8=8B=E8=BF=9B=E5=85=A5=E4=BA=A4=E4=BA=92=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F,=20=E4=BB=A3=E7=A0=81=E9=80=BB=E8=BE=91=E5=8F=98?= =?UTF-8?q?=E6=B8=85=E6=99=B0=20-=20make=20fmt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Gopkg.lock | 2 +- cmd/climc/climc.go | 110 +++++++++++++--------- cmd/climc/shell/cloudproviders.go | 12 +-- cmd/scheduler/app/server.go | 2 +- pkg/cloudcommon/cronman/cronman.go | 21 ++--- pkg/cloudcommon/db/taskman/subtasks.go | 3 +- pkg/compute/models/disks.go | 6 +- pkg/compute/models/guests.go | 6 +- pkg/compute/options/options.go | 8 +- pkg/compute/service/service.go | 4 +- pkg/util/aliyun/region.go | 5 +- pkg/util/aliyun/shell/instance.go | 3 +- pkg/util/esxi/manager.go | 4 +- pkg/util/esxi/mobase.go | 9 +- pkg/util/seclib2/seclib.go | 6 +- vendor/yunion.io/x/structarg/errors.go | 13 +++ vendor/yunion.io/x/structarg/structarg.go | 2 +- 17 files changed, 130 insertions(+), 86 deletions(-) create mode 100644 vendor/yunion.io/x/structarg/errors.go diff --git a/Gopkg.lock b/Gopkg.lock index 8323a741ba..5b287504a8 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -902,7 +902,7 @@ branch = "master" name = "yunion.io/x/structarg" packages = ["."] - revision = "adf929ce0f8bd62335ebe57d9b51f023c84e4d7a" + revision = "d5e5d87357b9bc2164215117763f6b15b2a3e75d" [solve-meta] analyzer-name = "dep" diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index 217724a188..af582a1693 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -133,6 +133,40 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { return session, nil } +func enterInteractiveMode( + parser *structarg.ArgumentParser, + sessionFactory func() *mcclient.ClientSession, +) { + promputils.InitEnv(parser, sessionFactory()) + defer fmt.Println("Bye!") + p := prompt.New( + promputils.Executor, + promputils.Completer, + prompt.OptionPrefix("climc> "), + prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"), + prompt.OptionMaxSuggestion(16), + ) + p.Run() +} + +func executeSubcommand( + subcmd *structarg.SubcommandArgument, + subparser *structarg.ArgumentParser, + options *BaseOptions, + sessionFactory func() *mcclient.ClientSession, +) { + var e error + suboptions := subparser.Options() + if options.SUBCOMMAND == "help" { + e = subcmd.Invoke(suboptions) + } else { + e = subcmd.Invoke(sessionFactory(), suboptions) + } + if e != nil { + showErrorAndExit(e) + } +} + func main() { parser, e := getSubcommandsParser() if e != nil { @@ -143,47 +177,39 @@ func main() { if options.Help { fmt.Print(parser.HelpString()) - } else if options.Version { - fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString()) - } else if len(os.Args) <= 1 { - session, e := newClientSession(options) - if e != nil { - showErrorAndExit(e) - } - promputils.InitEnv(parser, session) - defer fmt.Println("Bye!") - p := prompt.New( - promputils.Executor, - promputils.Completer, - prompt.OptionPrefix("climc> "), - prompt.OptionTitle("Climc, a Command Line Interface to Manage Clouds"), - prompt.OptionMaxSuggestion(16), - ) - p.Run() - } else { - subcmd := parser.GetSubcommand() - subparser := subcmd.GetSubParser() - if e != nil { - if subparser != nil { - fmt.Print(subparser.Usage()) - } else { - fmt.Print(parser.Usage()) - } - showErrorAndExit(e) - } else { - session, e := newClientSession(options) - if e != nil { - showErrorAndExit(e) - } - suboptions := subparser.Options() - if options.SUBCOMMAND == "help" { - e = subcmd.Invoke(suboptions) - } else { - e = subcmd.Invoke(session, suboptions) - } - if e != nil { - showErrorAndExit(e) - } - } + return } + + if options.Version { + fmt.Printf("Yunion API client version:\n %s\n", version.GetJsonString()) + return + } + + ensureSessionFactory := func() *mcclient.ClientSession { + session, err := newClientSession(options) + if err != nil { + showErrorAndExit(err) + } + return session + } + + // enter interactive mode when not enough argument and SUBCOMMAND is empty + if _, ok := e.(*structarg.NotEnoughArgumentsError); ok && options.SUBCOMMAND == "" { + enterInteractiveMode(parser, ensureSessionFactory) + return + } + + subcmd := parser.GetSubcommand() + subparser := subcmd.GetSubParser() + if e != nil { + if subparser != nil { + fmt.Print(subparser.Usage()) + } else { + fmt.Print(parser.Usage()) + } + showErrorAndExit(e) + } + + // execute subcommand in non-interactive mode + executeSubcommand(subcmd, subparser, options, ensureSessionFactory) } diff --git a/cmd/climc/shell/cloudproviders.go b/cmd/climc/shell/cloudproviders.go index edfb638d90..36c986ddf2 100644 --- a/cmd/climc/shell/cloudproviders.go +++ b/cmd/climc/shell/cloudproviders.go @@ -140,12 +140,12 @@ func init() { }) type CloudproviderSyncOptions struct { - ID string `help:"ID or Name of cloud provider"` - Force bool `help:"Force sync no matter what"` - FullSync bool `help:"Synchronize everything"` - Region []string `help:"region to sync"` - Zone []string `help:"region to sync"` - Host []string `help:"region to sync"` + ID string `help:"ID or Name of cloud provider"` + Force bool `help:"Force sync no matter what"` + FullSync bool `help:"Synchronize everything"` + Region []string `help:"region to sync"` + Zone []string `help:"region to sync"` + Host []string `help:"region to sync"` } R(&CloudproviderSyncOptions{}, "cloud-provider-sync", "Sync of a cloud provider account", func(s *mcclient.ClientSession, args *CloudproviderSyncOptions) error { params := jsonutils.NewDict() diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go index 3dc1ce9a4f..e3a8f93be8 100644 --- a/cmd/scheduler/app/server.go +++ b/cmd/scheduler/app/server.go @@ -8,10 +8,10 @@ import ( "net/http" "strconv" + "gopkg.in/gin-gonic/gin.v1" "yunion.io/x/log" "yunion.io/x/pkg/util/prometheus" "yunion.io/x/pkg/utils" - "gopkg.in/gin-gonic/gin.v1" o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/onecloud/pkg/mcclient/auth" diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index b2e443d983..d01cc096d5 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -1,33 +1,33 @@ package cronman import ( - "time" - "runtime/debug" "context" + "reflect" + "runtime" + "runtime/debug" + "time" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" - "reflect" - "runtime" ) const ( - DEFAULT_CRON_INTERVAL = 60*time.Second // default resolution is 1 monutes + DEFAULT_CRON_INTERVAL = 60 * time.Second // default resolution is 1 monutes ) type SCronJobManager struct { checkInterval time.Duration - timer *time.Timer - jobs []SCronJob + timer *time.Timer + jobs []SCronJob } type SCronJob struct { - name string + name string runInterval time.Duration - job func(ctx context.Context, userCred mcclient.TokenCredential) - lastRun time.Time + job func(ctx context.Context, userCred mcclient.TokenCredential) + lastRun time.Time } func NewCronJobManager(interval time.Duration) *SCronJobManager { @@ -92,4 +92,3 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr userCred := auth.AdminCredential() job(ctx, userCred) } - diff --git a/pkg/cloudcommon/db/taskman/subtasks.go b/pkg/cloudcommon/db/taskman/subtasks.go index cf82841cc8..8ad2c2902e 100644 --- a/pkg/cloudcommon/db/taskman/subtasks.go +++ b/pkg/cloudcommon/db/taskman/subtasks.go @@ -1,11 +1,12 @@ package taskman import ( + "database/sql" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" - "database/sql" ) const ( diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 4d0b190436..047863499a 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -634,8 +634,8 @@ func (self *SDisk) PerformCancelDelete(ctx context.Context, userCred mcclient.To return nil, nil } -func (manager *SDiskManager) getExpiredPendingDeleteDisks() ([]SDisk) { - deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds)*time.Second) +func (manager *SDiskManager) getExpiredPendingDeleteDisks() []SDisk { + deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds) * time.Second) q := manager.Query() q = q.IsTrue("pending_deleted").LT("pending_deleted_at", deadline).Limit(options.Options.PendingDeleteMaxCleanBatchSize) @@ -658,4 +658,4 @@ func (manager *SDiskManager) CleanPendingDeleteDisks(ctx context.Context, userCr for i := 0; i < len(disks); i += 1 { disks[i].StartDiskDeleteTask(ctx, userCred, "", false) } -} \ No newline at end of file +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8c93ed56fe..401bdbd1b0 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2661,8 +2661,8 @@ func (manager *SGuestManager) getIpsByExit(ips []string, isExitOnly bool) []stri return extRet } -func (manager *SGuestManager) getExpiredPendingDeleteGuests() ([]SGuest) { - deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds)*time.Second) +func (manager *SGuestManager) getExpiredPendingDeleteGuests() []SGuest { + deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds) * time.Second) q := manager.Query() q = q.IsTrue("pending_deleted").LT("pending_deleted_at", deadline).In("hypervisor", []string{"aliyun"}).Limit(options.Options.PendingDeleteMaxCleanBatchSize) @@ -2685,4 +2685,4 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use for i := 0; i < len(guests); i += 1 { guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true) } -} \ No newline at end of file +} diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index c873819ebd..a018a3e2f1 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -21,10 +21,10 @@ type ComputeOptions struct { DefaultDiskSize int `default:"30720" help:"Default disk size in MB if not specified, default to 30GiB"` - EnablePendingDelete bool `default:"true" help:"Turn on/off pending delete VM and disk, default is on"` - PendingDeleteCheckSeconds int `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"` - PendingDeleteExpireSeconds int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"` - PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"` + EnablePendingDelete bool `default:"true" help:"Turn on/off pending delete VM and disk, default is on"` + PendingDeleteCheckSeconds int `default:"3600" help:"How long to wait to scan pending delete VM or disks, default is 1 hour"` + PendingDeleteExpireSeconds int `default:"259200" help:"How long a pending delete VM/disks cleaned automatically, default 3 days"` + PendingDeleteMaxCleanBatchSize int `default:"50" help:"How many pending delete servers can be clean in a batch"` ImageCacheStoragePolicy string `default:"least_used" choices:"best_fit|least_used" help:"Policy to choose storage for image cache, best_fit or least_used"` MetricsRetentionDays int32 `default:"30" help:"Retention days for monitoring metrics in influxdb"` diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index c26b0bdab2..227874f9d6 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -9,14 +9,14 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/cloudcommon/cronman" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/compute" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/onecloud/pkg/cloudcommon/cronman" - _ "yunion.io/x/onecloud/pkg/compute/tasks" _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" + _ "yunion.io/x/onecloud/pkg/compute/tasks" _ "yunion.io/x/onecloud/pkg/util/aliyun/provider" _ "yunion.io/x/onecloud/pkg/util/esxi/provider" ) diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index c9a8954b67..cdea189749 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -9,9 +9,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type SRegion struct { @@ -563,4 +564,4 @@ func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname func (self *SRegion) UpdateInstancePassword(instId string, passwd string) error { return self.updateInstance(instId, "", "", passwd, "") -} \ No newline at end of file +} diff --git a/pkg/util/aliyun/shell/instance.go b/pkg/util/aliyun/shell/instance.go index 9c86e47aa8..495e2949cf 100644 --- a/pkg/util/aliyun/shell/instance.go +++ b/pkg/util/aliyun/shell/instance.go @@ -2,6 +2,7 @@ package shell import ( "fmt" + "yunion.io/x/onecloud/pkg/util/aliyun" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -82,7 +83,7 @@ func init() { }) type InstanceUpdatePasswordOptions struct { - ID string `help:"Instance ID"` + ID string `help:"Instance ID"` PASSWD string `help:"new password"` } shellutils.R(&InstanceUpdatePasswordOptions{}, "instance-update-password", "Update instance password", func(cli *aliyun.SRegion, args *InstanceUpdatePasswordOptions) error { diff --git a/pkg/util/esxi/manager.go b/pkg/util/esxi/manager.go index 0c0698bfcd..ed1dd5d76d 100644 --- a/pkg/util/esxi/manager.go +++ b/pkg/util/esxi/manager.go @@ -5,10 +5,12 @@ import ( "fmt" "net/url" "reflect" + "strings" "github.com/vmware/govmomi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/view" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" @@ -16,10 +18,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "github.com/vmware/govmomi/session" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "strings" ) const ( diff --git a/pkg/util/esxi/mobase.go b/pkg/util/esxi/mobase.go index 23f6aa9c61..c60498ebc5 100644 --- a/pkg/util/esxi/mobase.go +++ b/pkg/util/esxi/mobase.go @@ -1,10 +1,13 @@ package esxi import ( - "github.com/vmware/govmomi/vim25/mo" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "reflect" + + "github.com/vmware/govmomi/vim25/mo" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SManagedObject struct { diff --git a/pkg/util/seclib2/seclib.go b/pkg/util/seclib2/seclib.go index 86cee26422..d3fd831df3 100644 --- a/pkg/util/seclib2/seclib.go +++ b/pkg/util/seclib2/seclib.go @@ -1,10 +1,10 @@ package seclib2 import ( - "fmt" - "strings" "bytes" + "fmt" "math/rand" + "strings" ) const ( @@ -39,7 +39,7 @@ func RandomPassword2(width int) string { } buf.WriteByte(ch) } - if digitsCnt > 1 && letterCnt > 1 && upperCnt > 1 && puncCnt >=1 && puncCnt <= 2 { + if digitsCnt > 1 && letterCnt > 1 && upperCnt > 1 && puncCnt >= 1 && puncCnt <= 2 { return buf.String() } } diff --git a/vendor/yunion.io/x/structarg/errors.go b/vendor/yunion.io/x/structarg/errors.go new file mode 100644 index 0000000000..3aeeac0b81 --- /dev/null +++ b/vendor/yunion.io/x/structarg/errors.go @@ -0,0 +1,13 @@ +package structarg + +import ( + "fmt" +) + +type NotEnoughArgumentsError struct { + argument Argument +} + +func (e *NotEnoughArgumentsError) Error() string { + return fmt.Sprintf("Not enough arguments, missing %s", e.argument) +} diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 7e4c9193d5..9eb223f272 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -854,7 +854,7 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error } } if err == nil && pos_idx < len(this.posArgs) { - err = fmt.Errorf("Not enough arguments, missing %s", this.posArgs[pos_idx]) + err = &NotEnoughArgumentsError{argument: this.posArgs[pos_idx]} } if err == nil { err = this.Validate() From 8e8a636d239f4ae471dfea70cf4a5cf5bfb3f653 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Fri, 17 Aug 2018 12:30:33 +0800 Subject: [PATCH 13/71] climc: secure options rename to insecure --- cmd/climc/climc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index af582a1693..175543ccc8 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -21,7 +21,7 @@ type BaseOptions struct { Debug bool `help:"Show debug information"` Version bool `help:"Show version"` Timeout int `default:"600" help:"Number of seconds to wait for a response"` - Secure bool `default:"False" help:"do server cert verification if URL is https"` + Insecure bool `default:"False" help:"Allow skip server cert verification if URL is https" short-token:"k"` OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` // OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"` @@ -116,7 +116,7 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { client := mcclient.NewClient(options.OsAuthURL, options.Timeout, options.Debug, - options.Secure) + options.Insecure) token, err := client.Authenticate(options.OsUsername, options.OsPassword, options.OsDomainName, From c42db3568b0ee0c2e18e11549a028d8125a2969b Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 17 Aug 2018 09:41:04 +0000 Subject: [PATCH 14/71] vendor: dep ensure to update sqlchemy e2ab646 Fix insertion for auto_inc fields --- Gopkg.lock | 2 +- vendor/yunion.io/x/sqlchemy/insert.go | 35 ++++++++++++++++----------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 5b287504a8..85f7166e2f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -896,7 +896,7 @@ branch = "master" name = "yunion.io/x/sqlchemy" packages = ["."] - revision = "8d8043b1fc6f8a9192f1d086520bb167a8423475" + revision = "a74ef73e555a1ed2e19b1a47e0567814c289eaf9" [[projects]] branch = "master" diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index 06e3518d81..d9f10406c1 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -13,15 +13,7 @@ func (t *STableSpec) Insert(dt interface{}) error { return t.insert(dt, false) } -func (t *STableSpec) insert(dt interface{}, debug bool) error { - beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeInsert") - if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { - beforeInsertFunc.Call([]reflect.Value{}) - } - - // dataType := reflect.TypeOf(dt).Elem() - dataValue := reflect.ValueOf(dt).Elem() - +func (t *STableSpec) insertSqlPrep(dataFields map[string]interface{}) (string, []interface{}, error) { var autoIncField string createdAtFields := make([]string, 0) @@ -29,7 +21,6 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { format := make([]string, 0) values := make([]interface{}, 0) - fields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) for _, c := range t.columns { isAutoInc := false nc, ok := c.(*SIntegerColumn) @@ -40,7 +31,7 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { k := c.Name() dtc, ok := c.(*SDateTimeColumn) - ov := fields[k] + ov := dataFields[k] if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) @@ -57,8 +48,9 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { panic(fmt.Sprintf("multiple auto_increment columns: %q, %q", autoIncField, k)) } autoIncField = k + } else { + return "", nil, fmt.Errorf("cannot insert for null primary key %q", k) } - return fmt.Errorf("cannot insert for null primary key %q", k) } else if !c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value val := c.ConvertFromString(c.Default()) values = append(values, val) @@ -71,6 +63,21 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { t.name, strings.Join(names, ", "), strings.Join(format, ", ")) + return insertSql, values, nil +} + +func (t *STableSpec) insert(data interface{}, debug bool) error { + beforeInsertFunc := reflect.ValueOf(data).MethodByName("BeforeInsert") + if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { + beforeInsertFunc.Call([]reflect.Value{}) + } + + dataValue := reflect.ValueOf(data).Elem() + dataFields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) + insertSql, values, err := t.insertSqlPrep(dataFields) + if err != nil { + return err + } if DEBUG_SQLCHEMY || debug { log.Debugf("%s values: %s", insertSql, values) @@ -115,11 +122,11 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { q = q.Equals(c.Name(), lastId) } } else { - q = q.Equals(c.Name(), fields[c.Name()]) + q = q.Equals(c.Name(), dataFields[c.Name()]) } } } - err = q.First(dt) + err = q.First(data) if err != nil { err := fmt.Errorf("query after insert failed: %v", err) return err From d654c0f1d7d75a35e78f9a13f7b4f220a9a56ce1 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 13 Aug 2018 19:22:34 +0800 Subject: [PATCH 15/71] climc: k8s resource cli --- cmd/climc/shell/k8s/configmap.go | 27 +++++ cmd/climc/shell/k8s/deployment.go | 145 +++++++++++++++++++++++++ cmd/climc/shell/k8s/k8s.go | 67 +++++++++++- cmd/climc/shell/k8s/pods.go | 20 +++- cmd/climc/shell/k8s/raw.go | 35 ++++++ cmd/climc/shell/k8s/repos.go | 106 ++++++++++++++++++ cmd/climc/shell/k8s/service.go | 27 +++++ pkg/mcclient/modules/k8s/base.go | 47 +++++++- pkg/mcclient/modules/k8s/cluster.go | 4 +- pkg/mcclient/modules/k8s/configmap.go | 20 ++++ pkg/mcclient/modules/k8s/deployment.go | 20 ++++ pkg/mcclient/modules/k8s/event.go | 4 +- pkg/mcclient/modules/k8s/helm.go | 17 +++ pkg/mcclient/modules/k8s/node.go | 2 +- pkg/mcclient/modules/k8s/pod.go | 4 +- pkg/mcclient/modules/k8s/raw.go | 95 ++++++++++++++++ pkg/mcclient/modules/k8s/service.go | 20 ++++ 17 files changed, 646 insertions(+), 14 deletions(-) create mode 100644 cmd/climc/shell/k8s/configmap.go create mode 100644 cmd/climc/shell/k8s/deployment.go create mode 100644 cmd/climc/shell/k8s/raw.go create mode 100644 cmd/climc/shell/k8s/repos.go create mode 100644 cmd/climc/shell/k8s/service.go create mode 100644 pkg/mcclient/modules/k8s/configmap.go create mode 100644 pkg/mcclient/modules/k8s/deployment.go create mode 100644 pkg/mcclient/modules/k8s/helm.go create mode 100644 pkg/mcclient/modules/k8s/raw.go create mode 100644 pkg/mcclient/modules/k8s/service.go diff --git a/cmd/climc/shell/k8s/configmap.go b/cmd/climc/shell/k8s/configmap.go new file mode 100644 index 0000000000..30d8fd6d9e --- /dev/null +++ b/cmd/climc/shell/k8s/configmap.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initConfigMap() { + cmdN := func(suffix string) string { + return resourceCmdN("configmap", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s configmap", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.ConfigMaps.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.ConfigMaps.GetColumns(s)) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/deployment.go b/cmd/climc/shell/k8s/deployment.go new file mode 100644 index 0000000000..dfdc392bba --- /dev/null +++ b/cmd/climc/shell/k8s/deployment.go @@ -0,0 +1,145 @@ +package k8s + +import ( + "fmt" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initDeployment() { + cmdN := func(suffix string) string { + return resourceCmdN("deployment", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s deployment", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Deployments.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Deployments.GetColumns(s)) + return nil + }) + + type createOpt struct { + namespaceOptions + NAME string `help:"Name of deployment"` + Image string `help:"The image for the container to run"` + Replicas int64 `help:"Number of replicas for pods in this deployment"` + RunAsPrivileged bool `help:"Whether to run the container as privileged user"` + Labels string `help:"Comma separated labels to apply to the pod(s)"` + Env []string `help:"Environment variables to set in container"` + Port []string `help:"Port for the service that is created, format is :: e.g. tcp:80:3000"` + } + R(&createOpt{}, cmdN("create"), "Create deployment resource", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + if len(args.Image) == 0 { + return fmt.Errorf("Image must provided") + } + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.Image), "containerImage") + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + if args.Replicas > 1 { + params.Add(jsonutils.NewInt(args.Replicas), "replicas") + } + if args.RunAsPrivileged { + params.Add(jsonutils.JSONTrue, "runAsPrivileged") + } + if len(args.Port) != 0 { + portMappings, err := parsePortMappings(args.Port) + if err != nil { + return err + } + params.Add(portMappings, "portMappings") + } + ret, err := k8s.Deployments.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type getOpt struct { + resourceGetOptions + } + R(&getOpt{}, cmdN("show"), "Get deployment details", func(s *mcclient.ClientSession, args *getOpt) error { + id := args.NAME + params := jsonutils.NewDict() + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + ret, err := k8s.Deployments.GetInContexts(s, id, params, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(ret) + return nil + }) +} + +type portMapping struct { + Port int32 `json:"port"` + TargetPort int32 `json:"targetPort"` + Protocol string `json:"protocol"` +} + +func parsePortMapping(port string) (*portMapping, error) { + if len(port) == 0 { + return nil, fmt.Errorf("empty port mapping desc string") + } + parts := strings.Split(port, ":") + mapping := &portMapping{} + for _, part := range parts { + if sets.NewString("tcp", "udp").Has(strings.ToLower(part)) { + mapping.Protocol = strings.ToUpper(part) + } + if port, err := strconv.Atoi(part); err != nil { + continue + } else { + if mapping.Port == 0 { + mapping.Port = int32(port) + } else { + mapping.TargetPort = int32(port) + } + } + } + if mapping.Protocol == "" { + mapping.Protocol = "TCP" + } + if mapping.Port <= 0 { + return nil, fmt.Errorf("Service port not provided") + } + if mapping.TargetPort < 0 { + return nil, fmt.Errorf("Container invalid targetPort %d", mapping.TargetPort) + } + if mapping.TargetPort == 0 { + mapping.TargetPort = mapping.Port + } + return mapping, nil +} + +func parsePortMappings(ports []string) (*jsonutils.JSONArray, error) { + ret := jsonutils.NewArray() + for _, port := range ports { + mapping, err := parsePortMapping(port) + if err != nil { + return nil, fmt.Errorf("Port %q error: %v", port, err) + } + ret.Add(jsonutils.Marshal(mapping)) + } + return ret, nil +} diff --git a/cmd/climc/shell/k8s/k8s.go b/cmd/climc/shell/k8s/k8s.go index 2e63757294..af4892e71a 100644 --- a/cmd/climc/shell/k8s/k8s.go +++ b/cmd/climc/shell/k8s/k8s.go @@ -12,24 +12,81 @@ import ( ) func init() { + // cluster resources initCluster() initNode() + + // helm resources + initRepo() + + // kubernetes original resources + initRaw() + initConfigMap() + initDeployment() initPod() + initService() } type BaseListOptions shell.BaseListOptions type clusterBaseOptions struct { - Cluster string `default:"$K8S_CLUSTER" help:"Kubernetes cluster name"` + Cluster string `default:"$K8S_CLUSTER|default" help:"Kubernetes cluster name"` } func (o clusterBaseOptions) ClusterContext() []modules.ManagerContext { return []modules.ManagerContext{clusterContext(o.Cluster)} } -type k8sBaseListOptions struct { +type baseListOptions struct { + Limit int `default:"20" help:"Page limit"` + Offset int `default:"0" help:"page offset"` +} + +func fetchPagingParams(opt baseListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.Limit > 0 { + params.Add(jsonutils.NewInt(int64(opt.Limit)), "limit") + } + if opt.Offset > 0 { + params.Add(jsonutils.NewInt(int64(opt.Offset)), "offset") + } + return params +} + +type namespaceListOptions struct { + namespaceOptions + AllNamespace bool `help:"Show resource in all namespace"` +} + +type namespaceOptions struct { clusterBaseOptions - Limit int `default:"20" help:"Page limit"` + Namespace string `help:"Namespace of this resource"` +} + +type resourceGetOptions struct { + clusterBaseOptions + Namespace string `help:"Namespace of this resource"` + NAME string `help:"Name ident of the resource"` +} + +func (o resourceGetOptions) ToJSON() *jsonutils.JSONDict { + params := jsonutils.NewDict() + if o.Namespace != "" { + params.Add(jsonutils.NewString(o.Namespace), "namespace") + } + return params +} + +func fetchNamespaceParams(opt namespaceListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.AllNamespace { + params.Add(jsonutils.JSONTrue, "all_namespace") + return params + } + if opt.Namespace != "" { + params.Add(jsonutils.NewString(opt.Namespace), "namespace") + } + return params } var ( @@ -53,3 +110,7 @@ func clusterContext(clusterId string) modules.ManagerContext { InstanceId: clusterId, } } + +func printObjectYAML(obj jsonutils.JSONObject) { + fmt.Println(obj.YAMLString()) +} diff --git a/cmd/climc/shell/k8s/pods.go b/cmd/climc/shell/k8s/pods.go index 8d2fdd0d3b..ee8803c414 100644 --- a/cmd/climc/shell/k8s/pods.go +++ b/cmd/climc/shell/k8s/pods.go @@ -13,10 +13,26 @@ func initPod() { } type listOpt struct { - k8sBaseListOptions + namespaceListOptions + baseListOptions } R(&listOpt{}, cmdN("list"), "List k8s pod", func(s *mcclient.ClientSession, args *listOpt) error { - ret, err := k8s.Pods.ListInContexts(s, nil, args.ClusterContext()) + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Pods.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Pods.GetColumns(s)) + return nil + }) + + type deleteOpt struct { + resourceGetOptions + } + R(&deleteOpt{}, cmdN("delete"), "Delete pod", func(s *mcclient.ClientSession, args *deleteOpt) error { + id := args.NAME + ret, err := k8s.Pods.DeleteInContexts(s, id, args.ToJSON(), args.ClusterContext()) if err != nil { return err } diff --git a/cmd/climc/shell/k8s/raw.go b/cmd/climc/shell/k8s/raw.go new file mode 100644 index 0000000000..6aeb500f91 --- /dev/null +++ b/cmd/climc/shell/k8s/raw.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +type rawGetOpt struct { + namespaceOptions + KIND string `help:"resource kind"` + NAME string `help:"instance name"` +} + +type rawDeleteOpt struct { + rawGetOpt +} + +func initRaw() { + R(&rawGetOpt{}, "k8s-get", "Get k8s resource instance raw info", func(s *mcclient.ClientSession, args *rawGetOpt) error { + obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(obj) + return nil + }) + + R(&rawDeleteOpt{}, "k8s-delete", "Delete k8s resource instance", func(s *mcclient.ClientSession, args *rawDeleteOpt) error { + err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + return nil + }) +} diff --git a/cmd/climc/shell/k8s/repos.go b/cmd/climc/shell/k8s/repos.go new file mode 100644 index 0000000000..ee34d8a2f9 --- /dev/null +++ b/cmd/climc/shell/k8s/repos.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initRepo() { + cmdN := func(suffix string) string { + return resourceCmdN("repo", suffix) + } + type listOpt struct { + BaseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s global helm repos", func(s *mcclient.ClientSession, args *listOpt) error { + params := FetchPagingParams(args.BaseListOptions) + result, err := k8s.Repos.List(s, params) + if err != nil { + return err + } + printList(result, k8s.Repos.GetColumns(s)) + return nil + }) + + type getOpt struct { + NAME string `help:"ID or name of the repo"` + } + R(&getOpt{}, cmdN("show"), "Show details fo a repo", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Get(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type createOpt struct { + getOpt + URL string `help:"Repository url"` + Public bool `help:"Make repository public"` + } + R(&createOpt{}, cmdN("create"), "Add repository", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.URL), "url") + if args.Public { + params.Add(jsonutils.JSONTrue, "is_public") + } + repo, err := k8s.Repos.Create(s, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type updateOpt struct { + getOpt + Name string `help:"Repository name to change"` + Url string `help:"Repository url to change"` + } + R(&updateOpt{}, cmdN("update"), "Update helm repository", func(s *mcclient.ClientSession, args *updateOpt) error { + params := jsonutils.NewDict() + if args.Name != "" { + params.Add(jsonutils.NewString(args.Name), "name") + } + if args.Url != "" { + params.Add(jsonutils.NewString(args.Url), "url") + } + repo, err := k8s.Repos.Update(s, args.NAME, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("private"), "Make repository private", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "private", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("public"), "Make repository public", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "public", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("delete"), "Delete a repository", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Delete(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/service.go b/cmd/climc/shell/k8s/service.go new file mode 100644 index 0000000000..3260b28b88 --- /dev/null +++ b/cmd/climc/shell/k8s/service.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initService() { + cmdN := func(suffix string) string { + return resourceCmdN("service", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s service", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Services.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Services.GetColumns(s)) + return nil + }) +} diff --git a/pkg/mcclient/modules/k8s/base.go b/pkg/mcclient/modules/k8s/base.go index 6c5eef1847..3b5fc20b61 100644 --- a/pkg/mcclient/modules/k8s/base.go +++ b/pkg/mcclient/modules/k8s/base.go @@ -1,13 +1,56 @@ package k8s import ( + "yunion.io/x/pkg/util/sets" + "yunion.io/x/onecloud/pkg/mcclient/modules" ) -func NewManager(keyword, keywordPlural string, columns, adminColumns []string) *modules.ResourceManager { +func NewManager(keyword, keywordPlural string, columns, adminColumns *Columns) *modules.ResourceManager { return &modules.ResourceManager{ - BaseManager: *modules.NewBaseManager("k8s", "", "", columns, adminColumns), + BaseManager: *modules.NewBaseManager("k8s", "", "", columns.Array(), adminColumns.Array()), Keyword: keyword, KeywordPlural: keywordPlural, } } + +type Columns struct { + cols []string +} + +func NewColumns(cols ...string) *Columns { + c := &Columns{cols: make([]string, 0)} + return c.Add(cols...) +} + +func (c *Columns) Add(cols ...string) *Columns { + for _, col := range cols { + c.add(col) + } + return c +} + +func (c *Columns) add(col string) *Columns { + src := sets.NewString(c.Array()...) + if src.Has(col) { + return c + } + c.cols = append(c.cols, col) + return c +} + +func (c Columns) Array() []string { + return c.cols +} + +func NewNamespaceCols(col ...string) *Columns { + return NewColumns("name", "namespace").Add(col...) +} + +func NewClusterCols(col ...string) *Columns { + return NewColumns("cluster").Add(col...) +} + +func NewResourceCols(col ...string) *Columns { + return NewColumns("name", "id").Add(col...) +} diff --git a/pkg/mcclient/modules/k8s/cluster.go b/pkg/mcclient/modules/k8s/cluster.go index df2df5ca39..42ee41d6b1 100644 --- a/pkg/mcclient/modules/k8s/cluster.go +++ b/pkg/mcclient/modules/k8s/cluster.go @@ -10,7 +10,7 @@ var ( func init() { Clusters = NewManager("cluster", "clusters", - []string{"id", "name", "mode", "k8s_version", "status", "api_endpoint"}, - []string{}) + NewResourceCols("mode", "k8s_version", "status", "api_endpoint"), + NewColumns()) modules.Register(Clusters) } diff --git a/pkg/mcclient/modules/k8s/configmap.go b/pkg/mcclient/modules/k8s/configmap.go new file mode 100644 index 0000000000..4bfcbbafdf --- /dev/null +++ b/pkg/mcclient/modules/k8s/configmap.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ConfigMaps *ConfigMapManager + +type ConfigMapManager struct { + modules.ResourceManager +} + +func init() { + ConfigMaps = &ConfigMapManager{ + ResourceManager: *NewManager( + "configmap", "configmaps", + NewNamespaceCols(), + NewClusterCols())} + modules.Register(ConfigMaps) +} diff --git a/pkg/mcclient/modules/k8s/deployment.go b/pkg/mcclient/modules/k8s/deployment.go new file mode 100644 index 0000000000..f56a2947ef --- /dev/null +++ b/pkg/mcclient/modules/k8s/deployment.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Deployments *DeploymentManager + +type DeploymentManager struct { + modules.ResourceManager +} + +func init() { + Deployments = &DeploymentManager{ + ResourceManager: *NewManager( + "deployment", "deployments", + NewNamespaceCols("labels"), + NewClusterCols())} + modules.Register(Deployments) +} diff --git a/pkg/mcclient/modules/k8s/event.go b/pkg/mcclient/modules/k8s/event.go index f71c5541ef..584ddbc376 100644 --- a/pkg/mcclient/modules/k8s/event.go +++ b/pkg/mcclient/modules/k8s/event.go @@ -10,7 +10,7 @@ var ( func init() { Logs = NewManager("kube_event", "kube_events", - []string{"id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"}, - []string{}) + NewColumns("id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"), + NewColumns()) modules.Register(Logs) } diff --git a/pkg/mcclient/modules/k8s/helm.go b/pkg/mcclient/modules/k8s/helm.go new file mode 100644 index 0000000000..b3e04f21ea --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm.go @@ -0,0 +1,17 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Repos *modules.ResourceManager +) + +func init() { + Repos = NewManager("repo", "repos", + NewResourceCols("url", "is_public", "source"), + NewColumns(), + ) + modules.Register(Repos) +} diff --git a/pkg/mcclient/modules/k8s/node.go b/pkg/mcclient/modules/k8s/node.go index 5b8cb0798b..60f5bd6ade 100644 --- a/pkg/mcclient/modules/k8s/node.go +++ b/pkg/mcclient/modules/k8s/node.go @@ -9,6 +9,6 @@ var ( ) func init() { - Nodes = NewManager("node", "nodes", []string{"id", "name", "cluster", "roles", "address", "status"}, []string{}) + Nodes = NewManager("node", "nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns()) modules.Register(Nodes) } diff --git a/pkg/mcclient/modules/k8s/pod.go b/pkg/mcclient/modules/k8s/pod.go index 46417df838..218ef6a243 100644 --- a/pkg/mcclient/modules/k8s/pod.go +++ b/pkg/mcclient/modules/k8s/pod.go @@ -14,7 +14,7 @@ func init() { Pods = &PodManager{ ResourceManager: *NewManager( "pod", "pods", - []string{"id", "name", "cluster", "roles", "address", "status"}, - []string{})} + NewNamespaceCols("podIP", "status", "restartCount", "labels"), + NewClusterCols("nodeName"))} modules.Register(Pods) } diff --git a/pkg/mcclient/modules/k8s/raw.go b/pkg/mcclient/modules/k8s/raw.go new file mode 100644 index 0000000000..8853a18742 --- /dev/null +++ b/pkg/mcclient/modules/k8s/raw.go @@ -0,0 +1,95 @@ +package k8s + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var RawResource *RawResourceManager + +func init() { + RawResource = &RawResourceManager{ + serviceType: "k8s", + } +} + +type rawResourceContext struct { + kind string + name string + query jsonutils.JSONObject + ctxs []modules.ManagerContext +} + +func newRawResourceContext(kind, namespace, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) *rawResourceContext { + nsQuery := getNamespaceQuery(namespace) + if query != nil { + nsQuery.Update(query) + } + ctx := &rawResourceContext{kind: kind, name: name, query: nsQuery, ctxs: ctxs} + return ctx +} + +func (ctx rawResourceContext) contextPath() string { + segs := make([]string, 0) + ctxs := ctx.ctxs + if ctxs != nil && len(ctxs) > 0 { + for _, c := range ctxs { + segs = append(segs, c.InstanceManager.KeyString()) + if len(c.InstanceId) > 0 { + segs = append(segs, url.PathEscape(c.InstanceId)) + } + } + } + segs = append(segs, "_raw", ctx.kind, ctx.name) + path := fmt.Sprintf("/%s", strings.Join(segs, "/")) + if ctx.query != nil { + qs := ctx.query.QueryString() + if len(qs) > 0 { + path = fmt.Sprintf("%s?%s", path, qs) + } + } + return path +} + +type RawResourceManager struct { + serviceType string +} + +func (m *RawResourceManager) request(s *mcclient.ClientSession, method string, path string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + _, ret, err := s.JSONRequest(m.serviceType, "", method, path, nil, body) + return ret, err +} + +func getNamespaceQuery(namespace string) *jsonutils.JSONDict { + query := jsonutils.NewDict() + if namespace != "" { + query.Set("namespace", jsonutils.NewString(namespace)) + } + return query +} + +func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) (jsonutils.JSONObject, error) { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + return m.request(s, "GET", ctx.contextPath(), nil) +} + +func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + rawBytes := body.String() + newBody := jsonutils.NewDict() + newBody.Add(jsonutils.NewString(rawBytes), "raw") + ctx := newRawResourceContext(kind, namespace, name, nil, ctxs) + _, err := m.request(s, "PUT", ctx.contextPath(), newBody) + return err +} + +func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + _, err := m.request(s, "DELETE", ctx.contextPath(), nil) + return err +} diff --git a/pkg/mcclient/modules/k8s/service.go b/pkg/mcclient/modules/k8s/service.go new file mode 100644 index 0000000000..e66f38dda3 --- /dev/null +++ b/pkg/mcclient/modules/k8s/service.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Services *ServiceManager + +type ServiceManager struct { + modules.ResourceManager +} + +func init() { + Services = &ServiceManager{ + ResourceManager: *NewManager( + "service", "services", + NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"), + NewClusterCols())} + modules.Register(Services) +} From cca134d32e7320e1edf0a9116b30a5cc69b50828 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Mon, 20 Aug 2018 10:39:01 +0800 Subject: [PATCH 16/71] dep ensure pkg, fix climc parse input args --- Gopkg.lock | 2 +- cmd/climc/promputils/executor.go | 5 +- vendor/yunion.io/x/pkg/utils/utils.go | 68 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 85f7166e2f..8b923166b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -890,7 +890,7 @@ "util/workqueue", "utils" ] - revision = "30cdcc2f0c2ca0f3eef573c27eb7350bdd95a7dd" + revision = "56b426f0ca15288cc26442f5802c949f137ade60" [[projects]] branch = "master" diff --git a/cmd/climc/promputils/executor.go b/cmd/climc/promputils/executor.go index 7ef8e7dc89..06158ec9cd 100644 --- a/cmd/climc/promputils/executor.go +++ b/cmd/climc/promputils/executor.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "yunion.io/x/pkg/utils" "yunion.io/x/structarg" "yunion.io/x/onecloud/pkg/mcclient" @@ -51,8 +52,8 @@ func Executor(s string) { return } s = escaper(s) - args := strings.Split(s, " ") - e := parser.ParseArgs(strings.Split(s, " "), false) + args := utils.ArgsStringToArray(s) + e := parser.ParseArgs(args, false) subcmd := parser.GetSubcommand() subparser := subcmd.GetSubParser() if e != nil { diff --git a/vendor/yunion.io/x/pkg/utils/utils.go b/vendor/yunion.io/x/pkg/utils/utils.go index 4f61b411f6..0564c4b7fc 100644 --- a/vendor/yunion.io/x/pkg/utils/utils.go +++ b/vendor/yunion.io/x/pkg/utils/utils.go @@ -149,3 +149,71 @@ func FloatRound(num float64, precision int) float64 { num = temp return num } + +func getStringInQuote(s string, start, length int, quote byte, backIndex int) (string, int) { + var i = start + 1 + if quote != '\'' && quote != '"' { + for i < length { + if s[i] == ' ' || s[i] == '\'' || s[i] == '"' { + return s[backIndex:i], i + } + i++ + } + return s[backIndex:], i + } else { + for i < length { + if s[i] == quote { + return s[backIndex+1 : i], i + 1 + } + i++ + } + return s[backIndex:], i + } +} + +func ArgsStringToArray(s string) []string { + var args, i, j = make([]string, 0), 0, 0 + var length = len(s) + for i < length { + switch s[i] { + case ' ': + if i > j { + args = append(args, s[j:i]) + j = i + } + i++ + j = i + case '"': + var s1, s2 string + var oldStr = s[j:i] + s1, i = getStringInQuote(s, i, length, '"', i) + s1 = oldStr + s1 + for i < length && s[i] != ' ' { + s2, i = getStringInQuote(s, i, length, s[i], i) + s1 += s2 + fmt.Println(s2) + } + args = append(args, s1) + i++ + j = i + case '\'': + var s1, s2 string + var oldStr = s[j:i] + s1, i = getStringInQuote(s, i, length, '\'', i) + s1 = oldStr + s1 + for i < length && s[i] != ' ' { + s2, i = getStringInQuote(s, i, length, s[i], i) + s1 += s2 + } + args = append(args, s1) + i++ + j = i + default: + i++ + } + } + if j < length { + args = append(args, s[j:]) + } + return args +} From e3e341561c565a8596f9aeb4312db04a786c692b Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Mon, 20 Aug 2018 10:58:28 +0800 Subject: [PATCH 17/71] reserve args --- cmd/climc/promputils/executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/climc/promputils/executor.go b/cmd/climc/promputils/executor.go index 06158ec9cd..a3f579dfea 100644 --- a/cmd/climc/promputils/executor.go +++ b/cmd/climc/promputils/executor.go @@ -53,7 +53,7 @@ func Executor(s string) { } s = escaper(s) args := utils.ArgsStringToArray(s) - e := parser.ParseArgs(args, false) + e := parser.ParseArgs(utils.ArgsStringToArray(s), false) subcmd := parser.GetSubcommand() subparser := subcmd.GetSubParser() if e != nil { From 8feaa3538bd0dc9c1482cab4944b208e2e4c0dd9 Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 20 Aug 2018 11:02:37 +0800 Subject: [PATCH 18/71] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=85=8D=E7=BD=AE=E7=AD=89API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/db_dispatcher.go | 1 - pkg/cloudprovider/resources.go | 8 +- pkg/compute/guestdrivers/aliyun.go | 298 ++++++++++++++----- pkg/compute/models/guests.go | 48 ++- pkg/compute/tasks/guest_create_disk_task.go | 109 +++++++ pkg/compute/tasks/guest_deploy_task.go | 6 +- pkg/compute/tasks/guest_rebuild_root_task.go | 2 +- pkg/util/aliyun/disk.go | 23 +- pkg/util/aliyun/host.go | 4 +- pkg/util/aliyun/instance.go | 194 ++++++++++-- pkg/util/aliyun/keypair.go | 31 ++ pkg/util/aliyun/shell/disk.go | 11 + pkg/util/aliyun/shell/instance.go | 57 ++++ pkg/util/aliyun/storage.go | 2 +- pkg/util/esxi/datacenter.go | 1 + pkg/util/esxi/storage.go | 2 +- pkg/util/esxi/virtualmachine.go | 20 ++ 17 files changed, 683 insertions(+), 134 deletions(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 8b4179f50d..76a3a18c93 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -980,7 +980,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre return nil, httperrors.NewGeneralError(err) } item.PreUpdate(ctx, userCred, query, dataDict) - diff, err := manager.TableSpec().Update(item, func() error { filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...) err = filterData.Unmarshal(item) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f388d66feb..ec124341c0 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -153,7 +153,12 @@ type ICloudVM interface { StopVM(isForce bool) error DeleteVM() error + UpdateVM(name string) error + RebuildRoot(imageId string) error + DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error + ChangeConfig(instanceId string,ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) + AttachDisk(diskId string) error } type ICloudNic interface { @@ -193,7 +198,8 @@ type ICloudDisk interface { GetCacheMode() string GetMountpoint() string Delete() error - Resize(int64) error + + Resize(newSize int64) error } type ICloudVpc interface { diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 9477c541f6..d7489da423 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "time" - "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" @@ -103,7 +102,6 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo imageId := disk.GetTemplateId() scimg := models.StoragecachedimageManager.GetStoragecachedimage(cache.Id, imageId) config.ExternalImageId = scimg.ExternalId - img := scimg.GetCachedimage() config.OsDistribution, _ = img.Info.GetString("properties", "os_distribution") config.OsVersion, _ = img.Info.GetString("properties", "os_version") @@ -134,92 +132,134 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return err } - if action != "create" { - return fmt.Errorf("Action %s not supported", action) - } - ihost, err := host.GetIHost() if err != nil { return err } - desc := SAliyunVMCreateConfig{} - err = config.Unmarshal(&desc, "desc") - if err != nil { - return err - } - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - passwd := seclib2.RandomPassword2(12) - - iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) + if action == "create" { + desc := SAliyunVMCreateConfig{} + err = config.Unmarshal(&desc, "desc") if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) - err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) - if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) - - iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) - if err != nil { - log.Errorf("cannot find vm %s", err) - return nil, err + return err } - if len(guest.SecgrpId) > 0 { - if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { - log.Errorf("SyncSecurityGroup error: %v", err) - return nil, err - } - } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + passwd := seclib2.RandomPassword2(12) - if onfinish == "none" { - err = iVM.StartVM() + iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) if err != nil { return nil, err } - } - - encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - data.Add(jsonutils.NewString("root"), "account") - data.Add(jsonutils.NewString(encpasswd), "key") - - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - diskInfo[i] = dinfo + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return nil, err } - data.Add(jsonutils.Marshal(&diskInfo), "disks") + log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) + + iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) + if err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + if len(guest.SecgrpId) > 0 { + if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + log.Errorf("SyncSecurityGroup error: %v", err) + return nil, err + } + } + + if onfinish == "none" { + err = iVM.StartVM() + if err != nil { + return nil, err + } + } + + encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(iVM.GetOSType()), "os") + data.Add(jsonutils.NewString("root"), "account") + data.Add(jsonutils.NewString(encpasswd), "key") + + if len(desc.OsDistribution) > 0 { + data.Add(jsonutils.NewString(desc.OsDistribution), "distro") + } + if len(desc.OsVersion) > 0 { + data.Add(jsonutils.NewString(desc.OsVersion), "version") + } + + idisks, err := iVM.GetIDisks() + + if err != nil { + log.Errorf("GetiDisks error %s", err) + } else { + diskInfo := make([]SDiskInfo, len(idisks)) + for i := 0; i < len(idisks); i += 1 { + dinfo := SDiskInfo{} + dinfo.Uuid = idisks[i].GetGlobalId() + dinfo.Size = idisks[i].GetDiskSizeMB() + diskInfo[i] = dinfo + } + data.Add(jsonutils.Marshal(&diskInfo), "disks") + } + + data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + + return data, nil + }) + } else if action == "deploy" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") } - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + params := task.GetParams() + log.Debugf("Deploy VM params %s", params.String()) + var name string + if v, e := params.GetString("name"); e != nil { + name = v + } + var description string + if v, e := params.GetString("description"); e != nil { + description = v + } + resetPassword := jsonutils.QueryBoolean(params, "reset_password", false) + deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) + password, _ := params.GetString("password") + if resetPassword && len(password) == 0{ + password = seclib2.RandomPassword2(12) + } + + publicKey := "" + if k, e := config.GetString("public_key"); e != nil { + publicKey = k + } + + taskman.LocalTaskRun(task, func () (jsonutils.JSONObject, error) { + encpasswd, err := utils.EncryptAESBase64(guest.Id, password) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString("root"), "account") // 用户名 + data.Add(jsonutils.NewString(encpasswd), "key") // 密码 + e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description) + return data, e + }) + } else { + return fmt.Errorf("Action %s not supported", action) + } - return data, nil - }) return nil } @@ -277,3 +317,117 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue }) return nil } + +type SAliyunVMChangeConfig struct { + InstanceId string + Cpu int + Memory int +} + +func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "AliyunGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) + return nil +} + +func (self *SAliyunGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + config := SAliyunVMChangeConfig{} + config.InstanceId = guest.GetExternalId() + config.Cpu = int(vcpuCount) + config.Memory = int(vmemSize) + // taskman localTaskRun + ihost, err := guest.GetHost().GetIHost() + if err != nil { + return err + } + + iVM, err := ihost.GetIVMById(config.InstanceId) + if err != nil { + return err + } + + err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + // todo: wait status ready & check disk + if err != nil { + return err + } + log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300) + if err != nil { + return err + } + log.Debugf("VMchangeConfig %s, and status is ready", iVM.GetGlobalId()) + return nil +} + +func (self *SAliyunGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + ihost, e := host.GetIHost() + if e != nil { + return nil, e + } + + ivm, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + return nil, e + } + + err := ivm.StartVM() + if err != nil { + return nil, e + } + + result := jsonutils.NewDict() + result.Add(jsonutils.NewBool(true), "is_running") + return result, e +} + +func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + ihost, e := guest.GetHost().GetIHost() + if e != nil { + return e + } + + externalId := guest.GetExternalId() + if len(externalId) <= 0 { + return fmt.Errorf("external id not found") + } + + disks := guest.GetDisks() + if len(disks) <= 0 { + return fmt.Errorf("guest has no disk") + } + + imageId := guest.CategorizeDisks().Root.TemplateId + cacheId := disks[0].GetDisk().GetStorage().GetStoragecache().Id + externalImageId := models.StoragecachedimageManager.GetStoragecachedimage(cacheId, imageId).ExternalId + if len(externalImageId) <= 0 { + return fmt.Errorf("external image (%s) id is not found", imageId) + } + + iVM, err := ihost.GetIVMById(externalId) + if err != nil { + return err + } + + err = iVM.RebuildRoot(externalImageId) + if err != nil { + return err + } + + log.Debugf("VMrebuildRoot %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return err + } + log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId()) + + task.ScheduleRun(nil) + return nil +} \ No newline at end of file diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index c5dec34277..09cb942918 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -472,9 +472,11 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To return nil, err } - // if data.Contains("name") { - // return nil, httperrors.NewInputParameterError("cannot update server name") - // } + if data.Contains("name") { + if name, _ := data.GetString("name"); len(name) < 2 { + return nil, httperrors.NewInputParameterError("name is to short") + } + } /* if self.GetHypervisor() == HYPERVISOR_BAREMETAL { return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal") } @@ -1507,7 +1509,11 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr if !ok { return nil, fmt.Errorf("Parse query body error") } + + // 变更密码/密钥时需要Restart才能生效。更新普通字段不需要Restart + doRestart := false if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { + doRestart = true var kpId string if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { keypair, _ := kwargs.GetString("keypair") @@ -1529,8 +1535,9 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr kwargs.Set("reset_password", jsonutils.JSONTrue) } } + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) { - if self.Status == VM_RUNNING { + if doRestart && self.Status == VM_RUNNING { kwargs.Set("restart", jsonutils.JSONTrue) } err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") @@ -1874,12 +1881,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke if storage == nil { return nil, fmt.Errorf("No storage to create disk") } - disk, err := self.createDiskOnStorage(ctx, userCred, storage, diskConfig, pendingUsage) if err != nil { return nil, err } - err = self.attach2Disk(disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint) return disk, err } @@ -2412,16 +2417,27 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T diskIdx += 1 } - for storageId, needSize := range diskSizes { - iStorage, err := StorageManager.FetchById(storageId) - if err != nil { - return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) - } - storage := iStorage.(*SStorage) - if storage.GetFreeCapacity() < needSize { - return nil, httperrors.NewInsufficientResourceError("Not enough free space") - } + provider, e := self.GetHost().GetDriver() + if e != nil { + log.Errorf("Get Provider Error: %s", e) + return nil, httperrors.NewInsufficientResourceError("Provider Not Found") } + + if !provider.IsPublicCloud() { + for storageId, needSize := range diskSizes { + iStorage, err := StorageManager.FetchById(storageId) + if err != nil { + return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) + } + storage := iStorage.(*SStorage) + if storage.GetFreeCapacity() < needSize { + return nil, httperrors.NewInsufficientResourceError("Not enough free space") + } + } + } else { + log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetName()) + } + if newDisks.Length() > 0 { confs.Add(newDisks, "create") } @@ -3293,4 +3309,4 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use for i := 0; i < len(guests); i += 1 { guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true) } -} +} \ No newline at end of file diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index c7034bd735..bc734944cd 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/log" ) type GuestCreateDiskTask struct { @@ -118,7 +119,115 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob self.SetStageComplete(ctx, nil) } +type AliyunGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *AliyunGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_aliyun_disk_prepared", nil) + self.OnAliyunDiskPrepared(ctx, obj, data) +} + +func (self *AliyunGuestCreateDiskTask) OnAliyunDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + var diskIndex = 0 + var diskReady = true + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status == models.DISK_INIT { + snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + if err != nil { + snapInfo = "" + } + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + diskReady = false + break + } + diskIndex += 1 + } + diskIndex = 0 + guest := obj.(*models.SGuest) + + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status != models.DISK_READY { + diskReady = false + break + } + + ihost, err := guest.GetHost().GetIHost() + if err != nil { + self.SetStageFailed(ctx, "Host not found") + return + } + + iVM, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + self.SetStageFailed(ctx, "Aliyun VM not found") + return + } + + err = iVM.AttachDisk(disk.GetExternalId()) + if err != nil { + log.Debugf("Attach Disk %s to guest fail: %s", diskId, err) + self.SetStageFailed(ctx, "Attach Disk to guest fail") + return + } + diskIndex += 1 + } + + if diskReady { + if guest.Status == models.VM_RUNNING { + self.SetStage("on_config_sync_complete", nil) + err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + } else { + self.SetStageComplete(ctx, nil) + } + } +} + +func (self *AliyunGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *AliyunGuestCreateDiskTask) AttachAliyunDisks(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + func init() { taskman.RegisterTask(GuestCreateDiskTask{}) taskman.RegisterTask(KVMGuestCreateDiskTask{}) + taskman.RegisterTask(AliyunGuestCreateDiskTask{}) } diff --git a/pkg/compute/tasks/guest_deploy_task.go b/pkg/compute/tasks/guest_deploy_task.go index 47fd39b7c7..037a909a1c 100644 --- a/pkg/compute/tasks/guest_deploy_task.go +++ b/pkg/compute/tasks/guest_deploy_task.go @@ -54,14 +54,10 @@ func (self *GuestDeployTask) StartDeployGuestOnHost(ctx context.Context, guest * log.Errorf("request_deploy_guest_on_host %s", err) self.OnDeployGuestFail(ctx, guest, err) } else { - self.OnDeployGuestSucc(guest) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") } } -func (self *GuestDeployTask) OnDeployGuestSucc(guest *models.SGuest) { - guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") -} - func (self *GuestDeployTask) OnDeployGuestFail(ctx context.Context, guest *models.SGuest, err error) { guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go index 03345b85ac..50de80c0e9 100644 --- a/pkg/compute/tasks/guest_rebuild_root_task.go +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -141,4 +141,4 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { self.SetStageComplete(ctx, nil) -} +} \ No newline at end of file diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index aeeca8096c..ca6f21d4f8 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -138,6 +138,12 @@ func (self *SDisk) Refresh() error { return jsonutils.Update(self, new) } +func (self *SDisk) ResizeDisk(newSize int64) error { + // newSize 单位为 GB. 范围在20 ～2000. 只能往大调。不能调小 + // https://help.aliyun.com/document_detail/25522.html?spm=a2c4g.11174283.6.897.aHwqkS + return self.storage.zone.region.resizeDisk(self.DiskId, newSize) +} + func (self *SDisk) GetDiskFormat() string { return "vhd" } @@ -185,7 +191,7 @@ func (self *SDisk) GetMountpoint() string { return "" } -func (self *SRegion) createDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { +func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { params := make(map[string]string) params["ZoneId"] = zoneId params["DiskName"] = name @@ -223,11 +229,24 @@ func (self *SRegion) deleteDisk(diskId string) error { return err } +func (self *SRegion) DeleteDisk(diskId string) error { + params := make(map[string]string) + params["DiskId"] = diskId + + _, err := self.ecsRequest("DeleteDisk", params) + return err +} + func (self *SRegion) resizeDisk(diskId string, size int64) error { params := make(map[string]string) params["DiskId"] = diskId params["NewSize"] = fmt.Sprintf("%d", size) _, err := self.ecsRequest("ResizeDisk", params) - return err + if err != nil { + log.Errorf("ResizeDisk %s to %s GiB fail %s", diskId, size, err) + return err + } + + return nil } diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 3ee4a40236..cf842879ad 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -157,7 +157,7 @@ func (self *SHost) GetManagerId() string { return self.zone.region.client.providerId } -func (self *SHost) getInstanceById(instanceId string) (*SInstance, error) { +func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { inst, err := self.zone.region.GetInstance(instanceId) if err != nil { return nil, err @@ -171,7 +171,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, if err != nil { return nil, err } - vm, err := self.getInstanceById(vmId) + vm, err := self.GetInstanceById(vmId) if err != nil { return nil, err } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 80a2e9c267..9bf836a180 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" "time" + "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -110,6 +111,7 @@ type SInstance struct { InternetMaxBandwidthIn int InternetMaxBandwidthOut int IoOptimized bool + KeyPairName string Memory int NetworkInterfaces SNetworkInterfaces OSName string @@ -322,6 +324,81 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_ALIYUN } +func (self *SInstance) StartVM() error { + err := self.host.zone.region.StartVM(self.InstanceId) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes +} + +func (self *SInstance) StopVM(isForce bool) error { + err := self.host.zone.region.StopVM(self.InstanceId, isForce) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues +} + +func (self *SInstance) DeleteVM() error { + err := self.host.zone.region.DeleteVM(self.InstanceId) + if err != nil { + return err + } + return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes +} + +func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { + url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) + if err != nil { + return nil, err + } + passwd := seclib.RandomPassword(6) + err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) + if err != nil { + return nil, err + } + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(url), "url") + ret.Add(jsonutils.NewString(passwd), "password") + ret.Add(jsonutils.NewString("aliyun"), "protocol") + ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") + return ret, nil +} + +func (self *SInstance) UpdateVM(name string) error { + return self.host.zone.region.UpdateVM(self.InstanceId, name) +} + +func (self *SInstance) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + var keypairName string + if len(publicKey) > 0 { + key, e := self.host.lookUpAliyunKeypair(publicKey) + if e != nil { + key, e = self.host.importAliyunKeypair(publicKey) + if e != nil { + return e + } + } + + keypairName = key + } + + return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, resetPassword, deleteKeypair, description) +} + +func (self *SInstance) RebuildRoot(imageId string) error { + return self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId) +} + +func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { + return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) +} + +func (self *SInstance) AttachDisk(diskId string) error { + return self.host.zone.region.AttachDisk(self.InstanceId, diskId) +} + func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { instances, _, err := self.GetInstances("", []string{instanceId}, 0, 1) if err != nil { @@ -472,46 +549,99 @@ func (self *SRegion) DeleteVM(instanceId string) error { // } } -func (self *SInstance) StartVM() error { - err := self.host.zone.region.StartVM(self.InstanceId) +func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, resetPassword bool, deleteKeypair bool, description string) error { + instance, err := self.GetInstance(instanceId) if err != nil { return err } - return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes + + // 修改密钥时直接返回 + if deleteKeypair { + return self.DetachKeyPair(instanceId, instance.KeyPairName) + } + + if len(keypairName) > 0 { + return self.AttachKeypair(instanceId, keypairName) + } + + params := make(map[string]string) + if resetPassword { + params["Password"] = seclib2.RandomPassword2(12) + } + // 指定密码的情况下，使用指定的密码 + if len(password) > 0 { + params["Password"] = password + } + + if len(name) > 0 && instance.InstanceName != name { + params["InstanceName"] = name + params["HostName"] = name + } + + if len(description) > 0 && instance.Description != description { + params["Description"] = description + } + + if len(params) > 0 { + log.Debugf("DeployVM with params %s", params) + return self.modifyInstanceAttribute(instanceId, params) + } else { + return nil + } } -func (self *SInstance) StopVM(isForce bool) error { - err := self.host.zone.region.StopVM(self.InstanceId, isForce) +func (self *SRegion) UpdateVM(instanceId string, hostname string) error { + /* + api: ModifyInstanceAttribute + https://help.aliyun.com/document_detail/25503.html?spm=a2c4g.11186623.4.1.DrgpjW + */ + params := make(map[string]string) + params["HostName"] = hostname + return self.modifyInstanceAttribute(instanceId, params) +} + +func (self *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error { + return self.instanceOperation(instanceId, "ModifyInstanceAttribute", params) +} + +func (self *SRegion) ReplaceSystemDisk(instanceId string, image string) error { + params := make(map[string]string) + params["ImageId"] = image + return self.instanceOperation(instanceId, "ReplaceSystemDisk", params) +} + +func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { + // todo: support change disk config? + params := make(map[string]string) + instanceTypes, e := self.GetMatchInstanceTypes(ncpu, vmem, 0, zoneId) + if e != nil { + return e + } + + for _, instancetype := range instanceTypes { + params["InstanceType"] = instancetype.InstanceTypeId + params["ClientToken"] = utils.GenRequestId(20) + if err := self.instanceOperation(instanceId, "ModifyInstanceSpec", params); err != nil { + log.Errorf("Failed for %s: %s", instancetype.InstanceTypeId, err) + } else { + return nil + } + } + + return fmt.Errorf("Failed to change vm config, specification not supported") +} + +func (self *SRegion) AttachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["DiskId"] = diskId + _, err := self.ecsRequest("AttachDisk", params) if err != nil { + log.Errorf("AttachDisk %s to %s fail %s", diskId, instanceId, err) return err } - return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues -} -func (self *SInstance) DeleteVM() error { - err := self.host.zone.region.DeleteVM(self.InstanceId) - if err != nil { - return err - } - return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes -} - -func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { - url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) - if err != nil { - return nil, err - } - passwd := seclib.RandomPassword(6) - err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) - if err != nil { - return nil, err - } - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(url), "url") - ret.Add(jsonutils.NewString(passwd), "password") - ret.Add(jsonutils.NewString("aliyun"), "protocol") - ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") - return ret, nil + return nil } func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { @@ -538,4 +668,4 @@ func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules [ self.SecurityGroupIds.SecurityGroupId = []string{secgrpId} } return nil -} +} \ No newline at end of file diff --git a/pkg/util/aliyun/keypair.go b/pkg/util/aliyun/keypair.go index c8da45b6a0..e3b2a504a8 100644 --- a/pkg/util/aliyun/keypair.go +++ b/pkg/util/aliyun/keypair.go @@ -1,6 +1,7 @@ package aliyun import ( + "encoding/json" "fmt" "yunion.io/x/log" ) @@ -62,3 +63,33 @@ func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error } return &keypair, nil } + +func (self *SRegion) AttachKeypair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("AttachKeyPair", params) + if err != nil { + log.Errorf("AttachKeyPair fail %s", err) + return err + } + + return nil +} + +func (self *SRegion) DetachKeyPair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("DetachKeyPair", params) + if err != nil { + log.Errorf("DetachKeyPair fail %s", err) + return err + } + + return nil +} diff --git a/pkg/util/aliyun/shell/disk.go b/pkg/util/aliyun/shell/disk.go index 1196bb5739..19cc4c7693 100644 --- a/pkg/util/aliyun/shell/disk.go +++ b/pkg/util/aliyun/shell/disk.go @@ -21,4 +21,15 @@ func init() { printList(disks, total, args.Offset, args.Limit, []string{}) return nil }) + + type DiskDeleteOptions struct { + Instance string `help:"Instance ID"` + } + shellutils.R(&DiskDeleteOptions{}, "disk-delete", "List disks", func(cli *aliyun.SRegion, args *DiskDeleteOptions) error { + e := cli.DeleteDisk(args.Instance) + if e != nil { + return e + } + return nil + }) } diff --git a/pkg/util/aliyun/shell/instance.go b/pkg/util/aliyun/shell/instance.go index 495e2949cf..4a741d0b66 100644 --- a/pkg/util/aliyun/shell/instance.go +++ b/pkg/util/aliyun/shell/instance.go @@ -82,6 +82,63 @@ func init() { return nil }) + /* + server-change-config 更改系统配置 + server-reset + */ + type InstanceDeployOptions struct { + ID string `help:"instance ID"` + Name string `help:"new instance name"` + Hostname string `help:"new hostname"` + Keypair string `help:"Keypair Name"` + DeleteKeypair bool `help:"Remove SSH keypair"` + Password string `help:"new password"` + ResetPassword bool `help:"Force reset password"` + Description string `help:"new instances description"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceDeployOptions) error { + err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.ResetPassword, args.DeleteKeypair, args.Description) + if err != nil { + return err + } + return nil + }) + + type InstanceRebuildRootOptions struct { + ID string `help:"instance ID"` + Image string `help:"Image ID"` + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *aliyun.SRegion, args *InstanceRebuildRootOptions) error { + err := cli.ReplaceSystemDisk(args.ID, args.Image) + if err != nil { + return err + } + return nil + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"instance ID"` + Ncpu int `help:"number of CPU"` + Vmem int `help:"MiB of memory"` + Disk []int `help:"Data disk sizes int GB"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceChangeConfigOptions) error { + instance, e := cli.GetInstance(args.ID) + if e != nil { + return e + } + + // todo : add create disks + err := cli.ChangeVMConfig(instance.ZoneId, args.ID, args.Ncpu, args.Vmem, nil) + if err != nil { + return err + } + return nil + }) + type InstanceUpdatePasswordOptions struct { ID string `help:"Instance ID"` PASSWD string `help:"new password"` diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index b54700b1e6..8f66d8b68b 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -100,7 +100,7 @@ func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { } func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - diskId, err := self.zone.region.createDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) + diskId, err := self.zone.region.CreateDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) if err != nil { log.Errorf("createDisk fail %s", err) return nil, err diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 7da30e1dfb..6c9d1c6d0d 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -76,3 +76,4 @@ func (dc *SDatacenter) GetIStorages() ([]cloudprovider.ICloudStorage, error) { } return dc.istorages, nil } + diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 90846ed5a9..dc154261d2 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -89,4 +89,4 @@ func (self *SDatastore) GetManagerId() string { func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented -} +} \ No newline at end of file diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2005bbc9bc..75ccccc448 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -65,6 +65,22 @@ func (self *SVirtualMachine) IsEmulated() bool { return false } +func (self *SVirtualMachine) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) RebuildRoot(imageId string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) UpdateVM(name string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) AttachDisk(diskId string) error { + return cloudprovider.ErrNotImplemented +} + func (self *SVirtualMachine) getUuid() string { return self.getVirtualMachine().Summary.Config.Uuid } @@ -209,3 +225,7 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(url), "url") return ret, nil } + +func (dc *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return cloudprovider.ErrNotImplemented +} \ No newline at end of file From 9cdbeb943b5c08fa84a77ff68216797564bdb347 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 13 Aug 2018 19:22:34 +0800 Subject: [PATCH 19/71] climc: k8s resource cli --- cmd/climc/shell/k8s/configmap.go | 27 +++++ cmd/climc/shell/k8s/deployment.go | 145 +++++++++++++++++++++++++ cmd/climc/shell/k8s/k8s.go | 67 +++++++++++- cmd/climc/shell/k8s/pods.go | 20 +++- cmd/climc/shell/k8s/raw.go | 35 ++++++ cmd/climc/shell/k8s/repos.go | 106 ++++++++++++++++++ cmd/climc/shell/k8s/service.go | 27 +++++ pkg/mcclient/modules/k8s/base.go | 47 +++++++- pkg/mcclient/modules/k8s/cluster.go | 4 +- pkg/mcclient/modules/k8s/configmap.go | 20 ++++ pkg/mcclient/modules/k8s/deployment.go | 20 ++++ pkg/mcclient/modules/k8s/event.go | 4 +- pkg/mcclient/modules/k8s/helm.go | 17 +++ pkg/mcclient/modules/k8s/node.go | 2 +- pkg/mcclient/modules/k8s/pod.go | 4 +- pkg/mcclient/modules/k8s/raw.go | 95 ++++++++++++++++ pkg/mcclient/modules/k8s/service.go | 20 ++++ 17 files changed, 646 insertions(+), 14 deletions(-) create mode 100644 cmd/climc/shell/k8s/configmap.go create mode 100644 cmd/climc/shell/k8s/deployment.go create mode 100644 cmd/climc/shell/k8s/raw.go create mode 100644 cmd/climc/shell/k8s/repos.go create mode 100644 cmd/climc/shell/k8s/service.go create mode 100644 pkg/mcclient/modules/k8s/configmap.go create mode 100644 pkg/mcclient/modules/k8s/deployment.go create mode 100644 pkg/mcclient/modules/k8s/helm.go create mode 100644 pkg/mcclient/modules/k8s/raw.go create mode 100644 pkg/mcclient/modules/k8s/service.go diff --git a/cmd/climc/shell/k8s/configmap.go b/cmd/climc/shell/k8s/configmap.go new file mode 100644 index 0000000000..30d8fd6d9e --- /dev/null +++ b/cmd/climc/shell/k8s/configmap.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initConfigMap() { + cmdN := func(suffix string) string { + return resourceCmdN("configmap", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s configmap", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.ConfigMaps.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.ConfigMaps.GetColumns(s)) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/deployment.go b/cmd/climc/shell/k8s/deployment.go new file mode 100644 index 0000000000..dfdc392bba --- /dev/null +++ b/cmd/climc/shell/k8s/deployment.go @@ -0,0 +1,145 @@ +package k8s + +import ( + "fmt" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initDeployment() { + cmdN := func(suffix string) string { + return resourceCmdN("deployment", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s deployment", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Deployments.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Deployments.GetColumns(s)) + return nil + }) + + type createOpt struct { + namespaceOptions + NAME string `help:"Name of deployment"` + Image string `help:"The image for the container to run"` + Replicas int64 `help:"Number of replicas for pods in this deployment"` + RunAsPrivileged bool `help:"Whether to run the container as privileged user"` + Labels string `help:"Comma separated labels to apply to the pod(s)"` + Env []string `help:"Environment variables to set in container"` + Port []string `help:"Port for the service that is created, format is :: e.g. tcp:80:3000"` + } + R(&createOpt{}, cmdN("create"), "Create deployment resource", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + if len(args.Image) == 0 { + return fmt.Errorf("Image must provided") + } + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.Image), "containerImage") + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + if args.Replicas > 1 { + params.Add(jsonutils.NewInt(args.Replicas), "replicas") + } + if args.RunAsPrivileged { + params.Add(jsonutils.JSONTrue, "runAsPrivileged") + } + if len(args.Port) != 0 { + portMappings, err := parsePortMappings(args.Port) + if err != nil { + return err + } + params.Add(portMappings, "portMappings") + } + ret, err := k8s.Deployments.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type getOpt struct { + resourceGetOptions + } + R(&getOpt{}, cmdN("show"), "Get deployment details", func(s *mcclient.ClientSession, args *getOpt) error { + id := args.NAME + params := jsonutils.NewDict() + if args.Namespace != "" { + params.Add(jsonutils.NewString(args.Namespace), "namespace") + } + ret, err := k8s.Deployments.GetInContexts(s, id, params, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(ret) + return nil + }) +} + +type portMapping struct { + Port int32 `json:"port"` + TargetPort int32 `json:"targetPort"` + Protocol string `json:"protocol"` +} + +func parsePortMapping(port string) (*portMapping, error) { + if len(port) == 0 { + return nil, fmt.Errorf("empty port mapping desc string") + } + parts := strings.Split(port, ":") + mapping := &portMapping{} + for _, part := range parts { + if sets.NewString("tcp", "udp").Has(strings.ToLower(part)) { + mapping.Protocol = strings.ToUpper(part) + } + if port, err := strconv.Atoi(part); err != nil { + continue + } else { + if mapping.Port == 0 { + mapping.Port = int32(port) + } else { + mapping.TargetPort = int32(port) + } + } + } + if mapping.Protocol == "" { + mapping.Protocol = "TCP" + } + if mapping.Port <= 0 { + return nil, fmt.Errorf("Service port not provided") + } + if mapping.TargetPort < 0 { + return nil, fmt.Errorf("Container invalid targetPort %d", mapping.TargetPort) + } + if mapping.TargetPort == 0 { + mapping.TargetPort = mapping.Port + } + return mapping, nil +} + +func parsePortMappings(ports []string) (*jsonutils.JSONArray, error) { + ret := jsonutils.NewArray() + for _, port := range ports { + mapping, err := parsePortMapping(port) + if err != nil { + return nil, fmt.Errorf("Port %q error: %v", port, err) + } + ret.Add(jsonutils.Marshal(mapping)) + } + return ret, nil +} diff --git a/cmd/climc/shell/k8s/k8s.go b/cmd/climc/shell/k8s/k8s.go index 2e63757294..af4892e71a 100644 --- a/cmd/climc/shell/k8s/k8s.go +++ b/cmd/climc/shell/k8s/k8s.go @@ -12,24 +12,81 @@ import ( ) func init() { + // cluster resources initCluster() initNode() + + // helm resources + initRepo() + + // kubernetes original resources + initRaw() + initConfigMap() + initDeployment() initPod() + initService() } type BaseListOptions shell.BaseListOptions type clusterBaseOptions struct { - Cluster string `default:"$K8S_CLUSTER" help:"Kubernetes cluster name"` + Cluster string `default:"$K8S_CLUSTER|default" help:"Kubernetes cluster name"` } func (o clusterBaseOptions) ClusterContext() []modules.ManagerContext { return []modules.ManagerContext{clusterContext(o.Cluster)} } -type k8sBaseListOptions struct { +type baseListOptions struct { + Limit int `default:"20" help:"Page limit"` + Offset int `default:"0" help:"page offset"` +} + +func fetchPagingParams(opt baseListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.Limit > 0 { + params.Add(jsonutils.NewInt(int64(opt.Limit)), "limit") + } + if opt.Offset > 0 { + params.Add(jsonutils.NewInt(int64(opt.Offset)), "offset") + } + return params +} + +type namespaceListOptions struct { + namespaceOptions + AllNamespace bool `help:"Show resource in all namespace"` +} + +type namespaceOptions struct { clusterBaseOptions - Limit int `default:"20" help:"Page limit"` + Namespace string `help:"Namespace of this resource"` +} + +type resourceGetOptions struct { + clusterBaseOptions + Namespace string `help:"Namespace of this resource"` + NAME string `help:"Name ident of the resource"` +} + +func (o resourceGetOptions) ToJSON() *jsonutils.JSONDict { + params := jsonutils.NewDict() + if o.Namespace != "" { + params.Add(jsonutils.NewString(o.Namespace), "namespace") + } + return params +} + +func fetchNamespaceParams(opt namespaceListOptions) *jsonutils.JSONDict { + params := jsonutils.NewDict() + if opt.AllNamespace { + params.Add(jsonutils.JSONTrue, "all_namespace") + return params + } + if opt.Namespace != "" { + params.Add(jsonutils.NewString(opt.Namespace), "namespace") + } + return params } var ( @@ -53,3 +110,7 @@ func clusterContext(clusterId string) modules.ManagerContext { InstanceId: clusterId, } } + +func printObjectYAML(obj jsonutils.JSONObject) { + fmt.Println(obj.YAMLString()) +} diff --git a/cmd/climc/shell/k8s/pods.go b/cmd/climc/shell/k8s/pods.go index 8d2fdd0d3b..ee8803c414 100644 --- a/cmd/climc/shell/k8s/pods.go +++ b/cmd/climc/shell/k8s/pods.go @@ -13,10 +13,26 @@ func initPod() { } type listOpt struct { - k8sBaseListOptions + namespaceListOptions + baseListOptions } R(&listOpt{}, cmdN("list"), "List k8s pod", func(s *mcclient.ClientSession, args *listOpt) error { - ret, err := k8s.Pods.ListInContexts(s, nil, args.ClusterContext()) + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Pods.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Pods.GetColumns(s)) + return nil + }) + + type deleteOpt struct { + resourceGetOptions + } + R(&deleteOpt{}, cmdN("delete"), "Delete pod", func(s *mcclient.ClientSession, args *deleteOpt) error { + id := args.NAME + ret, err := k8s.Pods.DeleteInContexts(s, id, args.ToJSON(), args.ClusterContext()) if err != nil { return err } diff --git a/cmd/climc/shell/k8s/raw.go b/cmd/climc/shell/k8s/raw.go new file mode 100644 index 0000000000..6aeb500f91 --- /dev/null +++ b/cmd/climc/shell/k8s/raw.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +type rawGetOpt struct { + namespaceOptions + KIND string `help:"resource kind"` + NAME string `help:"instance name"` +} + +type rawDeleteOpt struct { + rawGetOpt +} + +func initRaw() { + R(&rawGetOpt{}, "k8s-get", "Get k8s resource instance raw info", func(s *mcclient.ClientSession, args *rawGetOpt) error { + obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + printObjectYAML(obj) + return nil + }) + + R(&rawDeleteOpt{}, "k8s-delete", "Delete k8s resource instance", func(s *mcclient.ClientSession, args *rawDeleteOpt) error { + err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + return nil + }) +} diff --git a/cmd/climc/shell/k8s/repos.go b/cmd/climc/shell/k8s/repos.go new file mode 100644 index 0000000000..ee34d8a2f9 --- /dev/null +++ b/cmd/climc/shell/k8s/repos.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initRepo() { + cmdN := func(suffix string) string { + return resourceCmdN("repo", suffix) + } + type listOpt struct { + BaseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s global helm repos", func(s *mcclient.ClientSession, args *listOpt) error { + params := FetchPagingParams(args.BaseListOptions) + result, err := k8s.Repos.List(s, params) + if err != nil { + return err + } + printList(result, k8s.Repos.GetColumns(s)) + return nil + }) + + type getOpt struct { + NAME string `help:"ID or name of the repo"` + } + R(&getOpt{}, cmdN("show"), "Show details fo a repo", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Get(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type createOpt struct { + getOpt + URL string `help:"Repository url"` + Public bool `help:"Make repository public"` + } + R(&createOpt{}, cmdN("create"), "Add repository", func(s *mcclient.ClientSession, args *createOpt) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(args.NAME), "name") + params.Add(jsonutils.NewString(args.URL), "url") + if args.Public { + params.Add(jsonutils.JSONTrue, "is_public") + } + repo, err := k8s.Repos.Create(s, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + type updateOpt struct { + getOpt + Name string `help:"Repository name to change"` + Url string `help:"Repository url to change"` + } + R(&updateOpt{}, cmdN("update"), "Update helm repository", func(s *mcclient.ClientSession, args *updateOpt) error { + params := jsonutils.NewDict() + if args.Name != "" { + params.Add(jsonutils.NewString(args.Name), "name") + } + if args.Url != "" { + params.Add(jsonutils.NewString(args.Url), "url") + } + repo, err := k8s.Repos.Update(s, args.NAME, params) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("private"), "Make repository private", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "private", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("public"), "Make repository public", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.PerformAction(s, args.NAME, "public", nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) + + R(&getOpt{}, cmdN("delete"), "Delete a repository", func(s *mcclient.ClientSession, args *getOpt) error { + repo, err := k8s.Repos.Delete(s, args.NAME, nil) + if err != nil { + return err + } + printObject(repo) + return nil + }) +} diff --git a/cmd/climc/shell/k8s/service.go b/cmd/climc/shell/k8s/service.go new file mode 100644 index 0000000000..3260b28b88 --- /dev/null +++ b/cmd/climc/shell/k8s/service.go @@ -0,0 +1,27 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initService() { + cmdN := func(suffix string) string { + return resourceCmdN("service", suffix) + } + + type listOpt struct { + namespaceListOptions + baseListOptions + } + R(&listOpt{}, cmdN("list"), "List k8s service", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + ret, err := k8s.Services.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Services.GetColumns(s)) + return nil + }) +} diff --git a/pkg/mcclient/modules/k8s/base.go b/pkg/mcclient/modules/k8s/base.go index 6c5eef1847..3b5fc20b61 100644 --- a/pkg/mcclient/modules/k8s/base.go +++ b/pkg/mcclient/modules/k8s/base.go @@ -1,13 +1,56 @@ package k8s import ( + "yunion.io/x/pkg/util/sets" + "yunion.io/x/onecloud/pkg/mcclient/modules" ) -func NewManager(keyword, keywordPlural string, columns, adminColumns []string) *modules.ResourceManager { +func NewManager(keyword, keywordPlural string, columns, adminColumns *Columns) *modules.ResourceManager { return &modules.ResourceManager{ - BaseManager: *modules.NewBaseManager("k8s", "", "", columns, adminColumns), + BaseManager: *modules.NewBaseManager("k8s", "", "", columns.Array(), adminColumns.Array()), Keyword: keyword, KeywordPlural: keywordPlural, } } + +type Columns struct { + cols []string +} + +func NewColumns(cols ...string) *Columns { + c := &Columns{cols: make([]string, 0)} + return c.Add(cols...) +} + +func (c *Columns) Add(cols ...string) *Columns { + for _, col := range cols { + c.add(col) + } + return c +} + +func (c *Columns) add(col string) *Columns { + src := sets.NewString(c.Array()...) + if src.Has(col) { + return c + } + c.cols = append(c.cols, col) + return c +} + +func (c Columns) Array() []string { + return c.cols +} + +func NewNamespaceCols(col ...string) *Columns { + return NewColumns("name", "namespace").Add(col...) +} + +func NewClusterCols(col ...string) *Columns { + return NewColumns("cluster").Add(col...) +} + +func NewResourceCols(col ...string) *Columns { + return NewColumns("name", "id").Add(col...) +} diff --git a/pkg/mcclient/modules/k8s/cluster.go b/pkg/mcclient/modules/k8s/cluster.go index df2df5ca39..42ee41d6b1 100644 --- a/pkg/mcclient/modules/k8s/cluster.go +++ b/pkg/mcclient/modules/k8s/cluster.go @@ -10,7 +10,7 @@ var ( func init() { Clusters = NewManager("cluster", "clusters", - []string{"id", "name", "mode", "k8s_version", "status", "api_endpoint"}, - []string{}) + NewResourceCols("mode", "k8s_version", "status", "api_endpoint"), + NewColumns()) modules.Register(Clusters) } diff --git a/pkg/mcclient/modules/k8s/configmap.go b/pkg/mcclient/modules/k8s/configmap.go new file mode 100644 index 0000000000..4bfcbbafdf --- /dev/null +++ b/pkg/mcclient/modules/k8s/configmap.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ConfigMaps *ConfigMapManager + +type ConfigMapManager struct { + modules.ResourceManager +} + +func init() { + ConfigMaps = &ConfigMapManager{ + ResourceManager: *NewManager( + "configmap", "configmaps", + NewNamespaceCols(), + NewClusterCols())} + modules.Register(ConfigMaps) +} diff --git a/pkg/mcclient/modules/k8s/deployment.go b/pkg/mcclient/modules/k8s/deployment.go new file mode 100644 index 0000000000..f56a2947ef --- /dev/null +++ b/pkg/mcclient/modules/k8s/deployment.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Deployments *DeploymentManager + +type DeploymentManager struct { + modules.ResourceManager +} + +func init() { + Deployments = &DeploymentManager{ + ResourceManager: *NewManager( + "deployment", "deployments", + NewNamespaceCols("labels"), + NewClusterCols())} + modules.Register(Deployments) +} diff --git a/pkg/mcclient/modules/k8s/event.go b/pkg/mcclient/modules/k8s/event.go index f71c5541ef..584ddbc376 100644 --- a/pkg/mcclient/modules/k8s/event.go +++ b/pkg/mcclient/modules/k8s/event.go @@ -10,7 +10,7 @@ var ( func init() { Logs = NewManager("kube_event", "kube_events", - []string{"id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"}, - []string{}) + NewColumns("id", "ops_time", "obj_id", "obj_type", "obj_name", "user", "user_id", "tenant", "tenant_id", "owner_tenant_id", "action", "notes"), + NewColumns()) modules.Register(Logs) } diff --git a/pkg/mcclient/modules/k8s/helm.go b/pkg/mcclient/modules/k8s/helm.go new file mode 100644 index 0000000000..b3e04f21ea --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm.go @@ -0,0 +1,17 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Repos *modules.ResourceManager +) + +func init() { + Repos = NewManager("repo", "repos", + NewResourceCols("url", "is_public", "source"), + NewColumns(), + ) + modules.Register(Repos) +} diff --git a/pkg/mcclient/modules/k8s/node.go b/pkg/mcclient/modules/k8s/node.go index 5b8cb0798b..60f5bd6ade 100644 --- a/pkg/mcclient/modules/k8s/node.go +++ b/pkg/mcclient/modules/k8s/node.go @@ -9,6 +9,6 @@ var ( ) func init() { - Nodes = NewManager("node", "nodes", []string{"id", "name", "cluster", "roles", "address", "status"}, []string{}) + Nodes = NewManager("node", "nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns()) modules.Register(Nodes) } diff --git a/pkg/mcclient/modules/k8s/pod.go b/pkg/mcclient/modules/k8s/pod.go index 46417df838..218ef6a243 100644 --- a/pkg/mcclient/modules/k8s/pod.go +++ b/pkg/mcclient/modules/k8s/pod.go @@ -14,7 +14,7 @@ func init() { Pods = &PodManager{ ResourceManager: *NewManager( "pod", "pods", - []string{"id", "name", "cluster", "roles", "address", "status"}, - []string{})} + NewNamespaceCols("podIP", "status", "restartCount", "labels"), + NewClusterCols("nodeName"))} modules.Register(Pods) } diff --git a/pkg/mcclient/modules/k8s/raw.go b/pkg/mcclient/modules/k8s/raw.go new file mode 100644 index 0000000000..8853a18742 --- /dev/null +++ b/pkg/mcclient/modules/k8s/raw.go @@ -0,0 +1,95 @@ +package k8s + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var RawResource *RawResourceManager + +func init() { + RawResource = &RawResourceManager{ + serviceType: "k8s", + } +} + +type rawResourceContext struct { + kind string + name string + query jsonutils.JSONObject + ctxs []modules.ManagerContext +} + +func newRawResourceContext(kind, namespace, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) *rawResourceContext { + nsQuery := getNamespaceQuery(namespace) + if query != nil { + nsQuery.Update(query) + } + ctx := &rawResourceContext{kind: kind, name: name, query: nsQuery, ctxs: ctxs} + return ctx +} + +func (ctx rawResourceContext) contextPath() string { + segs := make([]string, 0) + ctxs := ctx.ctxs + if ctxs != nil && len(ctxs) > 0 { + for _, c := range ctxs { + segs = append(segs, c.InstanceManager.KeyString()) + if len(c.InstanceId) > 0 { + segs = append(segs, url.PathEscape(c.InstanceId)) + } + } + } + segs = append(segs, "_raw", ctx.kind, ctx.name) + path := fmt.Sprintf("/%s", strings.Join(segs, "/")) + if ctx.query != nil { + qs := ctx.query.QueryString() + if len(qs) > 0 { + path = fmt.Sprintf("%s?%s", path, qs) + } + } + return path +} + +type RawResourceManager struct { + serviceType string +} + +func (m *RawResourceManager) request(s *mcclient.ClientSession, method string, path string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { + _, ret, err := s.JSONRequest(m.serviceType, "", method, path, nil, body) + return ret, err +} + +func getNamespaceQuery(namespace string) *jsonutils.JSONDict { + query := jsonutils.NewDict() + if namespace != "" { + query.Set("namespace", jsonutils.NewString(namespace)) + } + return query +} + +func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) (jsonutils.JSONObject, error) { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + return m.request(s, "GET", ctx.contextPath(), nil) +} + +func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + rawBytes := body.String() + newBody := jsonutils.NewDict() + newBody.Add(jsonutils.NewString(rawBytes), "raw") + ctx := newRawResourceContext(kind, namespace, name, nil, ctxs) + _, err := m.request(s, "PUT", ctx.contextPath(), newBody) + return err +} + +func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) error { + ctx := newRawResourceContext(kind, namespace, name, query, ctxs) + _, err := m.request(s, "DELETE", ctx.contextPath(), nil) + return err +} diff --git a/pkg/mcclient/modules/k8s/service.go b/pkg/mcclient/modules/k8s/service.go new file mode 100644 index 0000000000..e66f38dda3 --- /dev/null +++ b/pkg/mcclient/modules/k8s/service.go @@ -0,0 +1,20 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var Services *ServiceManager + +type ServiceManager struct { + modules.ResourceManager +} + +func init() { + Services = &ServiceManager{ + ResourceManager: *NewManager( + "service", "services", + NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"), + NewClusterCols())} + modules.Register(Services) +} From d33757fbbb19ab2c09e82363a12b62094972ee1c Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 20 Aug 2018 15:10:46 +0800 Subject: [PATCH 20/71] =?UTF-8?q?=E5=A2=9E=E5=8A=A0CPU=E3=80=81=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=8F=98=E6=9B=B4=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 11 ++++++----- pkg/util/aliyun/instance.go | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index d7489da423..7baad16137 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -342,7 +342,6 @@ func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest config.InstanceId = guest.GetExternalId() config.Cpu = int(vcpuCount) config.Memory = int(vmemSize) - // taskman localTaskRun ihost, err := guest.GetHost().GetIHost() if err != nil { return err @@ -353,11 +352,13 @@ func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest return err } - err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) - // todo: wait status ready & check disk - if err != nil { - return err + if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory { + err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + if err != nil { + return err + } } + log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300) if err != nil { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 9bf836a180..e74189be1c 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -583,7 +583,6 @@ func (self *SRegion) DeployVM(instanceId string, name string, password string, k } if len(params) > 0 { - log.Debugf("DeployVM with params %s", params) return self.modifyInstanceAttribute(instanceId, params) } else { return nil From 71f5e1973034b50583941aaa6816349589f0bb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 21 Aug 2018 15:45:04 +0800 Subject: [PATCH 21/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dhost=3D=3Dnil=E6=97=B6?= =?UTF-8?q?=E8=B0=83=E7=94=A8host.GetHostDriver()=E5=87=BA=E9=94=99?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/tasks/disk_delete_task.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 436165f427..ff6c4f76c1 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" ) type DiskDeleteTask struct { @@ -51,7 +52,9 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD self.OnGuestDiskDeleteSucc(ctx, disk, nil) } else { self.SetStage("on_guest_disk_delete_succ", nil) - if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { + if host == nil { + self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host")) + } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { self.OnGuestDiskDeleteFailed(ctx, disk, err) } } From 935b8f02a3945ae38d28c1cadcef2dc3a512f0cf Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Tue, 21 Aug 2018 07:25:35 +0000 Subject: [PATCH 22/71] dep: dep ensure update - jsonutils: bugfix for unmarshaling non-nil slice - prettytable: support printing multiline cells - sqlchemy: fix update for ptr fields Update printutils to remove the redundant newline --- Gopkg.lock | 6 +- pkg/util/printutils/printjson.go | 4 +- vendor/yunion.io/x/jsonutils/unmarshal.go | 29 ++--- vendor/yunion.io/x/pkg/gotypes/deepcopy.go | 100 ++++++++++++++++++ .../x/pkg/prettytable/prettytable.go | 59 ++++++++--- vendor/yunion.io/x/sqlchemy/update.go | 5 +- 6 files changed, 167 insertions(+), 36 deletions(-) create mode 100644 vendor/yunion.io/x/pkg/gotypes/deepcopy.go diff --git a/Gopkg.lock b/Gopkg.lock index 8b923166b6..5305cc626e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -844,7 +844,7 @@ branch = "master" name = "yunion.io/x/jsonutils" packages = ["."] - revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" + revision = "38477c9cceb895816fe21507d73da30376d358b7" [[projects]] branch = "master" @@ -890,13 +890,13 @@ "util/workqueue", "utils" ] - revision = "56b426f0ca15288cc26442f5802c949f137ade60" + revision = "6c349ecbdb90d5f6ecc9b86596428176c1937b32" [[projects]] branch = "master" name = "yunion.io/x/sqlchemy" packages = ["."] - revision = "a74ef73e555a1ed2e19b1a47e0567814c289eaf9" + revision = "1d049058c031cdc229b0fb92ed902664186657e3" [[projects]] branch = "master" diff --git a/pkg/util/printutils/printjson.go b/pkg/util/printutils/printjson.go index ed01e9ce4e..da14f48590 100644 --- a/pkg/util/printutils/printjson.go +++ b/pkg/util/printutils/printjson.go @@ -58,7 +58,7 @@ func PrintJSONList(list *modules.ListResult, columns []string) { } rows = append(rows, row) } - fmt.Println(pt.GetString(rows)) + fmt.Print(pt.GetString(rows)) title := fmt.Sprintf("Total: %d", list.Total) if list.Limit == 0 && list.Total > len(list.Data) { list.Limit = len(list.Data) @@ -95,7 +95,7 @@ func PrintJSONObject(obj jsonutils.JSONObject) { } rows = append(rows, row) } - fmt.Println(pt.GetString(rows)) + fmt.Print(pt.GetString(rows)) } func PrintJSONBatchResults(results []modules.SubmitResult, columns []string) { diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index e1f3b62144..941fcdf729 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -358,12 +358,12 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error { val.SetString(this.String()) return nil case reflect.Ptr: - if val.IsNil() { - kind := val.Type().Elem().Kind() - if kind == reflect.Array || kind == reflect.Slice { + kind := val.Type().Elem().Kind() + if kind == reflect.Array || kind == reflect.Slice { + if val.IsNil() { val.Set(reflect.New(val.Type().Elem())) - return this.unmarshalValue(val.Elem()) } + return this.unmarshalValue(val.Elem()) } return fmt.Errorf("JSONArray type mismatch %s", val.Type()) case reflect.Slice, reflect.Array: @@ -373,9 +373,12 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error { val.Type(), val.Len(), len(this.data)) } } else if val.Kind() == reflect.Slice { - if val.Len() < len(this.data) { - newVal := reflect.MakeSlice(val.Type(), len(this.data), len(this.data)) + dataLen := len(this.data) + if val.Cap() < dataLen { + newVal := reflect.MakeSlice(val.Type(), dataLen, dataLen) val.Set(newVal) + } else if val.Len() != dataLen { + val.SetLen(dataLen) } } for i, json := range this.data { @@ -412,15 +415,15 @@ func (this *JSONDict) unmarshalValue(val reflect.Value) error { case reflect.Struct: return this.unmarshalStruct(val) case reflect.Ptr: - if val.IsNil() { - kind := val.Type().Elem().Kind() - if kind != reflect.Struct && kind != reflect.Map { - return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) + kind := val.Type().Elem().Kind() + if kind == reflect.Struct || kind == reflect.Map { + if val.IsNil() { + newVal := reflect.New(val.Type().Elem()) + val.Set(newVal) } - newVal := reflect.New(val.Type().Elem()) - val.Set(newVal) + return this.unmarshalValue(val.Elem()) } - return this.unmarshalValue(val.Elem()) + fallthrough default: return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) } diff --git a/vendor/yunion.io/x/pkg/gotypes/deepcopy.go b/vendor/yunion.io/x/pkg/gotypes/deepcopy.go new file mode 100644 index 0000000000..f8a186c6d1 --- /dev/null +++ b/vendor/yunion.io/x/pkg/gotypes/deepcopy.go @@ -0,0 +1,100 @@ +package gotypes + +import ( + "reflect" +) + +type DeepCopyFlags uintptr + +func DeepCopy(v interface{}) interface{} { + rv := reflect.ValueOf(v) + cpRv := DeepCopyRv(rv) + cpV := cpRv.Interface() + return cpV +} + +func DeepCopyRv(rv reflect.Value) reflect.Value { + kind := rv.Kind() + if kind == reflect.Invalid { + return reflect.Value{} + } + typ := rv.Type() + switch kind { + case reflect.Ptr: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + elemRv := rv.Elem() + cpElemRv := DeepCopyRv(elemRv) + cpRv := reflect.New(typ.Elem()) + cpRv.Elem().Set(cpElemRv) + return cpRv + case reflect.Slice, reflect.Array: + var cpRv reflect.Value + switch kind { + case reflect.Slice: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + cpRv = reflect.MakeSlice(typ, rv.Len(), rv.Cap()) + case reflect.Array: + cpRv = reflect.New(typ).Elem() + } + n := rv.Len() + for i := 0; i < n; i++ { + elem := rv.Index(i) + cpElem := DeepCopyRv(elem) + cpRvElem := cpRv.Index(i) + cpRvElem.Set(cpElem) + } + return cpRv + case reflect.Struct: + cpRv := reflect.New(typ).Elem() + if typ == TimeType { + cpRv.Set(rv) + } + n := rv.NumField() + for i := 0; i < n; i++ { + f := rv.Field(i) + if f.CanInterface() { + cpRvF := cpRv.Field(i) + cpF := DeepCopyRv(f) + cpRvF.Set(cpF) + } + } + return cpRv + case reflect.Map: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + cpRv := reflect.MakeMap(typ) + mk := rv.MapKeys() + for _, k := range mk { + v := rv.MapIndex(k) + cpK := DeepCopyRv(k) + cpV := DeepCopyRv(v) + cpRv.SetMapIndex(cpK, cpV) + } + return cpRv + case reflect.Interface: + cpRv := reflect.New(typ).Elem() + if !rv.IsNil() { + elemRv := rv.Elem() + cpElemRv := DeepCopyRv(elemRv) + cpRv.Set(cpElemRv) + } + return cpRv + case reflect.Chan: + if rv.IsNil() { + return reflect.New(typ).Elem() + } + return reflect.MakeChan(typ.Elem(), rv.Cap()) + default: + // Invalid, Bool, Int*, Uint*, Float*, Complex*, Func, String + // TODO UnsafePointer + cpRv := reflect.New(typ).Elem() + cpRv.Set(rv) + return cpRv + } + panic("unhandled kind " + kind.String()) +} diff --git a/vendor/yunion.io/x/pkg/prettytable/prettytable.go b/vendor/yunion.io/x/pkg/prettytable/prettytable.go index 8de58e04b0..c0e10aa550 100644 --- a/vendor/yunion.io/x/pkg/prettytable/prettytable.go +++ b/vendor/yunion.io/x/pkg/prettytable/prettytable.go @@ -2,6 +2,7 @@ package prettytable import ( "bytes" + "strings" ) type AlignmentType uint8 @@ -40,6 +41,7 @@ func rowLine(buf *bytes.Buffer, widths []int) { buf.WriteByte('-') buf.WriteByte('+') } + buf.WriteByte('\n') } func textCell(buf *bytes.Buffer, col ptColumn, width int) { @@ -65,13 +67,45 @@ func textCell(buf *bytes.Buffer, col ptColumn, width int) { } func textLine(buf *bytes.Buffer, columns []ptColumn, widths []int) { - buf.WriteByte('|') - for i, w := range widths { - buf.WriteByte(' ') - textCell(buf, columns[i], w) - buf.WriteByte(' ') - buf.WriteByte('|') + nlines := 0 + splitted := make([][]string, len(columns)) + for i, column := range columns { + title := strings.TrimRight(column.Title, "\n") + s := strings.Split(title, "\n") + splitted[i] = s + if nlines < len(s) { + nlines = len(s) + } } + for j := 0; j < nlines; j++ { + buf.WriteByte('|') + for i, w := range widths { + buf.WriteByte(' ') + var text string + if j < len(splitted[i]) { + text = splitted[i][j] + } + c := ptColumn{ + Align: columns[i].Align, + Title: text, + } + textCell(buf, c, w) + buf.WriteByte(' ') + buf.WriteByte('|') + } + buf.WriteByte('\n') + } +} + +func cellWidth(cell string) int { + width := 0 + lines := strings.Split(cell, "\n") + for _, line := range lines { + if width < len(line) { + width = len(line) + } + } + return width } func (this *PrettyTable) GetString(fields [][]string) string { @@ -80,37 +114,32 @@ func (this *PrettyTable) GetString(fields [][]string) string { } var widths = make([]int, len(this.columns)) for i, c := range this.columns { - if widths[i] < len(c.Title) { - widths[i] = len(c.Title) - } + widths[i] = cellWidth(c.Title) } for _, line := range fields { for i, cell := range line { - if widths[i] < len(cell) { - widths[i] = len(cell) + cw := cellWidth(cell) + if widths[i] < cw { + widths[i] = cw } } } var columns = make([]ptColumn, len(this.columns)) var buf bytes.Buffer rowLine(&buf, widths) - buf.WriteByte('\n') for i := 0; i < len(columns); i++ { columns[i].Title = this.columns[i].Title columns[i].Align = AlignCenter } textLine(&buf, columns, widths) - buf.WriteByte('\n') rowLine(&buf, widths) for _, line := range fields { for i := 0; i < len(line); i++ { columns[i].Title = line[i] columns[i].Align = this.columns[i].Align } - buf.WriteByte('\n') textLine(&buf, columns, widths) } - buf.WriteByte('\n') rowLine(&buf, widths) return buf.String() } diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 73cd0b8dfc..d99ef0b95e 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -7,6 +7,7 @@ import ( "strings" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/utils" ) @@ -20,7 +21,6 @@ func (ts *STableSpec) prepareUpdate(dt interface{}) (*SUpdateSession, error) { if reflect.ValueOf(dt).Kind() != reflect.Ptr { return nil, fmt.Errorf("Update input must be a Pointer") } - dataType := reflect.TypeOf(dt).Elem() dataValue := reflect.ValueOf(dt).Elem() fields := reflectutils.FetchStructFieldNameValueInterfaces(dataValue) // fetchStructFieldNameValue(dataType, dataValue) @@ -41,8 +41,7 @@ func (ts *STableSpec) prepareUpdate(dt interface{}) (*SUpdateSession, error) { strings.Join(zeroPrimary, ","), strings.Join(zeroKeyIndex, ",")) } - originValue := reflect.Indirect(reflect.New(dataType)) - originValue.Set(dataValue) + originValue := gotypes.DeepCopyRv(dataValue) us := SUpdateSession{oValue: originValue, tableSpec: ts} return &us, nil } From b95af3dc79e418a6d33b16835b43b49d958383ba Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 21 Aug 2018 17:39:55 +0800 Subject: [PATCH 23/71] climc: helm api --- cmd/climc/shell/k8s/charts.go | 63 ++++++++ cmd/climc/shell/k8s/helm_list.go | 51 +++++++ cmd/climc/shell/k8s/k8s.go | 3 + cmd/climc/shell/k8s/releases.go | 177 +++++++++++++++++++++++ cmd/climc/shell/k8s/tiller.go | 56 +++++++ pkg/mcclient/modules/k8s/helm_chart.go | 16 ++ pkg/mcclient/modules/k8s/helm_release.go | 23 +++ pkg/mcclient/modules/k8s/helm_tiller.go | 23 +++ 8 files changed, 412 insertions(+) create mode 100644 cmd/climc/shell/k8s/charts.go create mode 100644 cmd/climc/shell/k8s/helm_list.go create mode 100644 cmd/climc/shell/k8s/releases.go create mode 100644 cmd/climc/shell/k8s/tiller.go create mode 100644 pkg/mcclient/modules/k8s/helm_chart.go create mode 100644 pkg/mcclient/modules/k8s/helm_release.go create mode 100644 pkg/mcclient/modules/k8s/helm_tiller.go diff --git a/cmd/climc/shell/k8s/charts.go b/cmd/climc/shell/k8s/charts.go new file mode 100644 index 0000000000..8cb609642a --- /dev/null +++ b/cmd/climc/shell/k8s/charts.go @@ -0,0 +1,63 @@ +package k8s + +import ( + "fmt" + + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initChart() { + cmdN := func(suffix string) string { + return resourceCmdN("chart", suffix) + } + + type listOpt struct { + baseListOptions + Name string `help:"Chart name"` + Repo string `help:"Repository name"` + RepoUrl string `help:"Repository url"` + AllVersion bool `json:"Get Chart all history versions"` + Keyword string `json:"Chart keyword"` + } + R(&listOpt{}, cmdN("list"), "List k8s helm global charts", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchPagingParams(args.baseListOptions) + if len(args.Name) != 0 { + params.Add(json.NewString(args.Name), "name") + } + if len(args.Repo) != 0 { + params.Add(json.NewString(args.Repo), "repo") + } + if len(args.RepoUrl) != 0 { + params.Add(json.NewString(args.RepoUrl), "repo_url") + } + if args.AllVersion { + params.Add(json.JSONTrue, "all_version") + } + if len(args.Keyword) != 0 { + params.Add(json.NewString(args.Keyword), "keyword") + } + charts, err := k8s.Charts.List(s, params) + if err != nil { + return err + } + PrintHelmListResult(charts, ChartPrinter{}) + return nil + }) +} + +type ChartPrinter struct{} + +func (p ChartPrinter) Columns() []interface{} { + return []interface{}{"REPO/NAME", "VERSION", "DESCRIPTION"} +} + +func (p ChartPrinter) RowKeys(obj *json.JSONDict) []interface{} { + repo, _ := obj.GetString("repo") + name, _ := obj.GetString("chart", "metadata", "name") + version, _ := obj.GetString("chart", "metadata", "version") + desc, _ := obj.GetString("chart", "metadata", "description") + return []interface{}{fmt.Sprintf("%s/%s", repo, name), version, desc} +} diff --git a/cmd/climc/shell/k8s/helm_list.go b/cmd/climc/shell/k8s/helm_list.go new file mode 100644 index 0000000000..df1055a29e --- /dev/null +++ b/cmd/climc/shell/k8s/helm_list.go @@ -0,0 +1,51 @@ +package k8s + +import ( + "fmt" + + "github.com/gosuri/uitable" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type ListPrinter interface { + Columns() []interface{} + RowKeys(obj *jsonutils.JSONDict) []interface{} +} + +func ListerTable(res *modules.ListResult, printer ListPrinter) *uitable.Table { + min := func(x, y int) int { + if x < y { + return x + } + return y + } + table := uitable.New() + table.MaxColWidth = 80 + table.AddRow(printer.Columns()...) + var idx int + for ; idx < min(res.Limit, res.Total); idx++ { + table.AddRow(printer.RowKeys(res.Data[idx].(*jsonutils.JSONDict))...) + } + return table +} + +func PrintHelmListResult(res *modules.ListResult, printer ListPrinter) { + fmt.Println(ListerTable(res, printer)) + + table := uitable.New() + total := res.Total + offset := res.Offset + limit := res.Limit + page := (offset / limit) + 1 + pages := total / limit + if pages*limit < total { + pages += 1 + } + table.AddRow("") + table.AddRow("Total", "Pages", "Limit", "Offset", "Page") + table.AddRow(total, pages, limit, offset, page) + fmt.Println(table) +} diff --git a/cmd/climc/shell/k8s/k8s.go b/cmd/climc/shell/k8s/k8s.go index af4892e71a..d7b46f482a 100644 --- a/cmd/climc/shell/k8s/k8s.go +++ b/cmd/climc/shell/k8s/k8s.go @@ -17,7 +17,10 @@ func init() { initNode() // helm resources + initTiller() initRepo() + initChart() + initRelease() // kubernetes original resources initRaw() diff --git a/cmd/climc/shell/k8s/releases.go b/cmd/climc/shell/k8s/releases.go new file mode 100644 index 0000000000..179fb199b4 --- /dev/null +++ b/cmd/climc/shell/k8s/releases.go @@ -0,0 +1,177 @@ +package k8s + +import ( + "fmt" + "io/ioutil" + + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initRelease() { + cmdN := func(suffix string) string { + return resourceCmdN("release", suffix) + } + type listOpt struct { + namespaceListOptions + baseListOptions + Filter string `help:"Filter, split by space"` + Admin bool `help:"Admin to show all namespace releases"` + Deployed bool `help:"Show deployed status releases"` + Deleted bool `help:"Show deleted status releases"` + Deleting bool `help:"Show deleting status releases"` + Failed bool `help:"Show failed status releases"` + Superseded bool `help:"Show superseded status releases"` + Pending bool `help:"Show pending status releases"` + } + R(&listOpt{}, cmdN("list"), "List k8s cluster helm releases", func(s *mcclient.ClientSession, args *listOpt) error { + params := fetchNamespaceParams(args.namespaceListOptions) + params.Update(fetchPagingParams(args.baseListOptions)) + if args.Filter != "" { + params.Add(json.NewString(args.Filter), "filter") + } + if args.Namespace != "" { + params.Add(json.NewString(args.Namespace), "namespace") + } + params.Add(json.JSONTrue, "all") + if args.Admin { + params.Add(json.JSONTrue, "admin") + } + if args.Deployed { + params.Add(json.JSONTrue, "deployed") + } + if args.Deleted { + params.Add(json.JSONTrue, "deleted") + } + if args.Deleting { + params.Add(json.JSONTrue, "deleting") + } + if args.Failed { + params.Add(json.JSONTrue, "failed") + } + if args.Superseded { + params.Add(json.JSONTrue, "superseded") + } + if args.Pending { + params.Add(json.JSONTrue, "pending") + } + ret, err := k8s.Releases.ListInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printList(ret, k8s.Releases.GetColumns(s)) + return nil + }) + + type showOpt struct { + clusterBaseOptions + NAME string `help:"Release instance name"` + } + R(&showOpt{}, cmdN("show"), "Get helm release details", func(s *mcclient.ClientSession, args *showOpt) error { + ret, err := k8s.Releases.GetInContexts(s, args.NAME, nil, args.ClusterContext()) + if err != nil { + return err + } + resources, err := ret.GetString("info", "status", "resources") + if err != nil { + return err + } + fmt.Println(resources) + return nil + }) + + type releaseCUOpts struct { + Values string `help:"Specify values in a YAML file (can specify multiple)" short-token:"f"` + Version string `help:"Specify the exact chart version to install. If not specified, latest version installed"` + //Set []string `help:"set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)"` + DryRun bool `help:"Simulate an install"` + Details bool `help:"Show release deploy details, include kubernetes created resources"` + Timeout int64 `help:"Time in seconds to wait for any individual kubernetes operation (like Jobs for hooks)" default:"600"` + } + releaseCUDict := func(args releaseCUOpts) (*json.JSONDict, error) { + params := json.NewDict() + if args.Version != "" { + params.Add(json.NewString(args.Version), "version") + } + if args.DryRun { + params.Add(json.JSONTrue, "dry_run") + } + params.Add(json.NewInt(args.Timeout), "timeout") + if args.Values != "" { + //vals, err := helm.MergeValuesF(args.Values, args.Set, []string{}) + vals, err := ioutil.ReadFile(args.Values) + if err != nil { + return nil, err + } + params.Add(json.NewString(string(vals)), "values") + } + return params, nil + } + + type releaseCreateOpts struct { + namespaceOptions + releaseCUOpts + Name string `help:"Release name, If unspecified, it will autogenerate one for you"` + CHARTNAME string `help:"Helm chart name, e.g stable/etcd"` + } + R(&releaseCreateOpts{}, cmdN("create"), "Create release with specified helm chart", func(s *mcclient.ClientSession, args *releaseCreateOpts) error { + params, err := releaseCUDict(args.releaseCUOpts) + if err != nil { + return err + } + params.Add(json.NewString(args.CHARTNAME), "chart_name") + if args.Namespace != "" { + params.Add(json.NewString(args.Namespace), "namespace") + } + if args.Name != "" { + params.Add(json.NewString(args.Name), "release_name") + } + ret, err := k8s.Releases.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type releaseUpgradeOpts struct { + clusterBaseOptions + releaseCUOpts + NAME string `help:"Release instance name"` + CHARTNAME string `help:"Helm chart name, e.g stable/etcd"` + ReuseValues bool `help:"When upgrading, reuse the last release's values, and merge in any new values. If '--reset-values' is specified, this is ignored"` + ResetValues bool `help:"When upgrading, reset the values to the ones built into the chart"` + } + R(&releaseUpgradeOpts{}, cmdN("upgrade"), "Upgrade release", func(s *mcclient.ClientSession, args *releaseUpgradeOpts) error { + params, err := releaseCUDict(args.releaseCUOpts) + if err != nil { + return err + } + params.Add(json.NewString(args.CHARTNAME), "chart_name") + params.Add(json.NewString(args.NAME), "release_name") + if args.ReuseValues { + params.Add(json.JSONTrue, "reuse_values") + } + if args.ResetValues { + params.Add(json.JSONTrue, "reset_values") + } + + res, err := k8s.Releases.PutInContexts(s, args.NAME, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(res) + return nil + }) + + type deleteOpt struct { + clusterBaseOptions + NAME string `help:"Release instance name"` + } + R(&deleteOpt{}, cmdN("delete"), "Delete release", func(s *mcclient.ClientSession, args *deleteOpt) error { + _, err := k8s.Releases.DeleteInContexts(s, args.NAME, nil, args.ClusterContext()) + return err + }) +} diff --git a/cmd/climc/shell/k8s/tiller.go b/cmd/climc/shell/k8s/tiller.go new file mode 100644 index 0000000000..247e5fba13 --- /dev/null +++ b/cmd/climc/shell/k8s/tiller.go @@ -0,0 +1,56 @@ +package k8s + +import ( + json "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" +) + +func initTiller() { + cmdN := func(suffix string) string { + return resourceCmdN("tiller", suffix) + } + type createOpt struct { + clusterBaseOptions + KubeContext string `json:"kube_context"` + Namespace string `json:"namespace" default:"kube-system"` + // Upgrade if Tiller is already installed + Upgrade bool `json:"upgrade"` + // Name of service account + ServiceAccount string `json:"service_account" default:"tiller"` + // Use the canary Tiller image + Canary bool `json:"canary_image"` + + // Override Tiller image + Image string `json:"tiller_image" default:"yunion/tiller:v2.9.0"` + // Limit the maximum number of revisions saved per release. Use 0 for no limit. + MaxHistory int `json:"history_max"` + } + R(&createOpt{}, cmdN("create"), "Install helm tiller server to Kubernetes cluster", func(s *mcclient.ClientSession, args *createOpt) error { + params := json.NewDict() + if len(args.KubeContext) > 0 { + params.Add(json.NewString(args.KubeContext), "kube_context") + } + params.Add(json.NewString(args.Namespace), "namespace") + params.Add(json.NewString(args.ServiceAccount), "service_account") + if args.Canary { + params.Add(json.JSONTrue, "canary_image") + } + if args.Upgrade { + params.Add(json.JSONTrue, "upgrade") + } + if len(args.Image) > 0 { + params.Add(json.NewString(args.Image), "tiller_image") + } + if args.MaxHistory > 0 { + params.Add(json.NewInt(int64(args.MaxHistory)), "history_max") + } + ret, err := k8s.Tiller.CreateInContexts(s, params, args.ClusterContext()) + if err != nil { + return err + } + printObject(ret) + return nil + }) +} diff --git a/pkg/mcclient/modules/k8s/helm_chart.go b/pkg/mcclient/modules/k8s/helm_chart.go new file mode 100644 index 0000000000..522628793f --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_chart.go @@ -0,0 +1,16 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Charts *modules.ResourceManager +) + +func init() { + Charts = NewManager("chart", "charts", + NewResourceCols(), + NewColumns()) + modules.Register(Charts) +} diff --git a/pkg/mcclient/modules/k8s/helm_release.go b/pkg/mcclient/modules/k8s/helm_release.go new file mode 100644 index 0000000000..ef254c0a45 --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_release.go @@ -0,0 +1,23 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Releases *ReleaseManager +) + +type ReleaseManager struct { + modules.ResourceManager +} + +func init() { + Releases = &ReleaseManager{ + ResourceManager: *NewManager( + "release", "releases", + NewNamespaceCols(""), + NewColumns(), + )} + modules.Register(Releases) +} diff --git a/pkg/mcclient/modules/k8s/helm_tiller.go b/pkg/mcclient/modules/k8s/helm_tiller.go new file mode 100644 index 0000000000..ddb442561f --- /dev/null +++ b/pkg/mcclient/modules/k8s/helm_tiller.go @@ -0,0 +1,23 @@ +package k8s + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + Tiller *TillerManager +) + +type TillerManager struct { + modules.ResourceManager +} + +func init() { + Tiller = &TillerManager{ + ResourceManager: *NewManager( + "tiller", "tiller", + NewColumns(), + NewColumns(), + )} + modules.Register(Tiller) +} From 295b9d8fae6b21649bc2f654cb48a01b23421a9b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 21 Aug 2018 20:38:22 +0800 Subject: [PATCH 24/71] vendor update --- Gopkg.lock | 341 ++++++++++++++++-- .../services/push/push_notice_to_android.go | 110 ------ .../services/push/push_notice_toi_os.go | 111 ------ vendor/github.com/gosuri/uitable/.travis.yml | 8 + vendor/github.com/gosuri/uitable/LICENSE | 10 + vendor/github.com/gosuri/uitable/Makefile | 4 + vendor/github.com/gosuri/uitable/README.md | 67 ++++ vendor/github.com/gosuri/uitable/table.go | 203 +++++++++++ .../gosuri/uitable/util/strutil/strutil.go | 76 ++++ .../gosuri/uitable/util/wordwrap/LICENSE.md | 21 ++ .../gosuri/uitable/util/wordwrap/README.md | 39 ++ .../gosuri/uitable/util/wordwrap/wordwrap.go | 84 +++++ .../govmomi/simulator/license_manager.go | 156 -------- 13 files changed, 824 insertions(+), 406 deletions(-) delete mode 100644 vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go delete mode 100644 vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go create mode 100644 vendor/github.com/gosuri/uitable/.travis.yml create mode 100644 vendor/github.com/gosuri/uitable/LICENSE create mode 100644 vendor/github.com/gosuri/uitable/Makefile create mode 100644 vendor/github.com/gosuri/uitable/README.md create mode 100644 vendor/github.com/gosuri/uitable/table.go create mode 100644 vendor/github.com/gosuri/uitable/util/strutil/strutil.go create mode 100644 vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md create mode 100644 vendor/github.com/gosuri/uitable/util/wordwrap/README.md create mode 100644 vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go delete mode 100644 vendor/github.com/vmware/govmomi/simulator/license_manager.go diff --git a/Gopkg.lock b/Gopkg.lock index 8b923166b6..2499f89c7f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,22 +2,27 @@ [[projects]] + digest = "1:f7a3877d7116adfcdcd641f9f6ad18fc2126bd5c080d8854cc3b395a01e0b7f3" name = "github.com/DataDog/dd-trace-go" packages = [ "opentracing", "tracer", - "tracer/ext" + "tracer/ext", ] + pruneopts = "UT" revision = "27617015d45e6cd550b9a7ac7715c37cc2f7d020" version = "v0.6.1" [[projects]] + digest = "1:66b8ed452b31eb9075bc53295952487c333a9cb555de57ed61f5664c058d9050" name = "github.com/Shopify/sarama" packages = ["."] + pruneopts = "UT" revision = "35324cf48e33d8260e1c7c18854465a904ade249" version = "v1.17.0" [[projects]] + digest = "1:37a8ee6652ff3c42043fa26c0930858d1a72f26928ae26e5261fe81c5f0c2e8d" name = "github.com/aliyun/alibaba-cloud-sdk-go" packages = [ "sdk", @@ -28,47 +33,61 @@ "sdk/errors", "sdk/requests", "sdk/responses", - "sdk/utils" + "sdk/utils", ] + pruneopts = "UT" revision = "f53c66e9cf53d68bcf063643c09cc667efa0e613" version = "1.23.3" [[projects]] + digest = "1:fff66556e25e676b296f9f2c78853831911e6ae41d84303a9be7a534bd1c0901" name = "github.com/aliyun/aliyun-oss-go-sdk" packages = ["oss"] + pruneopts = "UT" revision = "36bf7aa2f916f274137aedd05d1d36f6a3c0f4a6" version = "1.9.0" [[projects]] + digest = "1:8f5416c7f59da8600725ae1ff00a99af1da8b04c211ae6f3c8f8bcab0164f650" name = "github.com/aokoli/goutils" packages = ["."] + pruneopts = "UT" revision = "3391d3790d23d03408670993e957e8f408993c34" version = "v1.0.1" [[projects]] + digest = "1:eee9386329f4fcdf8d6c0def0c9771b634bdd5ba460d888aa98c17d59b37a76c" name = "github.com/apache/thrift" packages = ["lib/go/thrift"] + pruneopts = "UT" revision = "e59b73d3c2bf1c328ccb78e683c0462fa1a473c7" [[projects]] branch = "master" + digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" packages = ["quantile"] + pruneopts = "UT" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] + digest = "1:b520b55fc1146c5b0eea03b07233f7a3d4a9be985c037c91ea6b82ecb81bd521" name = "github.com/bitly/go-simplejson" packages = ["."] + pruneopts = "UT" revision = "aabad6e819789e569bd6aabf444c935aa9ba1e44" version = "v0.5.0" [[projects]] + digest = "1:a2486f94bda33b301614ff5cfce102b7eed4c923fa650c2b83dba1fcf2117466" name = "github.com/c-bata/go-prompt" packages = ["."] + pruneopts = "UT" revision = "c52492ff1b386e5c0ba5271b5eaad165fab09eca" version = "v0.2.2" [[projects]] + digest = "1:40098afbdd06a76dee4b6bcb85a22fed81ac9d2ebaf775c91e558c80f57aaff1" name = "github.com/coredns/coredns" packages = [ "core/dnsserver", @@ -118,108 +137,140 @@ "plugin/reload", "plugin/test", "plugin/trace", - "request" + "request", ] + pruneopts = "UT" revision = "2e322f6e8a54f18c6aef9c25a7c432c291a3d9f7" version = "v1.2.0" [[projects]] + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:737260890e4b00d73a76993eac1a772cf6a57292f723c47e26f005821efcabcf" name = "github.com/dnstap/golang-dnstap" packages = ["."] + pruneopts = "UT" revision = "2cf77a2b5e11ac8d0ba3892772ac8e1f7b528344" [[projects]] + digest = "1:1f0c7ab489b407a7f8f9ad16c25a504d28ab461517a971d341388a56156c1bd7" name = "github.com/eapache/go-resiliency" packages = ["breaker"] + pruneopts = "UT" revision = "ea41b0fad31007accc7f806884dcdf3da98b79ce" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:0448d1c1a596941c608762912fe7c865f88d9ffa45afb8af1edcf1401762bf7e" name = "github.com/eapache/go-xerial-snappy" packages = ["."] + pruneopts = "UT" revision = "040cc1a32f578808623071247fdbd5cc43f37f5f" [[projects]] + digest = "1:444b82bfe35c83bbcaf84e310fb81a1f9ece03edfed586483c869e2c046aef69" name = "github.com/eapache/queue" packages = ["."] + pruneopts = "UT" revision = "44cc805cf13205b55f69e14bcb69867d1ae92f98" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:a76c5e933af8688d9dcfbe718f2b0bfc955208fe0e47e74218b45f1df83ef2ae" name = "github.com/farsightsec/golang-framestream" packages = ["."] + pruneopts = "UT" revision = "c06a5734334d9629b3db143d74b47eb94ea68612" [[projects]] + digest = "1:865079840386857c809b72ce300be7580cb50d3d3129ce11bf9aa6ca2bc1934a" name = "github.com/fatih/color" packages = ["."] + pruneopts = "UT" revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" [[projects]] branch = "master" + digest = "1:50a46ab1d5edbbdd55125b4d37f1bf503d0807c26461f9ad7b358d6006641d09" name = "github.com/flynn/go-shlex" packages = ["."] + pruneopts = "UT" revision = "3f9db97f856818214da2e1057f8ad84803971cff" [[projects]] + digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" name = "github.com/ghodss/yaml" packages = ["."] + pruneopts = "UT" revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:36fe9527deed01d2a317617e59304eb2c4ce9f8a24115bcc5c2e37b3aee5bae4" name = "github.com/gin-contrib/sse" packages = ["."] + pruneopts = "UT" revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae" [[projects]] + digest = "1:ab00ecdc9607692ffd5fb1ac21a9ba2d6cc95bae011f327022c0b1c323d218e2" name = "github.com/gin-gonic/gin" packages = [ "binding", - "render" + "render", ] + pruneopts = "UT" revision = "d459835d2b077e44f7c9b453505ee29881d5d12d" version = "v1.2" [[projects]] + digest = "1:31a18dae27a29aa074515e43a443abfd2ba6deb6d69309d8d7ce789c45f34659" name = "github.com/go-logfmt/logfmt" packages = ["."] + pruneopts = "UT" revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" [[projects]] + digest = "1:adea5a94903eb4384abef30f3d878dc9ff6b6b5b0722da25b82e5169216dfb61" name = "github.com/go-sql-driver/mysql" packages = ["."] + pruneopts = "UT" revision = "d523deb1b23d913de5bdada721a6071e71283618" version = "v1.4.0" [[projects]] + digest = "1:34e709f36fd4f868fb00dbaf8a6cab4c1ae685832d392874ba9d7c5dec2429d1" name = "github.com/gogo/protobuf" packages = [ "proto", - "sortkeys" + "sortkeys", ] + pruneopts = "UT" revision = "636bf0302bc95575d69441b25a2603156ffdddf1" version = "v1.1.1" [[projects]] + digest = "1:94d8c9e3a4954505ca89243e2121180972eecf8e3d160f3b34a48825ddd95cc1" name = "github.com/golang-plus/errors" packages = ["."] + pruneopts = "UT" revision = "a61c34ead85085f3e1b87b8c04e49bbf60c99fb1" version = "v1.0.0" [[projects]] + digest = "1:224bf08a201edf15ec9f086549321257d5a2ee0d23dc649e7e6ad90ce5502439" name = "github.com/golang-plus/uuid" packages = [ ".", @@ -229,205 +280,275 @@ "internal/namebased/md5", "internal/namebased/sha1", "internal/random", - "internal/timebased" + "internal/timebased", ] + pruneopts = "UT" revision = "a0510c37ef1f49c8290c3a9911d0f7f9ed24d335" version = "v1.0.0" [[projects]] branch = "master" + digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" name = "github.com/golang/glog" packages = ["."] + pruneopts = "UT" revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] + digest = "1:17fe264ee908afc795734e8c4e63db2accabaf57326dbf21763a7d6b86096260" name = "github.com/golang/protobuf" packages = [ "proto", "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/timestamp" + "ptypes/timestamp", ] + pruneopts = "UT" revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:4a0c6bb4805508a6287675fac876be2ac1182539ca8a32468d8128882e9d5009" name = "github.com/golang/snappy" packages = ["."] + pruneopts = "UT" revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" [[projects]] branch = "master" + digest = "1:9887333bbef17574b1db5f9893ea137ac44107235d624408a3ac9e0b98fbb2cb" name = "github.com/google/btree" packages = ["."] + pruneopts = "UT" revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" [[projects]] branch = "master" + digest = "1:3ee90c0d94da31b442dde97c99635aaafec68d0b8a3c12ee2075c6bdabeec6bb" name = "github.com/google/gofuzz" packages = ["."] + pruneopts = "UT" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] + digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" name = "github.com/google/uuid" packages = ["."] + pruneopts = "UT" revision = "064e2069ce9c359c118179501254f67d7d37ba24" version = "0.2" [[projects]] + digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", "compiler", - "extensions" + "extensions", ] + pruneopts = "UT" revision = "7c663266750e7d82587642f65e60bc4083f1f84e" version = "v0.2.0" [[projects]] branch = "master" + digest = "1:4e08dc2383a46b3107f0b34ca338c4459e8fc8ee90e46a60e728aa8a2b21d558" + name = "github.com/gosuri/uitable" + packages = [ + ".", + "util/strutil", + "util/wordwrap", + ] + pruneopts = "UT" + revision = "36ee7e946282a3fb1cfecd476ddc9b35d8847e42" + +[[projects]] + branch = "master" + digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" name = "github.com/gregjones/httpcache" packages = [ ".", - "diskcache" + "diskcache", ] + pruneopts = "UT" revision = "9cad4c3443a7200dd6400aef47183728de563a38" [[projects]] branch = "master" + digest = "1:1a1206efd03a54d336dce7bb8719e74f2f8932f661cb9f57d5813a1d99c083d8" name = "github.com/grpc-ecosystem/grpc-opentracing" packages = ["go/otgrpc"] + pruneopts = "UT" revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746" [[projects]] + digest = "1:8eb1de8112c9924d59bf1d3e5c26f5eaa2bfc2a5fcbb92dc1c2e4546d695f277" name = "github.com/imdario/mergo" packages = ["."] + pruneopts = "UT" revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" version = "v0.3.6" [[projects]] + digest = "1:b6f4477f09f11785097a121811f3a90d8d26cab73c1aa71abda859bdf6ad2334" name = "github.com/jinzhu/gorm" packages = [ ".", - "dialects/mysql" + "dialects/mysql", ] + pruneopts = "UT" revision = "6ed508ec6a4ecb3531899a69cbc746ccf65a4166" version = "v1.9.1" [[projects]] branch = "master" + digest = "1:fd97437fbb6b7dce04132cf06775bd258cce305c44add58eb55ca86c6c325160" name = "github.com/jinzhu/inflection" packages = ["."] + pruneopts = "UT" revision = "04140366298a54a039076d798123ffa108fff46c" [[projects]] + digest = "1:b87714e57a511d88f307aba7d5b63522da12bed0a050889c81272fc50f71100e" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "UT" revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9" version = "0.2.2" [[projects]] + digest = "1:eaefc85d32c03e5f0c2b88ea2f79fce3d993e2c78316d21319575dd4ea9153ca" name = "github.com/json-iterator/go" packages = ["."] + pruneopts = "UT" revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82" version = "1.1.4" [[projects]] branch = "master" + digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72" name = "github.com/kr/logfmt" packages = ["."] + pruneopts = "UT" revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" [[projects]] + digest = "1:c658e84ad3916da105a761660dcaeb01e63416c8ec7bc62256a9b411a05fcd67" name = "github.com/mattn/go-colorable" packages = ["."] + pruneopts = "UT" revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" version = "v0.0.9" [[projects]] + digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb" name = "github.com/mattn/go-isatty" packages = ["."] + pruneopts = "UT" revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" [[projects]] branch = "master" + digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033" name = "github.com/mattn/go-runewidth" packages = ["."] + pruneopts = "UT" revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb" [[projects]] branch = "master" + digest = "1:d775613e6db40f54d34cc8b319ee5c2df56752f9f5e55e2f6004d11684bdc8fd" name = "github.com/mattn/go-tty" packages = ["."] + pruneopts = "UT" revision = "931426f7535ac39720c8909d70ece5a41a2502a6" [[projects]] + digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "UT" revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" version = "v1.0.1" [[projects]] branch = "master" + digest = "1:2b32af4d2a529083275afc192d1067d8126b578c7a9613b26600e4df9c735155" name = "github.com/mgutz/ansi" packages = ["."] + pruneopts = "UT" revision = "9520e82c474b0a04dd04f8a40959027271bab992" [[projects]] + digest = "1:2e9ce249cbffeaf99b7b8d13b2191591bda8421f416e1366bdde8f7785f034cd" name = "github.com/mholt/caddy" packages = [ ".", "caddyfile", "onevent/hook", - "startupshutdown" + "startupshutdown", ] + pruneopts = "UT" revision = "d3f338ddab9ee24b376b5c9c51e202581e2d43ba" version = "v0.10.11" [[projects]] + digest = "1:463e4140189f8194f9121ca1c7fe3b8e9e9a2ab3d949b43c835c21034927dc62" name = "github.com/miekg/dns" packages = ["."] + pruneopts = "UT" revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" version = "v1.0.8" [[projects]] + digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" packages = ["."] + pruneopts = "UT" revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" version = "1.0.3" [[projects]] + digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" name = "github.com/modern-go/reflect2" packages = ["."] + pruneopts = "UT" revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" [[projects]] branch = "master" + digest = "1:7aefb397a53fc437c90f0fdb3e1419c751c5a3a165ced52325d5d797edf1aca6" name = "github.com/moul/http2curl" packages = ["."] + pruneopts = "UT" revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d" [[projects]] branch = "master" + digest = "1:2da0e5077ed40453dc281b9a2428d84cf6ad14063aed189f6296ca5dd25cf13d" name = "github.com/opentracing-contrib/go-observer" packages = ["."] + pruneopts = "UT" revision = "a52f2342449246d5bcc273e65cbdcfa5f7d6c63c" [[projects]] + digest = "1:450b7623b185031f3a456801155c8320209f75d0d4c4e633c6b1e59d44d6e392" name = "github.com/opentracing/opentracing-go" packages = [ ".", "ext", - "log" + "log", ] + pruneopts = "UT" revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" version = "v1.0.2" [[projects]] + digest = "1:a427dd5035907bf7ad9a1fa269805f3a9ead9cfab056b32e2d86c230e28e31c0" name = "github.com/openzipkin/zipkin-go-opentracing" packages = [ ".", @@ -435,117 +556,149 @@ "thrift/gen-go/scribe", "thrift/gen-go/zipkincore", "types", - "wire" + "wire", ] + pruneopts = "UT" revision = "26cf9707480e6b90e5eff22cf0bbf05319154232" version = "v0.3.4" [[projects]] branch = "master" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" name = "github.com/petar/GoLLRB" packages = ["llrb"] + pruneopts = "UT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] + digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" name = "github.com/peterbourgon/diskv" packages = ["."] + pruneopts = "UT" revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" [[projects]] + digest = "1:29803f52611cbcc1dfe55b456e9fdac362af7248b3d29d7ea1bec0a12e71dff4" name = "github.com/pierrec/lz4" packages = [ ".", - "internal/xxh32" + "internal/xxh32", ] + pruneopts = "UT" revision = "1958fd8fff7f115e79725b1288e0b878b3e06b00" version = "v2.0.3" [[projects]] branch = "master" + digest = "1:4f43b3c1b7e44980a5f3c593f8bf0e18844dc44f451a071c93e77e28cf990db6" name = "github.com/pkg/term" packages = ["termios"] + pruneopts = "UT" revision = "cda20d4ac917ad418d86e151eff439648b06185b" [[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:d14a5f4bfecf017cb780bdde1b6483e5deb87e12c332544d2c430eda58734bcb" name = "github.com/prometheus/client_golang" packages = [ "prometheus", - "prometheus/promhttp" + "prometheus/promhttp", ] + pruneopts = "UT" revision = "c5b7fccd204277076155f10851dad72b76a49317" version = "v0.8.0" [[projects]] branch = "master" + digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "UT" revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f" [[projects]] branch = "master" + digest = "1:e469cd65badf7694aeb44874518606d93c1d59e7735d3754ad442782437d3cc3" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", - "model" + "model", ] + pruneopts = "UT" revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" [[projects]] branch = "master" + digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290" name = "github.com/prometheus/procfs" packages = [ ".", "internal/util", "nfs", - "xfs" + "xfs", ] + pruneopts = "UT" revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" [[projects]] branch = "master" + digest = "1:c4556a44e350b50a490544d9b06e9fba9c286c21d6c0e47f54f3a9214597298c" name = "github.com/rcrowley/go-metrics" packages = ["."] + pruneopts = "UT" revision = "e2704e165165ec55d062f5919b4b29494e9fa790" [[projects]] + digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925" name = "github.com/satori/go.uuid" packages = ["."] + pruneopts = "UT" revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" version = "v1.2.0" [[projects]] + digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" name = "github.com/sirupsen/logrus" packages = ["."] + pruneopts = "UT" revision = "3e01752db0189b9157070a0e1668a620f9a85da2" version = "v1.0.6" [[projects]] + digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7" name = "github.com/spf13/pflag" packages = ["."] + pruneopts = "UT" revision = "583c0c0531f06d5278b7d917446061adc344b5cd" version = "v1.0.1" [[projects]] + digest = "1:18752d0b95816a1b777505a97f71c7467a8445b8ffb55631a7bf779f6ba4fa83" name = "github.com/stretchr/testify" packages = ["assert"] + pruneopts = "UT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" [[projects]] + digest = "1:98e5cda86f67cd1ac95389d98670b66dea8cae480fe6292b83bccccfe60b4106" name = "github.com/ugorji/go" packages = ["codec"] + pruneopts = "UT" revision = "f3cacc17c85ecb7f1b6a9e373ee85d1480919868" [[projects]] branch = "master" + digest = "1:91771429c8de08d4d9b32663ad78bfa8ee1e0a146e921d9937c117665fd0377f" name = "github.com/vmware/govmomi" packages = [ ".", @@ -562,12 +715,14 @@ "vim25/progress", "vim25/soap", "vim25/types", - "vim25/xml" + "vim25/xml", ] + pruneopts = "UT" revision = "bbd99532a768d2fe369079ceda730e30726ae1a6" [[projects]] branch = "master" + digest = "1:014ee502e4880ffba2a0103cefa6af78cf8767ce08bc29376e6b86cdcaa2ec06" name = "golang.org/x/crypto" packages = [ "curve25519", @@ -577,12 +732,14 @@ "internal/subtle", "poly1305", "ssh", - "ssh/terminal" + "ssh/terminal", ] + pruneopts = "UT" revision = "c126467f60eb25f8f27e5a981f32a87e3965053f" [[projects]] branch = "master" + digest = "1:c945d925a3dae287abe075a89c6b09e1015da39d350eb512bb5d01ebcd7783e7" name = "golang.org/x/net" packages = [ "bpf", @@ -596,20 +753,24 @@ "internal/timeseries", "ipv4", "ipv6", - "trace" + "trace", ] + pruneopts = "UT" revision = "f4c29de78a2a91c00474a2e689954305c350adf9" [[projects]] branch = "master" + digest = "1:be95d758fc4d9216e5d41ff5b98b46938fba85ca5bc2ddc45a1b03e2bb10fe7c" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", ] + pruneopts = "UT" revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311" [[projects]] + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -625,30 +786,38 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:c9e7a4b4d47c0ed205d257648b0e5b0440880cb728506e318f8ac7cd36270bc4" name = "golang.org/x/time" packages = ["rate"] + pruneopts = "UT" revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" [[projects]] + digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" name = "google.golang.org/appengine" packages = ["cloudsql"] + pruneopts = "UT" revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] + pruneopts = "UT" revision = "daca94659cb50e9f37c1b834680f2e46358f10b0" [[projects]] + digest = "1:3dd7996ce6bf52dec6a2f69fa43e7c4cefea1d4dfa3c8ab7a5f8a9f7434e239d" name = "google.golang.org/grpc" packages = [ ".", @@ -676,36 +845,46 @@ "resolver/passthrough", "stats", "status", - "tap" + "tap", ] + pruneopts = "UT" revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" version = "v1.14.0" [[projects]] + digest = "1:2655728699af00d823b0a68bd7169fd241a30c04252f456b34421908701b9c3f" name = "gopkg.in/gin-gonic/gin.v1" packages = ["."] + pruneopts = "UT" revision = "d459835d2b077e44f7c9b453505ee29881d5d12d" version = "v1.2" [[projects]] + digest = "1:cbc72c4c4886a918d6ab4b95e347ffe259846260f99ebdd8a198c2331cf2b2e9" name = "gopkg.in/go-playground/validator.v8" packages = ["."] + pruneopts = "UT" revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf" version = "v8.18.2" [[projects]] + digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" name = "gopkg.in/inf.v0" packages = ["."] + pruneopts = "UT" revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" version = "v0.9.1" [[projects]] + digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "UT" revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" version = "v2.2.1" [[projects]] + digest = "1:74142cd2275f77547c35ac51514108d9798a09aa0cf377a5c1084718ef7aa225" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -736,11 +915,13 @@ "settings/v1alpha1", "storage/v1", "storage/v1alpha1", - "storage/v1beta1" + "storage/v1beta1", ] + pruneopts = "UT" revision = "072894a440bdee3a891dea811fe42902311cd2a3" [[projects]] + digest = "1:29d5abc33d8cbec19b4cd032a87d28010dd615578dd695a655fc3d5ec3ab86b6" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -777,11 +958,13 @@ "pkg/util/yaml", "pkg/version", "pkg/watch", - "third_party/forked/golang/reflect" + "third_party/forked/golang/reflect", ] + pruneopts = "UT" revision = "103fd098999dc9c0c88536f5c9ad2e5da39373ae" [[projects]] + digest = "1:ac46cc5d5c55dd93d4844327334d2253d563d76f14d4b38a8b522b97f0f52214" name = "k8s.io/client-go" packages = [ "discovery", @@ -835,28 +1018,34 @@ "util/connrotation", "util/flowcontrol", "util/homedir", - "util/integer" + "util/integer", ] + pruneopts = "UT" revision = "7d04d0e2a0a1a4d4a1cd6baa432a2301492e4e65" version = "v8.0.0" [[projects]] branch = "master" + digest = "1:5ede93047a3e04f6ede8fbc011f13ee94c397b727613fd9712cc99ebc47ff906" name = "yunion.io/x/jsonutils" packages = ["."] + pruneopts = "UT" revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" [[projects]] branch = "master" + digest = "1:b707a5d591b21dad9ebdb264281632ed43d34e2b980dce967acb2c7adcc93ad3" name = "yunion.io/x/log" packages = [ ".", - "hooks" + "hooks", ] + pruneopts = "UT" revision = "a10b94c5038480920262e0f344656328434955a2" [[projects]] branch = "master" + digest = "1:13da776b435ec7873c6f9309191296d920a110145372d2a9e1a58ab9e8b34b0d" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -888,25 +1077,119 @@ "util/version", "util/wait", "util/workqueue", - "utils" + "utils", ] + pruneopts = "UT" revision = "56b426f0ca15288cc26442f5802c949f137ade60" [[projects]] branch = "master" + digest = "1:3d3e1ecb41c63448df5d66eeafa5f6da720ca6e32543e89faa82b5bc4b9adebc" name = "yunion.io/x/sqlchemy" packages = ["."] + pruneopts = "UT" revision = "a74ef73e555a1ed2e19b1a47e0567814c289eaf9" [[projects]] branch = "master" + digest = "1:6baf7b4ee14e156a9dd6c5c850bb8201af47665e3267e3cdee26edf2a90eaf73" name = "yunion.io/x/structarg" packages = ["."] + pruneopts = "UT" revision = "d5e5d87357b9bc2164215117763f6b15b2a3e75d" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8b3cd1b8309273551ebd55dfb0cadeaaad0aecc1b3a94d680a4a1769d2f42512" + input-imports = [ + "github.com/aliyun/alibaba-cloud-sdk-go/sdk", + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests", + "github.com/aliyun/aliyun-oss-go-sdk/oss", + "github.com/aokoli/goutils", + "github.com/bitly/go-simplejson", + "github.com/c-bata/go-prompt", + "github.com/coredns/coredns/core/dnsserver", + "github.com/coredns/coredns/coremain", + "github.com/coredns/coredns/plugin", + "github.com/coredns/coredns/plugin/cache", + "github.com/coredns/coredns/plugin/chaos", + "github.com/coredns/coredns/plugin/debug", + "github.com/coredns/coredns/plugin/errors", + "github.com/coredns/coredns/plugin/etcd/msg", + "github.com/coredns/coredns/plugin/file", + "github.com/coredns/coredns/plugin/forward", + "github.com/coredns/coredns/plugin/health", + "github.com/coredns/coredns/plugin/hosts", + "github.com/coredns/coredns/plugin/log", + "github.com/coredns/coredns/plugin/metrics", + "github.com/coredns/coredns/plugin/nsid", + "github.com/coredns/coredns/plugin/pkg/dnsutil", + "github.com/coredns/coredns/plugin/pkg/fall", + "github.com/coredns/coredns/plugin/pkg/upstream", + "github.com/coredns/coredns/plugin/proxy", + "github.com/coredns/coredns/plugin/reload", + "github.com/coredns/coredns/plugin/trace", + "github.com/coredns/coredns/request", + "github.com/fatih/color", + "github.com/go-sql-driver/mysql", + "github.com/golang-plus/uuid", + "github.com/gosuri/uitable", + "github.com/jinzhu/gorm", + "github.com/jinzhu/gorm/dialects/mysql", + "github.com/json-iterator/go", + "github.com/mholt/caddy", + "github.com/mholt/caddy/startupshutdown", + "github.com/miekg/dns", + "github.com/moul/http2curl", + "github.com/stretchr/testify/assert", + "github.com/vmware/govmomi", + "github.com/vmware/govmomi/object", + "github.com/vmware/govmomi/property", + "github.com/vmware/govmomi/session", + "github.com/vmware/govmomi/view", + "github.com/vmware/govmomi/vim25/mo", + "github.com/vmware/govmomi/vim25/types", + "golang.org/x/crypto/ssh", + "gopkg.in/gin-gonic/gin.v1", + "k8s.io/api/core/v1", + "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/fields", + "k8s.io/apimachinery/pkg/labels", + "k8s.io/client-go/kubernetes", + "k8s.io/client-go/rest", + "k8s.io/client-go/tools/clientcmd", + "yunion.io/x/jsonutils", + "yunion.io/x/log", + "yunion.io/x/pkg/gotypes", + "yunion.io/x/pkg/prettytable", + "yunion.io/x/pkg/trace", + "yunion.io/x/pkg/tristate", + "yunion.io/x/pkg/util/cache", + "yunion.io/x/pkg/util/compare", + "yunion.io/x/pkg/util/errors", + "yunion.io/x/pkg/util/fifoutils", + "yunion.io/x/pkg/util/fileutils", + "yunion.io/x/pkg/util/filterclause", + "yunion.io/x/pkg/util/goroutine_pool", + "yunion.io/x/pkg/util/netutils", + "yunion.io/x/pkg/util/osprofile", + "yunion.io/x/pkg/util/prometheus", + "yunion.io/x/pkg/util/reflectutils", + "yunion.io/x/pkg/util/regutils", + "yunion.io/x/pkg/util/seclib", + "yunion.io/x/pkg/util/secrules", + "yunion.io/x/pkg/util/sets", + "yunion.io/x/pkg/util/stringutils", + "yunion.io/x/pkg/util/sysutils", + "yunion.io/x/pkg/util/timeutils", + "yunion.io/x/pkg/util/trace", + "yunion.io/x/pkg/util/ttlpool", + "yunion.io/x/pkg/util/version", + "yunion.io/x/pkg/util/wait", + "yunion.io/x/pkg/util/workqueue", + "yunion.io/x/pkg/utils", + "yunion.io/x/sqlchemy", + "yunion.io/x/structarg", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go deleted file mode 100644 index ef385f3a9f..0000000000 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_to_android.go +++ /dev/null @@ -1,110 +0,0 @@ -package push - -//Licensed under the Apache License, Version 2.0 (the "License"); -//you may not use this file except in compliance with the License. -//You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, software -//distributed under the License is distributed on an "AS IS" BASIS, -//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//See the License for the specific language governing permissions and -//limitations under the License. -// -// Code generated by Alibaba Cloud SDK Code Generator. -// Changes may cause incorrect behavior and will be lost if the code is regenerated. - -import ( - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" -) - -// PushNoticeToAndroid invokes the push.PushNoticeToAndroid API synchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -func (client *Client) PushNoticeToAndroid(request *PushNoticeToAndroidRequest) (response *PushNoticeToAndroidResponse, err error) { - response = CreatePushNoticeToAndroidResponse() - err = client.DoAction(request, response) - return -} - -// PushNoticeToAndroidWithChan invokes the push.PushNoticeToAndroid API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToAndroidWithChan(request *PushNoticeToAndroidRequest) (<-chan *PushNoticeToAndroidResponse, <-chan error) { - responseChan := make(chan *PushNoticeToAndroidResponse, 1) - errChan := make(chan error, 1) - err := client.AddAsyncTask(func() { - defer close(responseChan) - defer close(errChan) - response, err := client.PushNoticeToAndroid(request) - if err != nil { - errChan <- err - } else { - responseChan <- response - } - }) - if err != nil { - errChan <- err - close(responseChan) - close(errChan) - } - return responseChan, errChan -} - -// PushNoticeToAndroidWithCallback invokes the push.PushNoticeToAndroid API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoandroid.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToAndroidWithCallback(request *PushNoticeToAndroidRequest, callback func(response *PushNoticeToAndroidResponse, err error)) <-chan int { - result := make(chan int, 1) - err := client.AddAsyncTask(func() { - var response *PushNoticeToAndroidResponse - var err error - defer close(result) - response, err = client.PushNoticeToAndroid(request) - callback(response, err) - result <- 1 - }) - if err != nil { - defer close(result) - callback(nil, err) - result <- 0 - } - return result -} - -// PushNoticeToAndroidRequest is the request struct for api PushNoticeToAndroid -type PushNoticeToAndroidRequest struct { - *requests.RpcRequest - AppKey requests.Integer `position:"Query" name:"AppKey"` - Target string `position:"Query" name:"Target"` - TargetValue string `position:"Query" name:"TargetValue"` - Title string `position:"Query" name:"Title"` - Body string `position:"Query" name:"Body"` - JobKey string `position:"Query" name:"JobKey"` - ExtParameters string `position:"Query" name:"ExtParameters"` -} - -// PushNoticeToAndroidResponse is the response struct for api PushNoticeToAndroid -type PushNoticeToAndroidResponse struct { - *responses.BaseResponse - RequestId string `json:"RequestId" xml:"RequestId"` - MessageId string `json:"MessageId" xml:"MessageId"` -} - -// CreatePushNoticeToAndroidRequest creates a request to invoke PushNoticeToAndroid API -func CreatePushNoticeToAndroidRequest() (request *PushNoticeToAndroidRequest) { - request = &PushNoticeToAndroidRequest{ - RpcRequest: &requests.RpcRequest{}, - } - request.InitWithApiInfo("Push", "2016-08-01", "PushNoticeToAndroid", "", "") - return -} - -// CreatePushNoticeToAndroidResponse creates a response to parse from PushNoticeToAndroid response -func CreatePushNoticeToAndroidResponse() (response *PushNoticeToAndroidResponse) { - response = &PushNoticeToAndroidResponse{ - BaseResponse: &responses.BaseResponse{}, - } - return -} diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go deleted file mode 100644 index 73d1a588e6..0000000000 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/services/push/push_notice_toi_os.go +++ /dev/null @@ -1,111 +0,0 @@ -package push - -//Licensed under the Apache License, Version 2.0 (the "License"); -//you may not use this file except in compliance with the License. -//You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, software -//distributed under the License is distributed on an "AS IS" BASIS, -//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//See the License for the specific language governing permissions and -//limitations under the License. -// -// Code generated by Alibaba Cloud SDK Code Generator. -// Changes may cause incorrect behavior and will be lost if the code is regenerated. - -import ( - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" -) - -// PushNoticeToiOS invokes the push.PushNoticeToiOS API synchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -func (client *Client) PushNoticeToiOS(request *PushNoticeToiOSRequest) (response *PushNoticeToiOSResponse, err error) { - response = CreatePushNoticeToiOSResponse() - err = client.DoAction(request, response) - return -} - -// PushNoticeToiOSWithChan invokes the push.PushNoticeToiOS API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToiOSWithChan(request *PushNoticeToiOSRequest) (<-chan *PushNoticeToiOSResponse, <-chan error) { - responseChan := make(chan *PushNoticeToiOSResponse, 1) - errChan := make(chan error, 1) - err := client.AddAsyncTask(func() { - defer close(responseChan) - defer close(errChan) - response, err := client.PushNoticeToiOS(request) - if err != nil { - errChan <- err - } else { - responseChan <- response - } - }) - if err != nil { - errChan <- err - close(responseChan) - close(errChan) - } - return responseChan, errChan -} - -// PushNoticeToiOSWithCallback invokes the push.PushNoticeToiOS API asynchronously -// api document: https://help.aliyun.com/api/push/pushnoticetoios.html -// asynchronous document: https://help.aliyun.com/document_detail/66220.html -func (client *Client) PushNoticeToiOSWithCallback(request *PushNoticeToiOSRequest, callback func(response *PushNoticeToiOSResponse, err error)) <-chan int { - result := make(chan int, 1) - err := client.AddAsyncTask(func() { - var response *PushNoticeToiOSResponse - var err error - defer close(result) - response, err = client.PushNoticeToiOS(request) - callback(response, err) - result <- 1 - }) - if err != nil { - defer close(result) - callback(nil, err) - result <- 0 - } - return result -} - -// PushNoticeToiOSRequest is the request struct for api PushNoticeToiOS -type PushNoticeToiOSRequest struct { - *requests.RpcRequest - AppKey requests.Integer `position:"Query" name:"AppKey"` - Target string `position:"Query" name:"Target"` - TargetValue string `position:"Query" name:"TargetValue"` - ApnsEnv string `position:"Query" name:"ApnsEnv"` - Title string `position:"Query" name:"Title"` - Body string `position:"Query" name:"Body"` - JobKey string `position:"Query" name:"JobKey"` - ExtParameters string `position:"Query" name:"ExtParameters"` -} - -// PushNoticeToiOSResponse is the response struct for api PushNoticeToiOS -type PushNoticeToiOSResponse struct { - *responses.BaseResponse - RequestId string `json:"RequestId" xml:"RequestId"` - MessageId string `json:"MessageId" xml:"MessageId"` -} - -// CreatePushNoticeToiOSRequest creates a request to invoke PushNoticeToiOS API -func CreatePushNoticeToiOSRequest() (request *PushNoticeToiOSRequest) { - request = &PushNoticeToiOSRequest{ - RpcRequest: &requests.RpcRequest{}, - } - request.InitWithApiInfo("Push", "2016-08-01", "PushNoticeToiOS", "", "") - return -} - -// CreatePushNoticeToiOSResponse creates a response to parse from PushNoticeToiOS response -func CreatePushNoticeToiOSResponse() (response *PushNoticeToiOSResponse) { - response = &PushNoticeToiOSResponse{ - BaseResponse: &responses.BaseResponse{}, - } - return -} diff --git a/vendor/github.com/gosuri/uitable/.travis.yml b/vendor/github.com/gosuri/uitable/.travis.yml new file mode 100644 index 0000000000..2139b89ef0 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/.travis.yml @@ -0,0 +1,8 @@ +language: go +sudo: false +install: + - go get ./... +go: + - 1.4 + - 1.5.2 + - tip diff --git a/vendor/github.com/gosuri/uitable/LICENSE b/vendor/github.com/gosuri/uitable/LICENSE new file mode 100644 index 0000000000..e436d90439 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/LICENSE @@ -0,0 +1,10 @@ +MIT License +=========== + +Copyright (c) 2015, Greg Osuri + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/gosuri/uitable/Makefile b/vendor/github.com/gosuri/uitable/Makefile new file mode 100644 index 0000000000..60246a8a93 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/Makefile @@ -0,0 +1,4 @@ +test: + @go test ./... + +.PHONY: test diff --git a/vendor/github.com/gosuri/uitable/README.md b/vendor/github.com/gosuri/uitable/README.md new file mode 100644 index 0000000000..683b538d13 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/README.md @@ -0,0 +1,67 @@ +# uitable [![GoDoc](https://godoc.org/github.com/gosuri/uitable?status.svg)](https://godoc.org/github.com/gosuri/uitable) [![Build Status](https://travis-ci.org/gosuri/uitable.svg?branch=master)](https://travis-ci.org/gosuri/uitable) + +uitable is a go library for representing data as tables for terminal applications. It provides primitives for sizing and wrapping columns to improve readability. + +## Example Usage + +Full source code for the example is available at [example/main.go](example/main.go) + +```go +table := uitable.New() +table.MaxColWidth = 50 + +table.AddRow("NAME", "BIRTHDAY", "BIO") +for _, hacker := range hackers { + table.AddRow(hacker.Name, hacker.Birthday, hacker.Bio) +} +fmt.Println(table) +``` + +Will render the data as: + +```sh +NAME BIRTHDAY BIO +Ada Lovelace December 10, 1815 Ada was a British mathematician and writer, chi... +Alan Turing June 23, 1912 Alan was a British pioneering computer scientis... +``` + +For wrapping in two columns: + +```go +table = uitable.New() +table.MaxColWidth = 80 +table.Wrap = true // wrap columns + +for _, hacker := range hackers { + table.AddRow("Name:", hacker.Name) + table.AddRow("Birthday:", hacker.Birthday) + table.AddRow("Bio:", hacker.Bio) + table.AddRow("") // blank +} +fmt.Println(table) +``` + +Will render the data as: + +``` +Name: Ada Lovelace +Birthday: December 10, 1815 +Bio: Ada was a British mathematician and writer, chiefly known for her work on + Charles Babbage's early mechanical general-purpose computer, the Analytical + Engine + +Name: Alan Turing +Birthday: June 23, 1912 +Bio: Alan was a British pioneering computer scientist, mathematician, logician, + cryptanalyst and theoretical biologist +``` + +## Installation + +``` +$ go get -v github.com/gosuri/uitable +``` + + +[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/gosuri/uitable/trend.png)](https://bitdeli.com/free "Bitdeli Badge") + diff --git a/vendor/github.com/gosuri/uitable/table.go b/vendor/github.com/gosuri/uitable/table.go new file mode 100644 index 0000000000..b792c88367 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/table.go @@ -0,0 +1,203 @@ +// Package uitable provides a decorator for formating data as a table +package uitable + +import ( + "fmt" + "strings" + "sync" + + "github.com/gosuri/uitable/util/strutil" + "github.com/gosuri/uitable/util/wordwrap" + "github.com/mattn/go-runewidth" +) + +// Separator is the default column seperator +var Separator = "\t" + +// Table represents a decorator that renders the data in formatted in a table +type Table struct { + // Rows is the collection of rows in the table + Rows []*Row + + // MaxColWidth is the maximum allowed width for cells in the table + MaxColWidth uint + + // Wrap when set to true wraps the contents of the columns when the length exceeds the MaxColWidth + Wrap bool + + // Separator is the seperator for columns in the table. Default is "\t" + Separator string + + mtx *sync.RWMutex + rightAlign map[int]bool +} + +// New returns a new Table with default values +func New() *Table { + return &Table{ + Separator: Separator, + mtx: new(sync.RWMutex), + rightAlign: map[int]bool{}, + } +} + +// AddRow adds a new row to the table +func (t *Table) AddRow(data ...interface{}) *Table { + t.mtx.Lock() + defer t.mtx.Unlock() + r := NewRow(data...) + t.Rows = append(t.Rows, r) + return t +} + +// Bytes returns the []byte value of table +func (t *Table) Bytes() []byte { + return []byte(t.String()) +} + +func (t *Table) RightAlign(col int) { + t.mtx.Lock() + t.rightAlign[col] = true + t.mtx.Unlock() +} + +// String returns the string value of table +func (t *Table) String() string { + t.mtx.RLock() + defer t.mtx.RUnlock() + + if len(t.Rows) == 0 { + return "" + } + + // determine the width for each column (cell in a row) + var colwidths []uint + for _, row := range t.Rows { + for i, cell := range row.Cells { + // resize colwidth array + if i+1 > len(colwidths) { + colwidths = append(colwidths, 0) + } + cellwidth := cell.LineWidth() + if t.MaxColWidth != 0 && cellwidth > t.MaxColWidth { + cellwidth = t.MaxColWidth + } + + if cellwidth > colwidths[i] { + colwidths[i] = cellwidth + } + } + } + + var lines []string + for _, row := range t.Rows { + row.Separator = t.Separator + for i, cell := range row.Cells { + cell.Width = colwidths[i] + cell.Wrap = t.Wrap + cell.RightAlign = t.rightAlign[i] + } + lines = append(lines, row.String()) + } + return strutil.Join(lines, "\n") +} + +// Row represents a row in a table +type Row struct { + // Cells is the group of cell for the row + Cells []*Cell + + // Separator for tabular columns + Separator string +} + +// NewRow returns a new Row and adds the data to the row +func NewRow(data ...interface{}) *Row { + r := &Row{Cells: make([]*Cell, len(data))} + for i, d := range data { + r.Cells[i] = &Cell{Data: d} + } + return r +} + +// String returns the string representation of the row +func (r *Row) String() string { + // get the max number of lines for each cell + var lc int // line count + for _, cell := range r.Cells { + if clc := len(strings.Split(cell.String(), "\n")); clc > lc { + lc = clc + } + } + + // allocate a two-dimentional array of cells for each line and add size them + cells := make([][]*Cell, lc) + for x := 0; x < lc; x++ { + cells[x] = make([]*Cell, len(r.Cells)) + for y := 0; y < len(r.Cells); y++ { + cells[x][y] = &Cell{Width: r.Cells[y].Width} + } + } + + // insert each line in a cell as new cell in the cells array + for y, cell := range r.Cells { + lines := strings.Split(cell.String(), "\n") + for x, line := range lines { + cells[x][y].Data = line + } + } + + // format each line + lines := make([]string, lc) + for x := range lines { + line := make([]string, len(cells[x])) + for y := range cells[x] { + line[y] = cells[x][y].String() + } + lines[x] = strutil.Join(line, r.Separator) + } + return strutil.Join(lines, "\n") +} + +// Cell represents a column in a row +type Cell struct { + // Width is the width of the cell + Width uint + + // Wrap when true wraps the contents of the cell when the lenght exceeds the width + Wrap bool + + // RightAlign when true aligns contents to the right + RightAlign bool + + // Data is the cell data + Data interface{} +} + +// LineWidth returns the max width of all the lines in a cell +func (c *Cell) LineWidth() uint { + width := 0 + for _, s := range strings.Split(c.String(), "\n") { + w := runewidth.StringWidth(s) + if w > width { + width = w + } + } + return uint(width) +} + +// String returns the string formated representation of the cell +func (c *Cell) String() string { + if c.Data == nil { + return strutil.PadLeft(" ", int(c.Width), ' ') + } + s := fmt.Sprintf("%v", c.Data) + if c.Width > 0 { + if c.Wrap && uint(len(s)) > c.Width { + return wordwrap.WrapString(s, c.Width) + } else { + return strutil.Resize(s, c.Width, c.RightAlign) + } + } + return s +} diff --git a/vendor/github.com/gosuri/uitable/util/strutil/strutil.go b/vendor/github.com/gosuri/uitable/util/strutil/strutil.go new file mode 100644 index 0000000000..cb35bce272 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/strutil/strutil.go @@ -0,0 +1,76 @@ +// Package strutil provides various utilities for manipulating strings +package strutil + +import ( + "bytes" + "github.com/mattn/go-runewidth" +) + +// PadRight returns a new string of a specified length in which the end of the current string is padded with spaces or with a specified Unicode character. +func PadRight(str string, length int, pad byte) string { + slen := runewidth.StringWidth(str) + if slen >= length { + return str + } + buf := bytes.NewBufferString(str) + for i := 0; i < length-slen; i++ { + buf.WriteByte(pad) + } + return buf.String() +} + +// PadLeft returns a new string of a specified length in which the beginning of the current string is padded with spaces or with a specified Unicode character. +func PadLeft(str string, length int, pad byte) string { + slen := runewidth.StringWidth(str) + if slen >= length { + return str + } + var buf bytes.Buffer + for i := 0; i < length-slen; i++ { + buf.WriteByte(pad) + } + buf.WriteString(str) + return buf.String() +} + +// Resize resizes the string with the given length. It ellipses with '...' when the string's length exceeds +// the desired length or pads spaces to the right of the string when length is smaller than desired +func Resize(s string, length uint, rightAlign bool) string { + slen := runewidth.StringWidth(s) + n := int(length) + if slen == n { + return s + } + // Pads only when length of the string smaller than len needed + if rightAlign { + s = PadLeft(s, n, ' ') + } else { + s = PadRight(s, n, ' ') + } + if slen > n { + rs := []rune(s) + var buf bytes.Buffer + w := 0 + for _, r := range rs { + buf.WriteRune(r) + rw := runewidth.RuneWidth(r) + if w+rw >= n-3 { + break + } + w += rw + } + buf.WriteString("...") + s = buf.String() + } + return s +} + +// Join joins the list of the string with the delim provided +func Join(list []string, delim string) string { + var buf bytes.Buffer + for i := 0; i < len(list)-1; i++ { + buf.WriteString(list[i] + delim) + } + buf.WriteString(list[len(list)-1]) + return buf.String() +} diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md b/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md new file mode 100644 index 0000000000..2298515904 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/LICENSE.md @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/README.md b/vendor/github.com/gosuri/uitable/util/wordwrap/README.md new file mode 100644 index 0000000000..60ae311700 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/README.md @@ -0,0 +1,39 @@ +# go-wordwrap + +`go-wordwrap` (Golang package: `wordwrap`) is a package for Go that +automatically wraps words into multiple lines. The primary use case for this +is in formatting CLI output, but of course word wrapping is a generally useful +thing to do. + +## Installation and Usage + +Install using `go get github.com/mitchellh/go-wordwrap`. + +Full documentation is available at +http://godoc.org/github.com/mitchellh/go-wordwrap + +Below is an example of its usage ignoring errors: + +```go +wrapped := wordwrap.WrapString("foo bar baz", 3) +fmt.Println(wrapped) +``` + +Would output: + +``` +foo +bar +baz +``` + +## Word Wrap Algorithm + +This library doesn't use any clever algorithm for word wrapping. The wrapping +is actually very naive: whenever there is whitespace or an explicit linebreak. +The goal of this library is for word wrapping CLI output, so the input is +typically pretty well controlled human language. Because of this, the naive +approach typically works just fine. + +In the future, we'd like to make the algorithm more advanced. We would do +so without breaking the API. diff --git a/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go b/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go new file mode 100644 index 0000000000..e64dbc8016 --- /dev/null +++ b/vendor/github.com/gosuri/uitable/util/wordwrap/wordwrap.go @@ -0,0 +1,84 @@ +// Package wordwrap provides methods for wrapping the contents of a string +package wordwrap + +import ( + "bytes" + "github.com/mattn/go-runewidth" + "unicode" +) + +// WrapString wraps the given string within lim width in characters. +// +// Wrapping is currently naive and only happens at white-space. A future +// version of the library will implement smarter wrapping. This means that +// pathological cases can dramatically reach past the limit, such as a very +// long word. +func WrapString(s string, lim uint) string { + // Initialize a buffer with a slightly larger size to account for breaks + init := make([]byte, 0, len(s)) + buf := bytes.NewBuffer(init) + + var current uint + var wordBuf, spaceBuf bytes.Buffer + var wordWidth, spaceWidth int + + for _, char := range s { + if char == '\n' { + if wordBuf.Len() == 0 { + if current+uint(spaceWidth) > lim { + current = 0 + } else { + current += uint(spaceWidth) + spaceBuf.WriteTo(buf) + spaceWidth += runewidth.StringWidth(buf.String()) + } + spaceBuf.Reset() + spaceWidth = 0 + } else { + current += uint(spaceWidth + wordWidth) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + spaceWidth = 0 + wordWidth = 0 + } + buf.WriteRune(char) + current = 0 + } else if unicode.IsSpace(char) { + if spaceBuf.Len() == 0 || wordBuf.Len() > 0 { + current += uint(spaceWidth + wordWidth) + spaceBuf.WriteTo(buf) + spaceBuf.Reset() + wordBuf.WriteTo(buf) + wordBuf.Reset() + spaceWidth = 0 + wordWidth = 0 + } + + spaceBuf.WriteRune(char) + spaceWidth += runewidth.RuneWidth(char) + } else { + wordBuf.WriteRune(char) + wordWidth += runewidth.RuneWidth(char) + + if current+uint(spaceWidth+wordWidth) > lim && uint(wordWidth) < lim { + buf.WriteRune('\n') + current = 0 + spaceBuf.Reset() + spaceWidth = 0 + } + } + } + + if wordBuf.Len() == 0 { + if current+uint(spaceWidth) <= lim { + spaceBuf.WriteTo(buf) + } + } else { + spaceBuf.WriteTo(buf) + wordBuf.WriteTo(buf) + } + + return buf.String() +} diff --git a/vendor/github.com/vmware/govmomi/simulator/license_manager.go b/vendor/github.com/vmware/govmomi/simulator/license_manager.go deleted file mode 100644 index 13565d32ef..0000000000 --- a/vendor/github.com/vmware/govmomi/simulator/license_manager.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright (c) 2017 VMware, Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Copyright 2017 VMware, Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package simulator - -import ( - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/methods" - "github.com/vmware/govmomi/vim25/mo" - "github.com/vmware/govmomi/vim25/soap" - "github.com/vmware/govmomi/vim25/types" -) - -// EvalLicense is the default license -var EvalLicense = types.LicenseManagerLicenseInfo{ - LicenseKey: "00000-00000-00000-00000-00000", - EditionKey: "eval", - Name: "Evaluation Mode", - Properties: []types.KeyAnyValue{ - { - Key: "feature", - Value: types.KeyValue{ - Key: "serialuri:2", - Value: "Remote virtual Serial Port Concentrator", - }, - }, - { - Key: "feature", - Value: types.KeyValue{ - Key: "dvs", - Value: "vSphere Distributed Switch", - }, - }, - }, -} - -type LicenseManager struct { - mo.LicenseManager -} - -func NewLicenseManager(ref types.ManagedObjectReference) object.Reference { - m := &LicenseManager{} - m.Self = ref - m.Licenses = []types.LicenseManagerLicenseInfo{EvalLicense} - - if Map.IsVPX() { - am := Map.Put(&LicenseAssignmentManager{}).Reference() - m.LicenseAssignmentManager = &am - } - - return m -} - -func (m *LicenseManager) AddLicense(req *types.AddLicense) soap.HasFault { - body := &methods.AddLicenseBody{ - Res: &types.AddLicenseResponse{}, - } - - for _, license := range m.Licenses { - if license.LicenseKey == req.LicenseKey { - body.Res.Returnval = licenseInfo(license.LicenseKey, license.Labels) - return body - } - } - - m.Licenses = append(m.Licenses, types.LicenseManagerLicenseInfo{ - LicenseKey: req.LicenseKey, - Labels: req.Labels, - }) - - body.Res.Returnval = licenseInfo(req.LicenseKey, req.Labels) - - return body -} - -func (m *LicenseManager) RemoveLicense(req *types.RemoveLicense) soap.HasFault { - body := &methods.RemoveLicenseBody{ - Res: &types.RemoveLicenseResponse{}, - } - - for i, license := range m.Licenses { - if req.LicenseKey == license.LicenseKey { - m.Licenses = append(m.Licenses[:i], m.Licenses[i+1:]...) - return body - } - } - return body -} - -type LicenseAssignmentManager struct { - mo.LicenseAssignmentManager -} - -func (m *LicenseAssignmentManager) QueryAssignedLicenses(req *types.QueryAssignedLicenses) soap.HasFault { - body := &methods.QueryAssignedLicensesBody{ - Res: &types.QueryAssignedLicensesResponse{}, - } - - // EntityId can be a HostSystem or the vCenter InstanceUuid - if req.EntityId != "" { - if req.EntityId != Map.content().About.InstanceUuid { - id := types.ManagedObjectReference{ - Type: "HostSystem", - Value: req.EntityId, - } - - if Map.Get(id) == nil { - return body - } - } - } - - body.Res.Returnval = []types.LicenseAssignmentManagerLicenseAssignment{ - { - EntityId: req.EntityId, - AssignedLicense: EvalLicense, - }, - } - - return body -} - -func licenseInfo(key string, labels []types.KeyValue) types.LicenseManagerLicenseInfo { - info := EvalLicense - - info.LicenseKey = key - info.Labels = labels - - return info -} From 241600f4039507449133cd0eb45275d3275f615d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 21 Aug 2018 20:47:40 +0800 Subject: [PATCH 25/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9B=A0=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0nil=E5=88=B0errors=E5=AF=BC=E8=87=B4=E9=87=8E=E6=8C=87?= =?UTF-8?q?=E9=92=88=E5=BC=95=E7=94=A8bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/cloudregions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 3ac4efcade..5e5c12245a 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -145,9 +145,9 @@ func (manager *SCloudregionManager) SyncRegions(ctx context.Context, userCred mc _, err = removed[i].PerformDisable(ctx, userCred, nil, nil) } if err != nil { - syncResult.Delete() - } else { syncResult.DeleteError(err) + } else { + syncResult.Delete() } } } From 5e8b7b3ce3a13788f4dfad9eaa3ee884d0fc9cc6 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 21 Aug 2018 20:49:42 +0800 Subject: [PATCH 26/71] fix auto-merge conflict --- Gopkg.lock | 6 + Makefile | 12 +- cmd/climc/climc.go | 52 +- cmd/climc/promputils/executor.go | 5 + cmd/climc/shell/disks.go | 14 +- cmd/climc/shell/secgrouprules.go | 2 - cmd/climc/shell/servers.go | 14 +- pkg/appsrv/dispatcher/dispatcher.go | 10 +- pkg/cloudcommon/cronman/cronman.go | 2 + pkg/cloudcommon/db/db_dispatcher.go | 23 +- pkg/cloudcommon/db/quotas/quotas.go | 4 +- pkg/cloudcommon/db/taskman/localtaskworker.go | 6 +- pkg/cloudcommon/db/taskman/subtasks.go | 2 +- pkg/cloudcommon/db/taskman/tasks.go | 26 +- pkg/cloudcommon/db/virtualresource.go | 4 +- pkg/cloudcommon/options.go | 1 + pkg/cloudprovider/resources.go | 8 + pkg/compute/guestdrivers/aliyun.go | 23 + pkg/compute/guestdrivers/baremetals.go | 22 +- pkg/compute/guestdrivers/base.go | 65 +- pkg/compute/guestdrivers/esxi.go | 30 + pkg/compute/guestdrivers/kvm.go | 184 ++++- pkg/compute/guestdrivers/managedvirtual.go | 2 +- pkg/compute/guestdrivers/virtualization.go | 36 + pkg/compute/hostdrivers/aliyun.go | 104 +++ pkg/compute/hostdrivers/kvm.go | 126 ++++ pkg/compute/models/cachedimages.go | 53 +- pkg/compute/models/cloudregions.go | 2 +- pkg/compute/models/disks.go | 280 +++++++- pkg/compute/models/guestdisks.go | 50 +- pkg/compute/models/guestdrivers.go | 26 +- pkg/compute/models/guestnetworks.go | 2 + pkg/compute/models/guests.go | 652 +++++++++++++++++- pkg/compute/models/hostdrivers.go | 39 ++ pkg/compute/models/hosts.go | 37 +- pkg/compute/models/networks.go | 1 + pkg/compute/models/secgrouprules.go | 187 ++++- pkg/compute/models/secgroups.go | 137 +++- pkg/compute/models/storagecachedimages.go | 56 +- pkg/compute/models/storagecaches.go | 54 +- pkg/compute/models/storages.go | 30 +- pkg/compute/models/vpcs.go | 10 +- pkg/compute/service/service.go | 19 +- .../tasks/cloud_provider_sync_info_task.go | 26 +- pkg/compute/tasks/disk_base_task.go | 15 + pkg/compute/tasks/disk_create_task.go | 76 ++ pkg/compute/tasks/disk_delete_task.go | 16 +- pkg/compute/tasks/disk_resize_task.go | 108 +++ pkg/compute/tasks/guest_change_config_task.go | 208 ++++++ pkg/compute/tasks/guest_create_disk_task.go | 124 ++++ pkg/compute/tasks/guest_delete_task.go | 13 +- .../tasks/guest_detach_all_disks_task.go | 1 + pkg/compute/tasks/guest_detach_disk_task.go | 83 +++ pkg/compute/tasks/guest_insert_iso_task.go | 58 +- pkg/compute/tasks/guest_rebuild_root_task.go | 144 ++++ pkg/compute/tasks/guest_reset_task.go | 58 ++ pkg/compute/tasks/guest_start_task.go | 3 +- pkg/compute/tasks/guest_stop_task.go | 1 + pkg/compute/tasks/guest_suspend_task.go | 47 ++ pkg/compute/tasks/guest_sync_task.go | 69 ++ pkg/compute/tasks/guest_syncstatus_task.go | 4 +- pkg/compute/tasks/guest_undeploy_task.go | 3 +- pkg/compute/tasks/storage_cache_image_task.go | 26 +- pkg/httperrors/errors.go | 7 + pkg/mcclient/mcclient.go | 11 + pkg/mcclient/session.go | 4 + pkg/util/aliyun/disk.go | 17 + pkg/util/aliyun/host.go | 4 +- pkg/util/aliyun/instance.go | 31 + pkg/util/aliyun/region.go | 1 + pkg/util/aliyun/securitygroup.go | 399 ++++++++++- pkg/util/aliyun/storage.go | 9 + pkg/util/aliyun/vpc.go | 48 +- pkg/util/esxi/storage.go | 4 + pkg/util/esxi/virtualmachine.go | 10 +- vendor/github.com/serialx/hashring/LICENSE | 21 + vendor/github.com/serialx/hashring/README.md | 70 ++ .../github.com/serialx/hashring/hashring.go | 268 +++++++ 78 files changed, 4202 insertions(+), 203 deletions(-) create mode 100644 pkg/compute/hostdrivers/aliyun.go create mode 100644 pkg/compute/hostdrivers/kvm.go create mode 100644 pkg/compute/models/hostdrivers.go create mode 100644 pkg/compute/tasks/disk_create_task.go create mode 100644 pkg/compute/tasks/disk_resize_task.go create mode 100644 pkg/compute/tasks/guest_change_config_task.go create mode 100644 pkg/compute/tasks/guest_create_disk_task.go create mode 100644 pkg/compute/tasks/guest_rebuild_root_task.go create mode 100644 pkg/compute/tasks/guest_reset_task.go create mode 100644 pkg/compute/tasks/guest_suspend_task.go create mode 100644 pkg/compute/tasks/guest_sync_task.go create mode 100644 vendor/github.com/serialx/hashring/LICENSE create mode 100644 vendor/github.com/serialx/hashring/README.md create mode 100644 vendor/github.com/serialx/hashring/hashring.go diff --git a/Gopkg.lock b/Gopkg.lock index 2499f89c7f..ccd7d5e941 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -667,6 +667,12 @@ [[projects]] digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + branch = "master" + name = "github.com/serialx/hashring" + packages = ["."] + revision = "49a4782e9908fe098c907022a1bd7519c79803d6" + +[[projects]] name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" diff --git a/Makefile b/Makefile index 228037d447..6537fbd733 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell pwd) +ROOT_DIR := $(shell readlink -f `pwd`) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh @@ -84,6 +84,16 @@ output_dir: @mkdir -p $(BUILD_DIR) +dep: + cd $(ROOT_DIR) && dep ensure -v + +dep_clean: + rm -fr $(GOPATH)/pkg/dep/sources/* + +dep_install: + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + + .PHONY: all build prepare_dir clean fmt rpm diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index 175543ccc8..1d4bdec473 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -1,11 +1,14 @@ package main import ( + "encoding/json" "fmt" + "io/ioutil" "os" "strings" + "time" - "github.com/c-bata/go-prompt" + prompt "github.com/c-bata/go-prompt" "yunion.io/x/log" "yunion.io/x/pkg/util/version" "yunion.io/x/structarg" @@ -117,18 +120,51 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { options.Timeout, options.Debug, options.Insecure) - token, err := client.Authenticate(options.OsUsername, - options.OsPassword, - options.OsDomainName, - options.OsProjectName) - if err != nil { - return nil, err + + var cacheToken mcclient.TokenCredential + cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN") + if err == nil && cacheFile != nil { + fileInfo, _ := cacheFile.Stat() + dur, err := time.ParseDuration("-24h") + if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { + bytesToken, err := ioutil.ReadAll(cacheFile) + if err == nil { + token := client.NewAuthTokenCredential() + err := json.Unmarshal(bytesToken, token) + if err != nil { + fmt.Printf("Unmarshal token error:%s", err) + } else { + cacheToken = token + } + } + cacheFile.Close() + } + } + if cacheToken == nil { + token, err := client.Authenticate(options.OsUsername, + options.OsPassword, + options.OsDomainName, + options.OsProjectName) + if err != nil { + return nil, err + } + cacheToken = token + bytesCacheToken, err := json.Marshal(cacheToken) + if err != nil { + fmt.Printf("Marshal token error:%s", err) + } else { + fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN") + fo.Write(bytesCacheToken) + fo.Close() + } + } else { + fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********") } session := client.NewSession(options.OsRegionName, options.OsZoneName, options.OsEndpointType, - token, + cacheToken, options.ApiVersion) return session, nil } diff --git a/cmd/climc/promputils/executor.go b/cmd/climc/promputils/executor.go index a3f579dfea..2b48225b17 100644 --- a/cmd/climc/promputils/executor.go +++ b/cmd/climc/promputils/executor.go @@ -56,6 +56,11 @@ func Executor(s string) { e := parser.ParseArgs(utils.ArgsStringToArray(s), false) subcmd := parser.GetSubcommand() subparser := subcmd.GetSubParser() + if args[0] == "--debug" { + session.GetClient().SetDebug(true) + } else { + session.GetClient().SetDebug(false) + } if e != nil { if subparser != nil { fmt.Print(subparser.Usage()) diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 3f4d35033b..36654af96e 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -10,11 +10,12 @@ import ( func init() { type DiskListOptions struct { BaseListOptions - Unused bool `help:"Show unused disks"` - Share bool `help:"Show Share storage disks"` - Local bool `help:"Show Local storage disks"` - Guest string `help:"Guest ID or name"` - Storage string `help:"Storage ID or name"` + Unused bool `help:"Show unused disks"` + Share bool `help:"Show Share storage disks"` + Local bool `help:"Show Local storage disks"` + Guest string `help:"Guest ID or name"` + Storage string `help:"Storage ID or name"` + Provider string `help:"Provider for disk" choices:"Aliyun|VMware"` } R(&DiskListOptions{}, "disk-list", "List virtual disks", func(s *mcclient.ClientSession, suboptions *DiskListOptions) error { params := FetchPagingParams(suboptions.BaseListOptions) @@ -33,6 +34,9 @@ func init() { if len(suboptions.Storage) > 0 { params.Add(jsonutils.NewString(suboptions.Storage), "storage") } + if len(suboptions.Provider) > 0 { + params.Add(jsonutils.NewString(suboptions.Provider), "provider") + } result, err := modules.Disks.List(s, params) if err != nil { return err diff --git a/cmd/climc/shell/secgrouprules.go b/cmd/climc/shell/secgrouprules.go index eb9fd4b621..13fdd49730 100644 --- a/cmd/climc/shell/secgrouprules.go +++ b/cmd/climc/shell/secgrouprules.go @@ -59,7 +59,6 @@ func init() { }) type SecGroupRulesCreateOptions struct { - NAME string `help:"Name of security group rule to create"` SECGROUP string `help:"Secgroup ID or Name" metavar:"Secgroup"` Direction string `help:"Direction of rule" choices:"in|out"` Action string `help:"Action of rule" choices:"allow|deny"` @@ -72,7 +71,6 @@ func init() { R(&SecGroupRulesCreateOptions{}, "secgroup-rule-create", "Create all security group rule", func(s *mcclient.ClientSession, args *SecGroupRulesCreateOptions) error { params := jsonutils.NewDict() - params.Add(jsonutils.NewString(args.NAME), "name") if len(args.Desc) > 0 { params.Add(jsonutils.NewString(args.Desc), "description") } diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index fe663182b6..0a3b49acfb 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -315,7 +315,7 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-sync", "Sync servers status", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { + R(&ServerOpsOptions{}, "server-sync", "Sync servers configures", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { ret := modules.Servers.BatchPerformAction(s, args.ID, "sync", nil) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil @@ -343,8 +343,16 @@ func init() { return nil }) - R(&ServerOpsOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerOpsOptions) error { - ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", nil) + type ServerResetOptions struct { + ServerOpsOptions + Hard bool `help:"Hard reset or not; default soft"` + } + R(&ServerResetOptions{}, "server-reset", "Reset servers", func(s *mcclient.ClientSession, args *ServerResetOptions) error { + params := jsonutils.NewDict() + if args.Hard { + params.Add(jsonutils.JSONTrue, "is_hard") + } + ret := modules.Servers.BatchPerformAction(s, args.ID, "reset", params) printBatchResults(ret, modules.Servers.GetColumns(s)) return nil }) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index 56cb359b66..1aeff85240 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/httperrors" @@ -201,7 +202,14 @@ func createInContextHandler(ctx context.Context, w http.ResponseWriter, r *http. func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { manager, params, query, body := fetchEnv(ctx, w, r) - data, _ := body.Get(manager.KeywordPlural()) + var data jsonutils.JSONObject + if body != nil { + data, _ = body.Get(manager.KeywordPlural()) + // about string ?? + if data == nil { + data = body.(*jsonutils.JSONDict) + } + } if data == nil { data = jsonutils.NewDict() } diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index d01cc096d5..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -89,6 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index fe06fc2f9d..8b4179f50d 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -9,18 +9,18 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/filterclause" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/httputils" ) type DBModelDispatcher struct { @@ -841,7 +841,12 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act managerValue := reflect.ValueOf(dispatcher.modelManager) if action == "check-create-data" { - return dispatcher.modelManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data.(*jsonutils.JSONDict)) + manager := dispatcher.modelManager + if body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword()); err != nil { + return nil, httperrors.NewGeneralError(err) + } else { + return manager.ValidateCreateData(ctx, userCred, ownerProjId, query, body.(*jsonutils.JSONDict)) + } } return objectPerformAction(dispatcher, managerValue, ctx, userCred, action, query, data) } @@ -1039,13 +1044,13 @@ func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCr err := model.ValidateDeleteCondition(ctx) if err != nil { log.Errorf("validate delete condition error: %s", err) - return nil, httperrors.NewGeneralError(err) + return nil, httperrors.NewNotAcceptableError(err.Error()) } err = model.CustomizeDelete(ctx, userCred, query, data) if err != nil { log.Errorf("customize delete error: %s", err) - return nil, httperrors.NewGeneralError(err) + return nil, httperrors.NewNotAcceptableError(err.Error()) } details, err := getItemDetails(manager, model, ctx, userCred, query) diff --git a/pkg/cloudcommon/db/quotas/quotas.go b/pkg/cloudcommon/db/quotas/quotas.go index b26700d073..b80fb3c1d0 100644 --- a/pkg/cloudcommon/db/quotas/quotas.go +++ b/pkg/cloudcommon/db/quotas/quotas.go @@ -63,7 +63,9 @@ func (manager *SQuotaManager) _cancelPendingUsage(ctx context.Context, userCred log.Errorf("%s", err) return err } - localUsage.Sub(cancelUsage) + if localUsage != nil { + localUsage.Sub(cancelUsage) + } quota.Sub(cancelUsage) err = manager.pendingStore.SetQuota(ctx, userCred, projectId, quota) if err != nil { diff --git a/pkg/cloudcommon/db/taskman/localtaskworker.go b/pkg/cloudcommon/db/taskman/localtaskworker.go index 2e5ac395e9..f60e8b146f 100644 --- a/pkg/cloudcommon/db/taskman/localtaskworker.go +++ b/pkg/cloudcommon/db/taskman/localtaskworker.go @@ -15,7 +15,7 @@ func init() { localTaskWorkerMan = appsrv.NewWorkerManager("LocalTaskWorkerManager", 4, 10) } -func error2TaskData(err error) jsonutils.JSONObject { +func Error2TaskData(err error) jsonutils.JSONObject { errJson := jsonutils.NewDict() errJson.Add(jsonutils.NewString("ERROR"), "__status__") errJson.Add(jsonutils.NewString(err.Error()), "reason") @@ -32,12 +32,12 @@ func LocalTaskRun(task ITask, proc func() (jsonutils.JSONObject, error)) { if r := recover(); r != nil { log.Errorf("LocalTaskRun error: %s", r) debug.PrintStack() - task.ScheduleRun(error2TaskData(fmt.Errorf("LocalTaskRun error: %s", r))) + task.ScheduleRun(Error2TaskData(fmt.Errorf("LocalTaskRun error: %s", r))) } }() data, err := proc() if err != nil { - task.ScheduleRun(error2TaskData(err)) + task.ScheduleRun(Error2TaskData(err)) } else { task.ScheduleRun(data) } diff --git a/pkg/cloudcommon/db/taskman/subtasks.go b/pkg/cloudcommon/db/taskman/subtasks.go index 8ad2c2902e..d4d0bf50ed 100644 --- a/pkg/cloudcommon/db/taskman/subtasks.go +++ b/pkg/cloudcommon/db/taskman/subtasks.go @@ -40,7 +40,7 @@ func (manager *SSubTaskmanager) GetSubTask(ptaskId string, subtaskId string) *SS err := manager.Query().Equals("task_id", ptaskId).Equals("subtask_id", subtaskId).First(&subtask) if err != nil { if err != sql.ErrNoRows { - log.Errorf("GetSubTask fail %s", err) + log.Errorf("GetSubTask fail %s %s %s", err, ptaskId, subtaskId) } return nil } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..8e4f45acf6 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -307,7 +311,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, var stageName string if taskFailed { - stageName = fmt.Sprintf("%s_failed", task.Stage) + stageName = fmt.Sprintf("%sFailed", task.Stage) } else { stageName = task.Stage } @@ -315,6 +319,10 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, funcValue := taskValue.MethodByName(stageName) if !funcValue.IsValid() || funcValue.IsNil() { + log.Debugf("Stage %s not found, try kebab to camel and find again", stageName) + if taskFailed { + stageName = fmt.Sprintf("%s_failed", task.Stage) + } stageName = utils.Kebab2Camel(stageName, "_") funcValue = taskValue.MethodByName(stageName) @@ -387,7 +395,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, params[2] = reflect.ValueOf(data) - log.Debugf("Call %s with %s", funcValue, params) + log.Debugf("Call %s: %s with %s", stageName, funcValue, params) funcValue.Call(params) @@ -408,9 +416,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 28360ccb55..db56d599f4 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -8,13 +8,13 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SVirtualResourceBaseManager struct { diff --git a/pkg/cloudcommon/options.go b/pkg/cloudcommon/options.go index 7b91719b70..a6e4fa7336 100644 --- a/pkg/cloudcommon/options.go +++ b/pkg/cloudcommon/options.go @@ -33,6 +33,7 @@ type Options struct { NotifyAdminUser string `default:"sysadmin" help:"System administrator user ID or name to notify"` GlobalVirtualResourceNamespace bool `help:"Per project namespace or global namespace for virtual resources"` + DebugSqlchemy bool `default:"False" help:"Print SQL executed by sqlchemy"` structarg.BaseOptions } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 2ff3e397b9..f388d66feb 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/secrules" ) type ICloudResource interface { @@ -84,6 +85,7 @@ type ICloudStorage interface { GetManagerId() string CreateIDisk(name string, sizeGb int, desc string) (ICloudDisk, error) + GetIDisk(idStr string) (ICloudDisk, error) } type ICloudHost interface { @@ -142,6 +144,7 @@ type ICloudVM interface { GetBios() string GetMachine() string + SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error GetHypervisor() string // GetSecurityGroup() ICloudSecurityGroup @@ -168,6 +171,8 @@ type ICloudEIP interface { type ICloudSecurityGroup interface { ICloudResource + GetDescription() string + GetRules() ([]secrules.SecurityRule, error) } type ICloudDisk interface { @@ -187,6 +192,8 @@ type ICloudDisk interface { GetDriver() string GetCacheMode() string GetMountpoint() string + Delete() error + Resize(int64) error } type ICloudVpc interface { @@ -197,6 +204,7 @@ type ICloudVpc interface { GetCidrBlock() string // GetStatus() string GetIWires() ([]ICloudWire, error) + GetISecurityGroups() ([]ICloudSecurityGroup, error) GetManagerId() string diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 50ebab0f8d..9477c541f6 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -170,6 +170,13 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil, err } + if len(guest.SecgrpId) > 0 { + if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + log.Errorf("SyncSecurityGroup error: %v", err) + return nil, err + } + } + if onfinish == "none" { err = iVM.StartVM() if err != nil { @@ -254,3 +261,19 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } + +func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + if ihost, err := host.GetIHost(); err != nil { + return nil, err + } else if iVM, err := ihost.GetIVMById(guest.ExternalId); err != nil { + return nil, err + } else if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + return nil, err + } + } + return nil, nil + }) + return nil +} diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index d91cb4c49f..2ec935e955 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -2,13 +2,14 @@ package guestdrivers import ( "context" + "fmt" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" ) type SBaremetalGuestDriver struct { @@ -73,7 +74,7 @@ func (self *SBaremetalGuestDriver) RequestGuestCreateInsertIso(ctx context.Conte return nil } -func (self *SBaremetalGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { +func (self *SBaremetalGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { data := jsonutils.NewDict() // TODO return data, nil @@ -154,3 +155,20 @@ func (self *SBaremetalGuestDriver) RequestDeployGuestOnHost(ctx context.Context, return nil } + +func (self *SBaremetalGuestDriver) CanKeepDetachDisk() bool { + return false +} + +func (self *SBaremetalGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SBaremetalGuestDriver) StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + return fmt.Errorf("Cannot detach disk from a baremetal serer") +} + +func (self *SBaremetalGuestDriver) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + return fmt.Errorf("Cannot suspend a baremetal serer") +} diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 6d10f94aa5..a63aa1470c 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -54,16 +54,16 @@ func (self *SBaseGuestDriver) StartGuestCreateTask(guest *models.SGuest, ctx con } func (self *SBaseGuestDriver) OnGuestCreateTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - //if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { - // task.SetStage("on_auto_start_guest", nil) - // return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) - //} else { - task.SetStage("on_sync_status_complete", nil) - return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) - //} + if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { + task.SetStage("on_auto_start_guest", nil) + return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) + } else { + task.SetStage("on_sync_status_complete", nil) + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) + } } -func (self *SBaseGuestDriver) StartDeleteGuestTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { +func (self *SBaseGuestDriver) StartDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { task, err := taskman.TaskManager.NewTask(ctx, "GuestDeleteTask", guest, userCred, params, parentTaskId, "", nil) if err != nil { return err @@ -80,3 +80,52 @@ func (self *SBaseGuestDriver) RequestDetachDisksFromGuestForDelete(ctx context.C func (self *SBaseGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { return guest.DeleteAllDisksInDB(ctx, userCred) } + +func (self *SBaseGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SBaseGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{}, fmt.Errorf("This Guest driver dose not implement GetDetachDiskStatus") +} + +func (self *SBaseGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SBaseGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} + +func (self *SBaseGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} diff --git a/pkg/compute/guestdrivers/esxi.go b/pkg/compute/guestdrivers/esxi.go index 4d515c0f84..f1ab950f82 100644 --- a/pkg/compute/guestdrivers/esxi.go +++ b/pkg/compute/guestdrivers/esxi.go @@ -1,6 +1,9 @@ package guestdrivers import ( + "context" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -16,3 +19,30 @@ func init() { func (self *SESXiGuestDriver) GetHypervisor() string { return models.HYPERVISOR_ESXI } + +func (self *SESXiGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SESXiGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{models.VM_READY}, nil +} + +func (self *SESXiGuestDriver) CanKeepDetachDisk() bool { + return false +} + +func (self *SESXiGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + err := disk.RealDelete(ctx, task.GetUserCred()) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SESXiGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 4b43f4f79d..d98ed72b15 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -3,15 +3,17 @@ package guestdrivers import ( "context" "fmt" + "net/http" "regexp" "strconv" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SKVMGuestDriver struct { @@ -36,9 +38,12 @@ func (self *SKVMGuestDriver) RequestDetachDisksFromGuestForDelete(ctx context.Co return nil } -func (self *SKVMGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { - // guest.DeleteAllDisksInDB(ctx, userCred) - // do nothing +func (self *SKVMGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "KVMGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) return nil } @@ -90,40 +95,173 @@ func (self *SKVMGuestDriver) GetGuestVncInfo(userCred mcclient.TokenCredential, return retval, nil } -func (self *SKVMGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - // TODO +func (self *SKVMGuestDriver) RequestStopOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + body := jsonutils.NewDict() + params := task.GetParams() + timeout, err := params.Int("timeout") + if err != nil { + timeout = 30 + } + isForce, err := params.Bool("is_force") + if isForce { + timeout = 0 + } + body.Add(jsonutils.NewInt(timeout), "timeout") + + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + + url := fmt.Sprintf("%s/servers/%s/stop", host.ManagerUri, guest.Id) + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + return err +} + +func (self *SKVMGuestDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + url := fmt.Sprintf("%s/servers/%s", host.ManagerUri, guest.Id) + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "DELETE", url, header, nil, false) + if err != nil { + return err + } + delayClean := jsonutils.QueryBoolean(res, "delay_clean", false) + if res != nil && delayClean { + return nil + } + task.ScheduleRun(nil) return nil } func (self *SKVMGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO - + config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) + log.Debugf("RequestDeployGuestOnHost: %s", config) + if config.Contains("container") { + // ... + } + action, err := config.GetString("action") + if err != nil { + return err + } + url := fmt.Sprintf("%s/servers/%s/%s", host.ManagerUri, guest.Id, action) + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false) + if err != nil { + return err + } return nil } -func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - // TODO +func (self *SKVMGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } -func (self *SKVMGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - // TODO - return data, nil -} +func (self *SKVMGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") -func (self *SKVMGuestDriver) RequestStopOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO - return nil + config := jsonutils.NewDict() + desc := self.GetJsonDescAtHost(ctx, guest, host) + config.Add(desc, "desc") + params := task.GetParams() + if params.Length() > 0 { + config.Add(params, "params") + } + url := fmt.Sprintf("%s/servers/%s/start", host.ManagerUri, guest.Id) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, config, false) + if err != nil { + return nil, err + } + return res, nil } func (self *SKVMGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) { - data := jsonutils.NewDict() - // TODO - return data, nil + header := http.Header{} + header.Set("X-Auth-Token", userCred.GetTokenString()) + header.Set("X-Region-Version", "v2") + + url := fmt.Sprintf("%s/servers/%s/status", host.ManagerUri, guest.Id) + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "GET", url, header, nil, false) + if err != nil { + return nil, err + } + return res, nil } -func (self *SKVMGuestDriver) RequestUndeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // TODO +func (self *SKVMGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential) error { + return nil +} + +func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + // pass + return nil +} + +func (self *SKVMGuestDriver) RequestSoftReset(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + _, err := guest.SendMonitorCommand(ctx, task.GetUserCred(), "system_reset") + return err +} + +func (self *SKVMGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) +} + +func (self *SKVMGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{models.VM_READY, models.VM_RUNNING}, nil +} + +func (self *SKVMGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk *models.SDisk, task taskman.ITask, isPurge bool) error { + return disk.StartDiskDeleteTask(ctx, task.GetUserCred(), task.GetTaskId(), isPurge) +} + +func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) + body := jsonutils.NewDict() + body.Add(desc, "desc") + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + body.Add(jsonutils.JSONTrue, "fw_only") + } + url := fmt.Sprintf("/servers/%s/sync", guest.Id) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + host := guest.GetHost() + url := fmt.Sprintf("%s/servers/%s/suspend", host.ManagerUri, guest.Id) + header := http.Header{} + header.Add("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, nil, false) + return err +} + +func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartGuestCreateDiskTask(ctx, task.GetUserCred(), task.GetParams(), task.GetTaskId()) +} + +func (self *SKVMGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) +} + +func (self *SKVMGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "KVMGuestRebuildRootTask", guest, task.GetUserCred(), nil, task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) return nil } diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 486cca38ed..5976b6ff60 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -46,7 +46,7 @@ func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx co return nil } -func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { +func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(_ context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { ihost, err := host.GetIHost() if err != nil { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index 2ed28aca95..7efdd6d475 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -95,8 +95,22 @@ func (self *SVirtualizedGuestDriver) StartGuestStopTask(guest *models.SGuest, ct return nil } +func (self *SVirtualizedGuestDriver) StartGuestResetTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error { + var taskName = "GuestSoftResetTask" + if isHard { + taskName = "GuestHardResetTask" + } + task, err := taskman.TaskManager.NewTask(ctx, taskName, guest, userCred, nil, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SVirtualizedGuestDriver) OnGuestDeployTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { if jsonutils.QueryBoolean(task.GetParams(), "restart", false) { + task.SetStage("OnDeployStartGuestComplete", nil) return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) } else { guest.SetStatus(task.GetUserCred(), models.VM_READY, "ready") @@ -157,3 +171,25 @@ func (self *SVirtualizedGuestDriver) CheckDiskTemplateOnStorage(ctx context.Cont } return cache.StartImageCacheTask(ctx, userCred, imageId, false, task.GetTaskId()) } + +func (self *SVirtualizedGuestDriver) CanKeepDetachDisk() bool { + return true +} + +func (self *SVirtualizedGuestDriver) StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestDetachDiskTask", guest, userCred, params, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SVirtualizedGuestDriver) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestSuspendTask", guest, userCred, params, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go new file mode 100644 index 0000000000..0bcfa18005 --- /dev/null +++ b/pkg/compute/hostdrivers/aliyun.go @@ -0,0 +1,104 @@ +package hostdrivers + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SAliyunHostDriver struct { +} + +func init() { + driver := SAliyunHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SAliyunHostDriver) GetHostType() string { + return models.HOST_TYPE_ALIYUN +} + +func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, scimg *models.SStoragecachedimage, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + isForce := jsonutils.QueryBoolean(params, "is_force", false) + userCred := task.GetUserCred() + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iStorageCache, err := storageCache.GetIStorageCache() + if err != nil { + return nil, err + } + + extImgId, err := iStorageCache.UploadImage(userCred, imageId, scimg.ExternalId, isForce) + + if err != nil { + return nil, err + } else { + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(extImgId), "image_id") + return ret, nil + } + }) + return nil +} + +func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else { + if size, err := content.Int("size"); err != nil { + return err + } else { + size = size >> 10 + if iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil { + return err + } else { + if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.ExternalId = iDisk.GetGlobalId() + return nil + }); err != nil { + log.Errorf("Update disk externalId err: %v", err) + return err + } + data := jsonutils.NewDict() + data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") + data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") + task.ScheduleRun(data) + } + } + } + return nil +} + +func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { + return err + } else { + return iDisk.Delete() + } +} + +func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +} + +func (self *SAliyunHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + if iCloudStorage, err := storage.GetIStorage(); err != nil { + return err + } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { + return err + } else if err := iDisk.Resize(size >> 10); err != nil { + return err + } else { + task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) + } + return nil +} diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go new file mode 100644 index 0000000000..0a5cbca5b2 --- /dev/null +++ b/pkg/compute/hostdrivers/kvm.go @@ -0,0 +1,126 @@ +package hostdrivers + +import ( + "context" + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SKVMHostDriver struct { +} + +func init() { + driver := SKVMHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SKVMHostDriver) GetHostType() string { + return models.HOST_TYPE_HYPERVISOR +} + +func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, scimg *models.SStoragecachedimage, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + isForce := jsonutils.QueryBoolean(params, "is_force", false) + obj, err := models.CachedimageManager.FetchById(imageId) + if err != nil { + return err + } + cacheImage := obj.(*models.SCachedimage) + srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(models.HOST_TYPE_HYPERVISOR, []string{host.Id}, []*models.SZone{host.GetZone()}) + if err != nil { + return err + } + + content := jsonutils.NewDict() + content.Add(jsonutils.NewString(imageId), "image_id") + if srcHostCacheImage != nil { + err = srcHostCacheImage.AddDownloadRefcount() + if err != nil { + return err + } + srcHost, err := srcHostCacheImage.GetHost() + if err != nil { + return err + } + srcUrl := fmt.Sprintf("%s/download/images/%s", srcHost.ManagerUri, imageId) + content.Add(jsonutils.NewString(srcUrl), "src_url") + } + url := fmt.Sprintf("%s/disks/image_cache", host.ManagerUri) + + if isForce { + content.Add(jsonutils.NewBool(true), "is_force") + } + content.Add(jsonutils.NewString(storageCache.Id), "storagecache_id") + body := jsonutils.NewDict() + body.Add(content, "disk") + header := http.Header{} + header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) + header.Set("X-Task-Id", task.GetTaskId()) + header.Set("X-Region-Version", "v2") + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + if err != nil { + return err + } + return nil +} + +func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + body.Add(content, "disk") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + log.Infof("Deallocating disk on host %s", host.GetName()) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + url := fmt.Sprintf("/disks/%s/resize/%s", storage.Id, disk.Id) + body := jsonutils.NewDict() + content := jsonutils.NewDict() + content.Add(jsonutils.NewInt(size), "size") + body.Add(content, "disk") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + self.RequestResizeDiskOnHost(host, storage, disk, size, task) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + for _, guest := range disk.GetAttachedGuests() { + guestdisk := guest.GetGuestDisk(disk.GetId()) + url := fmt.Sprintf("/servers/%s/monitor", guest.GetId()) + body := jsonutils.NewDict() + cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, size) + body.Add(jsonutils.NewString(cmd), "cmd") + host.Request(task.GetUserCred(), "POST", url, header, body) + } + return nil +} diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index b1a9cbdf56..827fdc5d29 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "math/rand" "time" "yunion.io/x/jsonutils" @@ -170,7 +171,7 @@ func (manager *SCachedimageManager) cacheImageInfo(ctx context.Context, userCred } } -func (manager *SCachedimageManager) getImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { +func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { if !refresh { imgObj, _ := manager.FetchById(imageId) if imgObj != nil { @@ -207,7 +208,7 @@ func (manager *SCachedimageManager) getImageByName(ctx context.Context, userCred } func (manager *SCachedimageManager) getImageInfo(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { - img, err := manager.getImageById(ctx, userCred, imageId, refresh) + img, err := manager.GetImageById(ctx, userCred, imageId, refresh) if err == nil { return img, nil } @@ -236,7 +237,7 @@ func (self *SCachedimage) AllowPerformRefresh(ctx context.Context, userCred mccl } func (self *SCachedimage) PerformRefresh(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - img, err := CachedimageManager.getImageById(ctx, userCred, self.Id, true) + img, err := CachedimageManager.GetImageById(ctx, userCred, self.Id, true) if err != nil { return nil, err } @@ -257,6 +258,52 @@ func (self *SCachedimage) addRefCount() { } } +func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs interface{}) (*SStoragecachedimage, error) { + storageCachedImage := StoragecachedimageManager.Query().SubQuery() + storage := StorageManager.Query().SubQuery() + hostStorage := HoststorageManager.Query().SubQuery() + host := HostManager.Query().SubQuery() + + scimgs := make([]SStoragecachedimage, 0) + q := storageCachedImage.Query(). + Join(storage, sqlchemy.AND(sqlchemy.Equals(storage.Field("storagecache_id"), storageCachedImage.Field("storagecache_id")))). + Join(hostStorage, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("storage_id"), storage.Field("id")))). + Join(host, sqlchemy.AND(sqlchemy.Equals(hostStorage.Field("host_id"), host.Field("id")))). + Filter(sqlchemy.Equals(storageCachedImage.Field("cachedimage_id"), self.Id)). + Filter(sqlchemy.Equals(storageCachedImage.Field("status"), CACHED_IMAGE_STATUS_READY)). + Filter(sqlchemy.Equals(host.Field("status"), HOST_STATUS_RUNNING)). + Filter(sqlchemy.IsTrue(host.Field("enabled"))). + Filter(sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE)) + + if len(excludes) > 0 { + q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes)) + } + if len(hostType) > 0 { + q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType)) + } + + switch v := rangeObjs.(type) { + case []*SZone: + for _, obj := range v { + q = q.Filter(sqlchemy.Equals(host.Field("zone_id"), obj.Id)) + } + case []*SVCenter: + for _, obj := range v { + q = q.Filter(sqlchemy.Equals(host.Field("manager_id"), obj.Id)) + } + } + err := q.All(&scimgs) + if err != nil { + return nil, err + } + if len(scimgs) == 0 { + return nil, nil + } + + rand.Seed(time.Now().Unix()) + return &scimgs[rand.Intn(len(scimgs))], nil +} + func (manager *SCachedimageManager) ImageAddRefCount(imageId string) { cachedObj, _ := manager.FetchById(imageId) if cachedObj != nil { diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 3ac4efcade..e3c31efbba 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -220,7 +220,7 @@ func (self *SCloudregion) AllowPerformDefaultVpc(ctx context.Context, userCred m } func (self *SCloudregion) PerformDefaultVpc(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - vpcs, err := VpcManager.getVpcsByRegion(self) + vpcs, err := VpcManager.getVpcsByRegion(self, nil) if err != nil { return nil, err } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 047863499a..a2e827b7b9 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,10 +16,12 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -37,6 +39,7 @@ const ( DISK_DEALLOC = "deallocating" DISK_DEALLOC_FAILED = "dealloc_failed" DISK_UNKNOWN = "unknown" + DISK_DETACHING = "detaching" DISK_START_SAVE = "start_save" DISK_SAVING = "saving" @@ -86,6 +89,10 @@ type SDisk struct { Nonpersistent bool `default:"false" list:"user"` // Column(Boolean, default=False) } +func (manager *SDiskManager) GetContextManager() []db.IModelManager { + return []db.IModelManager{StorageManager} +} + func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { @@ -109,6 +116,14 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu sq := storages.Query(storages.Field("id")).Filter(sqlchemy.In(storages.Field("storage_type"), STORAGE_LOCAL_TYPES)) q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) } + if provier, _ := queryDict.GetString("provider"); len(provier) > 0 { + cloudprovider := CloudproviderManager.Query().SubQuery() + sq := storages.Query(storages.Field("id")).Join(cloudprovider, + sqlchemy.AND( + sqlchemy.Equals(cloudprovider.Field("id"), storages.Field("manager_id")), + sqlchemy.Equals(cloudprovider.Field("provider"), provier))) + q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) + } guestId, _ := queryDict.GetString("guest") if len(guestId) != 0 { guest := GuestManager.FetchGuestById(guestId) @@ -141,6 +156,10 @@ func (self *SDisk) GetGuestDiskCount() int { return guestdisks.Equals("disk_id", self.Id).Count() } +func (self *SDisk) isAttached() bool { + return GuestdiskManager.Query().Equals("disk_id", self.Id).Count() > 0 +} + func (self *SDisk) GetGuestdisks() []SGuestdisk { guestdisks := make([]SGuestdisk, 0) q := GuestdiskManager.Query().Equals("disk_id", self.Id) @@ -151,6 +170,193 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk { } return guestdisks } +func (self *SDisk) GetGuests() []SGuest { + result := make([]SGuest, 0) + query := GuestManager.Query() + guestdisks := GuestdiskManager.Query().SubQuery() + q := query.Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), query.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) + // q.DebugQuery() + err := db.FetchModelObjects(GuestManager, q, &result) + if err != nil { + log.Errorf(err.Error()) + return nil + } + return result +} + +func (self *SDisk) GetGuestsCount() int { + guests := GuestManager.Query().SubQuery() + guestdisks := GuestdiskManager.Query().SubQuery() + return guests.Query().Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)).Count() +} + +func (self *SDisk) GetRuningGuestCount() int { + guests := GuestManager.Query().SubQuery() + guestdisks := GuestdiskManager.Query().SubQuery() + return guests.Query().Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), guests.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)). + Filter(sqlchemy.Equals(guests.Field("status"), VM_RUNNING)).Count() +} + +func (self *SDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + diskConfig := SDiskConfig{} + if err := data.Unmarshal(&diskConfig, "disk"); err != nil { + return err + } else { + self.fetchDiskInfo(&diskConfig) + } + return self.SSharableVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data) +} + +func (manager *SDiskManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if disk, err := data.Get("disk"); err != nil { + return nil, err + } else { + if diskConfig, err := parseDiskInfo(ctx, userCred, disk); err != nil { + return nil, err + } else { + data.Add(jsonutils.Marshal(diskConfig), "disk") + if storageID, err := data.GetString("storage_id"); err != nil { + return nil, err + } else { + storages := StorageManager.Query().SubQuery() + storage := SStorage{} + storage.SetModelManager(StorageManager) + if err := storages.Query().Equals("id", storageID).First(&storage); err != nil { + return nil, err + } + if !storage.Enabled { + return nil, httperrors.NewInputParameterError("Cannot create disk with disabled storage[%s]", storage.Name) + } + if !utils.IsInStringArray(storage.Status, []string{STORAGE_ENABLED, STORAGE_ONLINE}) { + return nil, httperrors.NewInputParameterError("Cannot create disk with offline storage[%s]", storage.Name) + } + if storage.StorageType != diskConfig.Backend { + return nil, httperrors.NewInputParameterError("Storage type[%s] not match backend %s", storage.StorageType, diskConfig.Backend) + } + size := diskConfig.Size >> 10 + if storage.StorageType == STORAGE_RBD { + diskConfig.Format = "raw" + data.Add(jsonutils.Marshal(diskConfig), "disk") + } else if storage.StorageType == STORAGE_CLOUD_EFFICIENCY || storage.StorageType == STORAGE_CLOUD_SSD { + if size < 20 || size > 32768 { + return nil, httperrors.NewInputParameterError("cloud_ssd or cloud_efficiency disk only support 20G ~ 32768G") + } + } else if storage.StorageType == STORAGE_PUBLIC_CLOUD { + if size < 5 || size > 2000 { + return nil, httperrors.NewInputParameterError("cloud disk only support 5G ~ 2000G") + } + } + hoststorages := HoststorageManager.Query().SubQuery() + hoststorage := make([]SHoststorage, 0) + if err := hoststorages.Query().Equals("storage_id", storage.Id).All(&hoststorage); err != nil { + return nil, err + } + if len(hoststorage) == 0 { + return nil, httperrors.NewInputParameterError("Storage[%s] must attach to a host", storage.Name) + } + if diskConfig.Size > storage.GetFreeCapacity() && !storage.IsEmulated { + return nil, httperrors.NewInputParameterError("Not enough free space") + } + if _, err := manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data); err != nil { + return nil, err + } + pendingUsage := SQuota{Storage: diskConfig.Size} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { + return nil, err + } + } + } + } + return data, nil +} + +func (disk *SDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { + disk.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) + disk.StartDiskCreateTask(ctx, userCred, false, "", "") +} + +func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { + kwargs := jsonutils.NewDict() + if rebuild { + kwargs.Add(jsonutils.JSONTrue, "rebuild") + } + if len(snapshot) > 0 { + kwargs.Add(jsonutils.NewString(snapshot), "snapshot") + } + if task, err := taskman.TaskManager.NewTask(ctx, "DiskCreateTask", self, userCred, kwargs, parentTaskId, "", nil); err != nil { + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + +func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { + log.Infof("Allocating disk on host %s ...", host.GetName()) + + templateId := self.GetTemplateId() + fsFormat := self.GetFsFormat() + + content := jsonutils.NewDict() + content.Add(jsonutils.NewString(self.DiskFormat), "format") + content.Add(jsonutils.NewInt(int64(self.DiskSize)), "size") + if len(snapshot) > 0 { + content.Add(jsonutils.NewString(snapshot), "snapshot") + } else if len(templateId) > 0 { + content.Add(jsonutils.NewString(templateId), "image_id") + } + if len(fsFormat) > 0 { + content.Add(jsonutils.NewString(fsFormat), "fs_format") + if fsFormat == "ext4" { + name := strings.ToLower(self.GetName()) + for _, key := range []string{"encrypt", "secret", "cipher", "private"} { + if strings.Index(key, name) > 0 { + content.Add(jsonutils.JSONTrue, "encryption") + break + } + } + } + } + if rebuild { + content.Add(jsonutils.JSONTrue, "rebuild") + } + return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content) +} + +func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if sizeStr, err := data.GetString("size"); err != nil { + return nil, err + } else if size, err := fileutils.GetSizeMb(sizeStr, 'M', 1024); err != nil { + return nil, err + } else if self.Status != DISK_READY { + return nil, httperrors.NewResourceNotReadyError("Resize disk when disk is READY") + } else if size < self.DiskSize { + return nil, httperrors.NewUnsupportOperationError("Disk cannot be thrink") + } else if size == self.DiskSize { + return nil, nil + } else { + addDisk := size - self.DiskSize + storage := self.GetStorage() + if addDisk > storage.GetFreeCapacity() && !storage.IsEmulated { + return nil, httperrors.NewOutOfResourceError("Not enough free space") + } + pendingUsage := SQuota{Storage: int(addDisk)} + if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), &pendingUsage); err != nil { + return nil, httperrors.NewOutOfQuotaError(err.Error()) + } + return nil, self.StartDiskResizeTask(ctx, userCred, int64(size), "", &pendingUsage) + } +} func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { if self.GetGuestDiskCount() > 0 { @@ -159,10 +365,6 @@ func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx) } -func (self *SDisk) StartAllocate(host *SHost, Storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool) { - -} - func (self *SDisk) GetTemplateId() string { return self.TemplateId } @@ -183,6 +385,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -307,6 +516,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extDisk.Refresh() self.Name = extDisk.GetName() self.Status = extDisk.GetStatus() self.DiskFormat = extDisk.GetDiskFormat() @@ -401,6 +611,7 @@ type SDiskConfig struct { Cache string // Mountpoint string // Backend string // stroageType + Medium string ImageProperties map[string]string } @@ -415,6 +626,11 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info } return &diskConfig, nil } + + // default backend and medium type + diskConfig.Backend = STORAGE_LOCAL + diskConfig.Medium = DISK_TYPE_HYBRID + diskStr, err := info.GetString() if err != nil { log.Errorf("invalid diskinfo format %s", err) @@ -432,6 +648,8 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info diskConfig.Driver = p } else if utils.IsInStringArray(p, osprofile.DISK_CACHE_MODES) { diskConfig.Cache = p + } else if utils.IsInStringArray(p, DISK_TYPES) { + diskConfig.Medium = p } else if p[0] == '/' { diskConfig.Mountpoint = p } else if p == "autoextend" { @@ -536,6 +754,60 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error { + params := jsonutils.NewDict() + params.Add(jsonutils.NewInt(size), "size") + if task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage); err != nil { + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + func (self *SDisk) StartDiskDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, isPurge bool) error { params := jsonutils.NewDict() if isPurge { diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index 82c63895df..1f69a83d0d 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -6,9 +6,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SGuestdiskManager struct { @@ -47,6 +49,23 @@ func (self *SGuestdisk) AllowDeleteItem(ctx context.Context, userCred mcclient.T return false } +func (self *SGuestdisk) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if data.Contains("index") { + if index, err := data.Int("index"); err != nil { + return nil, err + } else { + guestdisk := GuestdiskManager.Query().SubQuery() + count := guestdisk.Query().Filter(sqlchemy.Equals(guestdisk.Field("guest_id"), self.GuestId)). + Filter(sqlchemy.NotEquals(guestdisk.Field("disk_id"), self.DiskId)). + Filter(sqlchemy.Equals(guestdisk.Field("index"), index)).Count() + if count > 0 { + return nil, httperrors.NewInputParameterError("DISK Index %d has been occupied", index) + } + } + } + return self.SGuestJointsBase.ValidateUpdateData(ctx, userCred, query, data) +} + func (joint *SGuestdisk) Master() db.IStandaloneModel { return db.JointMaster(joint) } @@ -111,9 +130,11 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { templateId := disk.GetTemplateId() if len(templateId) > 0 { desc.Add(jsonutils.NewString(templateId), "template_id") - // hostcachedimg = Hostcachedimages.get_host_cachedimage(host, template_id) - // if hostcachedimg is not None: - // desc['image_path'] = hostcachedimg.path + storage := disk.GetStorage() + storagecacheimg := StoragecachedimageManager.GetStoragecachedimage(storage.StoragecacheId, templateId) + if storagecacheimg != nil { + desc.Add(jsonutils.NewString(storagecacheimg.Path), "image_path") + } } if host.HostType == HOST_TYPE_HYPERVISOR && disk.IsLocal() { desc.Add(jsonutils.NewString(disk.StorageId), "storage_id") @@ -129,6 +150,8 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } } desc.Add(jsonutils.NewString(disk.DiskFormat), "format") + desc.Add(jsonutils.NewInt(int64(self.Index)), "index") + tid := disk.GetTemplateId() if len(tid) > 0 { desc.Add(jsonutils.NewString(tid), "template_id") @@ -147,6 +170,25 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { return desc } +func (self *SGuestdisk) GetDetailedJson() *jsonutils.JSONDict { + desc := jsonutils.NewDict() + disk := self.GetDisk() + storage := disk.GetStorage() + if fs := disk.GetFsFormat(); len(fs) > 0 { + desc.Add(jsonutils.NewString(fs), "fs") + } + desc.Add(jsonutils.NewString(disk.DiskType), "disk_type") + desc.Add(jsonutils.NewInt(int64(self.Index)), "index") + desc.Add(jsonutils.NewInt(int64(disk.DiskSize)), "size") + desc.Add(jsonutils.NewString(disk.DiskFormat), "disk_format") + desc.Add(jsonutils.NewString(self.Driver), "driver") + desc.Add(jsonutils.NewString(self.CacheMode), "cache_mode") + desc.Add(jsonutils.NewString(self.AioMode), "aio_mode") + desc.Add(jsonutils.NewString(storage.MediumType), "medium_type") + desc.Add(jsonutils.NewString(storage.StorageType), "storage_type") + return desc +} + func (self *SGuestdisk) GetDetailedString() string { disk := self.GetDisk() var fs string diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index f35b77e915..48406df24f 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -39,6 +39,9 @@ type IGuestDriver interface { RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *SGuest, task taskman.ITask) error StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error + + RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error @@ -48,13 +51,15 @@ type IGuestDriver interface { StartGuestSyncstatusTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error + RequestSyncConfigOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error + RequestSyncstatusOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) - RequestStartOnHost(guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) + RequestStartOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) RequestStopOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error - StartDeleteGuestTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error + StartDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error RequestStopGuestForDelete(ctx context.Context, guest *SGuest, task taskman.ITask) error @@ -69,6 +74,23 @@ type IGuestDriver interface { CheckDiskTemplateOnStorage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, storageId string, task taskman.ITask) error GetGuestVncInfo(userCred mcclient.TokenCredential, guest *SGuest, host *SHost) (*jsonutils.JSONDict, error) + + RequestDetachDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error + GetDetachDiskStatus() ([]string, error) + CanKeepDetachDisk() bool + + RequestDeleteDetachedDisk(ctx context.Context, disk *SDisk, task taskman.ITask, isPurge bool) error + StartGuestDetachdiskTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + + StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error + RqeuestSuspendOnHost(ctx context.Context, guest *SGuest, task taskman.ITask) error + + AllowReconfigGuest() bool + DoGuestCreateDisksTask(ctx context.Context, guest *SGuest, task taskman.ITask) error + RequestChangeVmConfig(ctx context.Context, guest *SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error + + RequestGuestHotAddIso(ctx context.Context, guest *SGuest, path string, task taskman.ITask) error + RequestRebuildRootDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..5ff6910060 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -315,9 +315,11 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest if regutils.MatchIP4Addr(gn.IpAddr) || regutils.MatchIP6Addr(gn.Ip6Addr) { net.updateDnsRecord(&gn, false) if regutils.MatchIP4Addr(gn.IpAddr) { + // ?? // netman.get_manager().netmap_remove_node(gn.ip_addr) } } + // ?? // gn.Delete(ctx, userCred) err = gn.Delete(ctx, userCred) if err != nil { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 401bdbd1b0..c5dec34277 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "fmt" + "net/http" "strconv" "strings" "time" @@ -17,6 +18,7 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" @@ -32,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -535,6 +538,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m hypervisor = HYPERVISOR_BAREMETAL } + // base validate_create_data if data.Contains("prefer_baremetal") || data.Contains("prefer_host") { if !userCred.IsSystemAdmin() { return nil, httperrors.NewNotSufficientPrivilegeError("Only system admin can specify preferred host") @@ -572,7 +576,6 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, err } - } else { schedtags := make(map[string]string) if data.Contains("aggregate_strategy") { @@ -627,12 +630,16 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m } } + // default hypervisor + if len(hypervisor) == 0 { + hypervisor = HYPERVISOR_KVM + } + if !utils.IsInStringArray(hypervisor, HYPERVISORS) { return nil, httperrors.NewInputParameterError("Hypervisor %s not supported", hypervisor) } data.Add(jsonutils.NewString(hypervisor), "hypervisor") - for idx := 1; data.Contains(fmt.Sprintf("disk.%d", idx)); idx += 1 { diskJson, err := data.Get(fmt.Sprintf("disk.%d", idx)) if err != nil { @@ -819,13 +826,27 @@ func getGuestResourceRequirements(ctx context.Context, userCred mcclient.TokenCr func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { guest.SVirtualResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) - + tags := []string{"cpu_bound", "io_bound", "io_hardlimit"} + appTags := make([]string, 0) + for _, tag := range tags { + if data.Contains(tag) { + appTags = append(appTags, tag) + } + } + guest.setApptags(ctx, appTags, userCred) osProfileJson, _ := data.Get("__os_profile__") if osProfileJson != nil { guest.setOSProfile(ctx, userCred, osProfileJson) } } +func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred mcclient.TokenCredential) { + err := guest.SetMetadata(ctx, "app_tags", strings.Join(appTags, ","), userCred) + if err != nil { + log.Errorln(err) + } +} + func (manager *SGuestManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { pendingUsage := getGuestResourceRequirements(ctx, userCred, data, len(items)) @@ -890,7 +911,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") } - extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") if self.PendingDeleted { pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) @@ -904,6 +925,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token extra := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query) extra.Add(jsonutils.NewString(self.getNetworksDetails()), "networks") extra.Add(jsonutils.NewString(self.getDisksDetails()), "disks") + extra.Add(self.getDisksInfoDetails(), "disks_info") extra.Add(jsonutils.NewInt(int64(self.getDiskSize())), "disk") cdrom := self.getCdrom() if cdrom != nil { @@ -911,7 +933,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token } // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") - extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") extra.Add(jsonutils.NewString(strings.Join(self.getIPs(), ",")), "ips") extra.Add(jsonutils.NewString(self.getSecurityRules()), "security_rules") extra.Add(jsonutils.NewString(self.getIsolatedDeviceDetails()), "isolated_devices") @@ -930,6 +952,8 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token if zone != nil { extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") extra.Add(jsonutils.NewString(zone.GetName()), "zone") + extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") + extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") } return extra } @@ -952,6 +976,14 @@ func (self *SGuest) getDisksDetails() string { return buf.String() } +func (self *SGuest) getDisksInfoDetails() *jsonutils.JSONArray { + details := jsonutils.NewArray() + for _, disk := range self.GetDisks() { + details.Add(disk.GetDetailedJson()) + } + return details +} + func (self *SGuest) getIsolatedDeviceDetails() string { var buf bytes.Buffer for _, dev := range self.GetIsolatedDevices() { @@ -1098,7 +1130,7 @@ func (self *SGuest) getAdminSecgroup() *SSecurityGroup { return SecurityGroupManager.FetchSecgroupById(self.AdminSecgrpId) } -func (self *SGuest) getSecgroupName() string { +func (self *SGuest) GetSecgroupName() string { secgrp := self.getSecgroup() if secgrp != nil { return secgrp.GetName() @@ -1114,6 +1146,22 @@ func (self *SGuest) getAdminSecgroupName() string { return "" } +func (self *SGuest) GetSecRules() []secrules.SecurityRule { + return self.getSecRules() +} + +func (self *SGuest) getSecRules() []secrules.SecurityRule { + if secgrp := self.getSecgroup(); secgrp != nil { + return secgrp.getSecRules() + } + if rule, err := secrules.ParseSecurityRule(options.Options.DefaultSecurityRules); err == nil { + return []secrules.SecurityRule{*rule} + } else { + log.Errorf("Default SecurityRules error: %v", err) + } + return []secrules.SecurityRule{} +} + func (self *SGuest) getSecurityRules() string { secgrp := self.getSecgroup() if secgrp != nil { @@ -1138,7 +1186,7 @@ func (self *SGuest) GetIsolatedDevices() []SIsolatedDevice { func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, extVM cloudprovider.ICloudVM) error { diff, err := GuestManager.TableSpec().Update(self, func() error { - + extVM.Refresh() self.Name = extVM.GetName() self.Status = extVM.GetStatus() self.VcpuCount = extVM.GetVcpuCount() @@ -1439,6 +1487,122 @@ func (self *SGuest) attach2Disk(disk *SDisk, userCred mcclient.TokenCredential, return err } +func (self *SGuest) AllowPerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if err := self.StartSyncTask(ctx, userCred, false, ""); err != nil { + return nil, err + } + return nil, nil +} + +func (self *SGuest) AllowPerformDeploy(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + kwargs, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil, fmt.Errorf("Parse query body error") + } + if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { + var kpId string + if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { + keypair, _ := kwargs.GetString("keypair") + iKp, err := KeypairManager.FetchByIdOrName(userCred.GetProjectId(), keypair) + if err != nil { + return nil, err + } + if iKp == nil { + return nil, fmt.Errorf("Fetch keypair error") + } + kp := iKp.(*SKeypair) + kpId = kp.Id + } + if self.KeypairId != kpId { + self.GetModelManager().TableSpec().Update(self, func() error { + self.KeypairId = kpId + return nil + }) + kwargs.Set("reset_password", jsonutils.JSONTrue) + } + } + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) { + if self.Status == VM_RUNNING { + kwargs.Set("restart", jsonutils.JSONTrue) + } + err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") + if err != nil { + return nil, err + } + return nil, nil + } + return nil, httperrors.NewServerStatusError("Cannot deploy in status %s", self.Status) +} + +func (self *SGuest) AllowPerformAttachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) ValidateAttachDisk(ctx context.Context, disk *SDisk) error { + if disk.isAttached() { + return httperrors.NewInputParameterError("Disk %s has been attached", disk.Name) + } else if len(disk.GetPathAtHost(self.GetHost())) == 0 { + return httperrors.NewInputParameterError("Disk %s not belong the guest's host", disk.Name) + } else if disk.Status != DISK_READY { + return httperrors.NewInputParameterError("Disk in %s not able to attach", disk.Status) + } else if !utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY}) { + return httperrors.NewInputParameterError("Server in %s not able to attach disk", self.Status) + } + return nil +} + +func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if diskId, err := data.GetString("disk_id"); err != nil { + return nil, err + } else { + if disk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId); err != nil { + return nil, err + } else if disk == nil { + return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) + } else if err := self.ValidateAttachDisk(ctx, disk.(*SDisk)); err != nil { + return nil, err + } else { + driver, _ := data.GetString("driver") + cache, _ := data.GetString("cache") + mountpoint, _ := data.GetString("mountpoint") + if err := self.attach2Disk(disk.(*SDisk), userCred, driver, cache, mountpoint); err != nil { + return nil, err + } else { + self.StartSyncTask(ctx, userCred, false, "") + } + } + } + return nil, nil +} + +func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, fw_only bool, parentTaskId string) error { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING}) { + return httperrors.NewResourceBusyError("Cannot sync in status %s", self.Status) + } + data := jsonutils.NewDict() + if fw_only { + data.Add(jsonutils.JSONTrue, "fw_only") + } else if err := self.SetStatus(userCred, VM_SYNC_CONFIG, ""); err != nil { + log.Errorf(err.Error()) + return err + } + if task, err := taskman.TaskManager.NewTask(ctx, "GuestSyncConfTask", self, userCred, data, parentTaskId, "", nil); err != nil { + log.Errorf(err.Error()) + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + type sSyncDiskPair struct { disk *SDisk vdisk cloudprovider.ICloudDisk @@ -1905,11 +2069,26 @@ func (self *SGuest) insertIso(imageId string) bool { return cdrom.insertIso(imageId) } -func (self *SGuest) insertIsoSucc(imageId string, path string, size int, name string) bool { +func (self *SGuest) InsertIsoSucc(imageId string, path string, size int, name string) bool { cdrom := self.getCdrom() return cdrom.insertIsoSucc(imageId, path, size, name) } +func (self *SGuest) GetDetailsIso(userCred mcclient.TokenCredential) jsonutils.JSONObject { + cdrom := self.getCdrom() + desc := jsonutils.NewDict() + if len(cdrom.ImageId) > 0 { + desc.Set("image_id", jsonutils.NewString(cdrom.ImageId)) + desc.Set("status", jsonutils.NewString("inserting")) + } + if len(cdrom.Path) > 0 { + desc.Set("name", jsonutils.NewString(cdrom.Name)) + desc.Set("size", jsonutils.NewInt(int64(cdrom.Size))) + desc.Set("status", jsonutils.NewString("ready")) + } + return desc +} + func (self *SGuest) StartInsertIsoTask(ctx context.Context, imageId string, hostId string, userCred mcclient.TokenCredential, parentTaskId string) error { self.insertIso(imageId) @@ -1917,7 +2096,7 @@ func (self *SGuest) StartInsertIsoTask(ctx context.Context, imageId string, host data.Add(jsonutils.NewString(imageId), "image_id") data.Add(jsonutils.NewString(hostId), "host_id") - task, err := taskman.TaskManager.NewTask(ctx, "GuestInsertISOTask", self, userCred, data, parentTaskId, "", nil) + task, err := taskman.TaskManager.NewTask(ctx, "GuestInsertIsoTask", self, userCred, data, parentTaskId, "", nil) if err != nil { return err } @@ -1935,6 +2114,15 @@ func (self *SGuest) StartGueststartTask(ctx context.Context, userCred mcclient.T return nil } +func (self *SGuest) StartGuestCreateDiskTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateDiskTask", self, userCred, data, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SGuest) StartSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { return self.GetDriver().StartGuestSyncstatusTask(self, ctx, userCred, parentTaskId) } @@ -1954,7 +2142,55 @@ func (self *SGuest) StartDeleteGuestTask(ctx context.Context, userCred mcclient. params.Add(jsonutils.JSONTrue, "override_pending_delete") } self.SetStatus(userCred, VM_START_DELETE, "") - return self.GetDriver().StartDeleteGuestTask(self, ctx, userCred, params, parentTaskId) + return self.GetDriver().StartDeleteGuestTask(ctx, userCred, self, params, parentTaskId) +} + +func (self *SGuest) AllowPerformAssignSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) AllowPerformRevokeSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformRevokeSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_SUSPEND}) { + return nil, httperrors.NewInputParameterError("Cannot revoke security rules in status %s", self.Status) + } else { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.SecgrpId = "" + return nil + }); err != nil { + return nil, err + } + if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil { + return nil, err + } + } + return nil, nil +} + +func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_SUSPEND}) { + return nil, httperrors.NewInputParameterError("Cannot assign security rules in status %s", self.Status) + } else { + if secgrp, err := data.GetString("secgrp"); err != nil { + return nil, err + } else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secgrp); err != nil { + return nil, httperrors.NewNotFoundError("SecurityGroup %s not found", secgrp) + } else { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.SecgrpId = sg.GetId() + return nil + }); err != nil { + return nil, err + } + if err := self.StartSyncTask(ctx, userCred, true, ""); err != nil { + return nil, err + } + } + } + return nil, nil } func (self *SGuest) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -1974,13 +2210,265 @@ func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCre return nil, err } -func (self *SGuest) detachDisk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential) { +func (self *SGuest) AllowPerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + imageId, _ := data.GetString("image_id") + if utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_ADMIN}) { + if !data.Contains("image_id") { + gdc := self.CategorizeDisks() + imageId = gdc.Root.GetTemplateId() + if len(imageId) == 0 { + return nil, httperrors.NewBadRequestError("No template for root disk") + } + img, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) + if err != nil { + return nil, httperrors.NewBadRequestError("Template %s not accessible: %s", imageId, err.Error()) + } + osType, _ := img.Properties["os_type"] + osName := self.GetMetadata("os_name", userCred) + if len(osName) == 0 && len(osType) == 0 && strings.ToLower(osType) != strings.ToLower(osName) { + return nil, httperrors.NewBadRequestError("Cannot switch OS between %s-%s", osName, osType) + } + } + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) + var needStop = false + if self.Status == VM_RUNNING { + needStop = true + } + err := self.StartRebuildRootTask(ctx, userCred, imageId, needStop, autoStart) + return nil, err + } + return nil, httperrors.NewInvalidStatusError("Cannot reset root in status %s", self.Status) +} + +func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, needStop, autoStart bool) error { + data := jsonutils.NewDict() + data.Set("image_id", jsonutils.NewString(imageId)) + if needStop { + data.Set("need_stop", jsonutils.JSONTrue) + } + if autoStart { + data.Set("auto_start", jsonutils.JSONTrue) + } + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + task, err := taskman.TaskManager.NewTask(ctx, "BaremetalServerRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } else { + task, err := taskman.TaskManager.NewTask(ctx, "GuestRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } + return nil +} + +func (self *SGuest) DetachDisk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential) { guestdisk := self.GetGuestDisk(disk.Id) if guestdisk != nil { guestdisk.Detach(ctx, userCred) } } +func (self *SGuest) AllowPerformDetachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + diskId, err := data.GetString("disk_id") + if err != nil { + return nil, err + } + keepDisk := jsonutils.QueryBoolean(data, "keep_disk", false) + iDisk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId) + if err != nil { + return nil, err + } + disk := iDisk.(*SDisk) + if disk != nil { + if self.isAttach2Disk(disk) { + detachDiskStatus, err := self.GetDriver().GetDetachDiskStatus() + if err != nil { + return nil, err + } + if keepDisk && !self.GetDriver().CanKeepDetachDisk() { + return nil, httperrors.NewInputParameterError("Cannot keep detached disk") + } + if utils.IsInStringArray(self.Status, detachDiskStatus) { + if disk.Status == DISK_INIT { + disk.SetStatus(userCred, DISK_DETACHING, "") + } + taskData := jsonutils.NewDict() + taskData.Add(jsonutils.NewString(diskId), "disk_id") + taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk") + self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "") + return nil, nil + } else { + return nil, httperrors.NewInvalidStatusError("Server in %s not able to detach disk", self.Status) + } + } else { + return nil, httperrors.NewInvalidStatusError("Disk %s not attached", diskId) + } + } + return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) +} + +func (self *SGuest) AllowPerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) || self.IsAdmin(userCred) +} + +func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Status, []string{VM_READY}) { + return nil, httperrors.NewInvalidStatusError("Cannot change config in %s", self.Status) + } + if !self.GetDriver().AllowReconfigGuest() { + return nil, httperrors.NewInvalidStatusError("Not allow to change config") + } + host := self.GetHost() + if host == nil { + return nil, httperrors.NewInvalidStatusError("No valid host") + } + var addCpu, addMem int + confs := jsonutils.NewDict() + vcpuCount, err := data.GetString("vcpu_count") + if err == nil { + nVcpu, err := strconv.ParseInt(vcpuCount, 10, 0) + if err != nil { + return nil, httperrors.NewBadRequestError("Params vcpu_count parse error") + } + err = confs.Add(jsonutils.NewInt(nVcpu), "vcpu_count") + if err != nil { + return nil, httperrors.NewBadRequestError("Params vcpu_count parse error") + } + addCpu = int(nVcpu - int64(self.VcpuCount)) + } + vmemSize, err := data.GetString("vmem_size") + if err == nil { + if !regutils.MatchSize(vmemSize) { + return nil, httperrors.NewBadRequestError("Memory size must be number[+unit], like 256M, 1G or 256") + } + nVmem, err := fileutils.GetSizeMb(vmemSize, 'M', 1024) + if err != nil { + httperrors.NewBadRequestError("Params vmem_size parse error") + } + err = confs.Add(jsonutils.NewInt(int64(nVmem)), "vmem_size") + if err != nil { + return nil, httperrors.NewBadRequestError("Params vmem_size parse error") + } + addMem = nVmem - self.VmemSize + } + disks := self.GetDisks() + var addDisk int + var diskIdx = 1 + var newDiskIdx = 0 + var diskSizes = make(map[string]int, 0) + var newDisks = jsonutils.NewDict() + var resizeDisks = jsonutils.NewArray() + for { + diskNum := fmt.Sprintf("disk.%d", diskIdx) + diskDesc, err := data.Get(diskNum) + if err != nil { + break + } + diskConf, err := parseDiskInfo(ctx, userCred, diskDesc) + if err != nil { + return nil, httperrors.NewBadRequestError("Parse disk info error: %s", err) + } + if diskConf.Size > 0 { + if diskIdx >= len(disks) { + newDisks.Add(jsonutils.Marshal(diskConf), fmt.Sprintf("disk.%d", newDiskIdx)) + newDiskIdx += 1 + addDisk += diskConf.Size + storage := host.GetLeastUsedStorage(diskConf.Backend) + _, ok := diskSizes[storage.Id] + if !ok { + diskSizes[storage.Id] = 0 + } + diskSizes[storage.Id] = diskSizes[storage.Id] + diskConf.Size + } else { + disk := disks[diskIdx].GetDisk() + oldSize := disk.DiskSize + if diskConf.Size < oldSize { + return nil, httperrors.NewInputParameterError("Cannot reduce disk size") + } else if diskConf.Size > oldSize { + arr := jsonutils.NewArray(jsonutils.NewString(disks[diskIdx].DiskId), jsonutils.NewInt(int64(diskConf.Size))) + resizeDisks.Add(arr) + addDisk += diskConf.Size - oldSize + storage := disks[diskIdx].GetDisk().GetStorage() + _, ok := diskSizes[storage.Id] + if !ok { + diskSizes[storage.Id] = 0 + } + diskSizes[storage.Id] = diskSizes[storage.Id] + diskConf.Size - oldSize + } + } + } + diskIdx += 1 + } + + for storageId, needSize := range diskSizes { + iStorage, err := StorageManager.FetchById(storageId) + if err != nil { + return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) + } + storage := iStorage.(*SStorage) + if storage.GetFreeCapacity() < needSize { + return nil, httperrors.NewInsufficientResourceError("Not enough free space") + } + } + if newDisks.Length() > 0 { + confs.Add(newDisks, "create") + } + if resizeDisks.Length() > 0 { + confs.Add(resizeDisks, "resize") + } + if jsonutils.QueryBoolean(data, "auto_start", false) { + confs.Add(jsonutils.NewBool(true), "auto_start") + } + pendingUsage := &SQuota{} + if addCpu > 0 { + pendingUsage.Cpu = addCpu + } + if addMem > 0 { + pendingUsage.Memory = addMem + } + if addDisk > 0 { + pendingUsage.Storage = addDisk + } + if !pendingUsage.IsEmpty() { + err := QuotaManager.CheckSetPendingQuota(ctx, userCred, userCred.GetProjectId(), pendingUsage) + if err != nil { + return nil, httperrors.NewBadRequestError("Check set pending quota error %s", err) + } + } + if newDisks.Length() > 0 { + err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage) + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) + return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err) + } + } + self.StartChangeConfigTask(ctx, userCred, confs, "", pendingUsage) + return nil, nil +} + +func (self *SGuest) StartChangeConfigTask(ctx context.Context, userCred mcclient.TokenCredential, + data *jsonutils.JSONDict, parentTaskId string, pendingUsage quotas.IQuota) error { + self.SetStatus(userCred, VM_CHANGE_FLAVOR, "") + task, err := taskman.TaskManager.NewTask(ctx, "GuestChangeConfigTask", self, userCred, data, parentTaskId, "", pendingUsage) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) { for _, guestdisk := range self.GetDisks() { disk := guestdisk.GetDisk() @@ -1988,7 +2476,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token if utils.IsInStringArray(storage.StorageType, sysutils.LOCAL_STORAGE_TYPES) || disk.DiskType == DISK_TYPE_SYS || disk.DiskType == DISK_TYPE_SWAP || self.Hypervisor == HYPERVISOR_ALIYUN { disk.DoPendingDelete(ctx, userCred) } else { - self.detachDisk(ctx, disk, userCred) + self.DetachDisk(ctx, disk, userCred) } } self.SVirtualResourceBase.DoPendingDelete(ctx, userCred) @@ -2028,7 +2516,24 @@ func (self *SGuest) StartUndeployGuestTask(ctx context.Context, userCred mcclien } func (self *SGuest) LeaveAllGroups(userCred mcclient.TokenCredential) { - // TODO + groupGuests := make([]SGroupguest, 0) + q := GroupguestManager.Query() + err := q.Filter(sqlchemy.Equals(q.Field("guest_id"), self.Id)).All(&groupGuests) + if err != nil { + log.Errorln(err.Error()) + return + } + for _, gg := range groupGuests { + gg.Delete(context.Background(), userCred) + var group SGroup + gq := GroupManager.Query() + err := gq.Filter(sqlchemy.Equals(gq.Field("id"), gg.SrvtagId)).First(&group) + if err != nil { + log.Errorln(err.Error()) + return + } + db.OpsLog.LogDetachEvent(self, &group, userCred, nil) + } } func (self *SGuest) DetachAllNetworks(ctx context.Context, userCred mcclient.TokenCredential) error { @@ -2076,9 +2581,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } @@ -2110,6 +2615,29 @@ func (self *SGuest) PerformSyncstatus(ctx context.Context, userCred mcclient.Tok return nil, err } +func (self *SGuest) isNotRunningStatus(status string) bool { + if status == VM_READY || status == VM_SUSPEND { + return true + } + return false +} + +func (self *SGuest) PerformStatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + preStatus := self.Status + _, err := self.SVirtualResourceBase.PerformStatus(ctx, userCred, query, data) + if err != nil { + return nil, err + } + if preStatus != self.Status && !self.isNotRunningStatus(preStatus) && self.isNotRunningStatus(self.Status) { + db.OpsLog.LogEvent(self, db.ACT_STOP, "", userCred) + if self.Status == VM_READY && !self.DisableDelete.Bool() && self.ShutdownBehavior == SHUTDOWN_TERMINATE { + err = self.StartAutoDeleteGuestTask(ctx, userCred, "") + return nil, err + } + } + return nil, nil +} + type SDeployConfig struct { Path string Action string @@ -2202,6 +2730,16 @@ func (self *SGuest) getExtraOptions() jsonutils.JSONObject { return self.GetMetadataJson("extra_options", nil) } +/* +func (self *SGuest) GetFlavor() *SFlav { + +} + +func (self *SGuest) getFlavorName() string { + f := self.GetFlavor() +} +*/ + func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *jsonutils.JSONDict { desc := jsonutils.NewDict() @@ -2218,13 +2756,14 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j desc.Add(jsonutils.NewString(self.getBios()), "bios") desc.Add(jsonutils.NewString(self.BootOrder), "boot_order") + // isolated devices isolatedDevs := IsolatedDeviceManager.generateJsonDescForGuest(self) - desc.Add(jsonutils.NewArray(isolatedDevs...), "solated_devices") + desc.Add(jsonutils.NewArray(isolatedDevs...), "isolated_devices") + // nics, domain jsonNics := make([]jsonutils.JSONObject, 0) nics := self.GetNetworks() domain := options.Options.DNSDomain - if nics != nil && len(nics) > 0 { for _, nic := range nics { nicDesc := nic.getJsonDescAtHost(host) @@ -2238,6 +2777,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j desc.Add(jsonutils.NewArray(jsonNics...), "nics") desc.Add(jsonutils.NewString(domain), "domain") + // disks jsonDisks := make([]jsonutils.JSONObject, 0) disks := self.GetDisks() if disks != nil && len(disks) > 0 { @@ -2248,17 +2788,19 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *j } desc.Add(jsonutils.NewArray(jsonDisks...), "disks") + // cdrom cdDesc := self.getCdrom().getJsonDesc() if cdDesc != nil { desc.Add(cdDesc, "cdrom") } + // tenant tc, _ := self.GetTenantCache(ctx) if tc != nil { desc.Add(jsonutils.NewString(tc.GetName()), "tenant") } - desc.Add(jsonutils.NewString(self.ProjectId), "tenant_id") + // flavor // desc.Add(jsonuitls.NewString(self.getFlavorName()), "flavor") @@ -2516,6 +3058,26 @@ func (self *SGuest) isAllDisksReady() bool { return ready } +func (self *SGuest) AllowPerformSuspend(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformSuspend(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status == VM_RUNNING { + err := self.StartSuspendTask(ctx, userCred) + return nil, err + } + return nil, httperrors.NewInvalidStatusError("Cannot suspend VM in status %s", self.Status) +} + +func (self *SGuest) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential) error { + err := self.SetStatus(userCred, VM_SUSPEND, "do suspend") + if err != nil { + return err + } + return self.GetDriver().StartSuspendTask(ctx, userCred, self, nil, "") +} + func (self *SGuest) AllowPerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, @@ -2528,20 +3090,36 @@ func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCre if utils.IsInStringArray(self.Status, []string{VM_READY, VM_START_FAILED, VM_SAVE_DISK_FAILED, VM_SUSPEND}) { if self.isAllDisksReady() { var kwargs *jsonutils.JSONDict - if data == nil { + if data != nil { kwargs = data.(*jsonutils.JSONDict) } err := self.GetDriver().PerformStart(ctx, userCred, self, kwargs) return nil, err } else { - msg := "Some disk not ready" - return nil, httperrors.NewResourceNotReadyError(msg) + return nil, httperrors.NewInvalidStatusError("Some disk not ready") } } else { return nil, httperrors.NewInvalidStatusError("Cannot do start server in status %s", self.Status) } } +func (self *SGuest) AllowPerformReset(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + isHard := jsonutils.QueryBoolean(data, "is_hard", false) + if self.Status == VM_RUNNING || self.Status == VM_STOP_FAILED { + self.GetDriver().StartGuestResetTask(self, ctx, userCred, isHard, "") + return nil, nil + } + return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status) +} + func (self *SGuest) AllowPerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, @@ -2600,6 +3178,36 @@ func (self *SGuest) GetDetailsVnc(ctx context.Context, userCred mcclient.TokenCr } } +func (self *SGuest) AllowGetDetailsMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) GetDetailsMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_SNAPSHOT_STREAM}) { + cmd, err := query.GetString("command") + if err != nil { + return nil, err + } + return self.SendMonitorCommand(ctx, userCred, cmd) + } + return nil, httperrors.NewInvalidStatusError("Cannot send command in status %s", self.Status) +} + +func (self *SGuest) SendMonitorCommand(ctx context.Context, userCred mcclient.TokenCredential, cmd string) (jsonutils.JSONObject, error) { + host := self.GetHost() + url := fmt.Sprintf("%s/servers/%s/monitor", host.ManagerUri, self.Id) + header := http.Header{} + header.Add("X-Auth-Token", userCred.GetTokenString()) + body := jsonutils.NewDict() + body.Add(jsonutils.NewString(cmd), "cmd") + _, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + if err != nil { + return nil, err + } + ret := res.(*jsonutils.JSONDict) + return ret, nil +} + func (self *SGuest) GetKeypairPublicKey() string { keypair := self.getKeypair() if keypair != nil { diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go new file mode 100644 index 0000000000..3f162d75da --- /dev/null +++ b/pkg/compute/models/hostdrivers.go @@ -0,0 +1,39 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +) + +type IHostDriver interface { + GetHostType() string + CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error + RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error + RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error + RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error + RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error +} + +var hostDrivers map[string]IHostDriver + +func init() { + hostDrivers = make(map[string]IHostDriver) +} + +func RegisterHostDriver(driver IHostDriver) { + hostDrivers[driver.GetHostType()] = driver +} + +func GetHostDriver(hostType string) IHostDriver { + driver, ok := hostDrivers[hostType] + if ok { + return driver + } else { + log.Fatalf("Unsupported hostType %s", hostType) + return nil + } +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index f4de4e2228..6b47c4e370 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" @@ -65,7 +66,7 @@ const ( HOST_STATUS_CONVERTING = "converting" ) -var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN} +var HOST_TYPES = []string{HOST_TYPE_BAREMETAL, HOST_TYPE_HYPERVISOR, HOST_TYPE_ESXI, HOST_TYPE_KUBELET, HOST_TYPE_XEN, HOST_TYPE_ALIYUN} var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN} type SHostManager struct { @@ -751,9 +752,19 @@ func (self *SHost) DeleteBaremetalnetwork(ctx context.Context, userCred mcclient } } -func (manager *SHostManager) getHostsByZone(zone *SZone) ([]SHost, error) { +func (self *SHost) GetHostDriver() IHostDriver { + if !utils.IsInStringArray(self.HostType, HOST_TYPES) { + log.Fatalf("Unsupported host type %s", self.HostType) + } + return GetHostDriver(self.HostType) +} + +func (manager *SHostManager) getHostsByZone(zone *SZone, provider *SCloudprovider) ([]SHost, error) { hosts := make([]SHost, 0) q := manager.Query().Equals("zone_id", zone.Id) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &hosts) if err != nil { log.Errorf("%s", err) @@ -762,12 +773,12 @@ func (manager *SHostManager) getHostsByZone(zone *SZone) ([]SHost, error) { return hosts, nil } -func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { +func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { localHosts := make([]SHost, 0) remoteHosts := make([]cloudprovider.ICloudHost, 0) syncResult := compare.SyncResult{} - dbHosts, err := manager.getHostsByZone(zone) + dbHosts, err := manager.getHostsByZone(zone, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -1457,6 +1468,8 @@ func (self *SHost) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict if zone != nil { extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") + extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") } server := self.getBaremetalServer() if server != nil { @@ -1533,6 +1546,22 @@ func (manager *SHostManager) GetHostsByManagerAndRegion(managerId string, region return ret } +func (self *SHost) StartImageCacheTask(ctx context.Context, userCred mcclient.TokenCredential, imageId, parentTaskId string, isForce bool) error { + //Todo + // HostcachedimagesManager.Register(userCred, self, imageId) + data := jsonutils.NewDict() + data.Set("image_id", jsonutils.NewString(imageId)) + if isForce { + data.Set("is_force", jsonutils.JSONTrue) + } + task, err := taskman.TaskManager.NewTask(ctx, "StorageCacheImageTask", self, userCred, data, parentTaskId, "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + func (self *SHost) Request(userCred mcclient.TokenCredential, method string, url string, headers http.Header, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { s := auth.GetSession(userCred, "", "") _, ret, err := s.JSONRequest(self.ManagerUri, "", method, url, headers, body) diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index e4df6a1ea0..a2d86c4f5c 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -458,6 +458,7 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, extNet cloudprovider.ICloudNetwork) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extNet.Refresh() self.Name = extNet.GetName() self.Status = extNet.GetStatus() self.GuestIpStart = extNet.GetIpStart() diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 38171617d4..4347ea48c9 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -3,6 +3,7 @@ package models import ( "context" "fmt" + "sort" "strconv" "strings" @@ -11,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/sqlchemy" @@ -39,6 +41,25 @@ type SSecurityGroupRule struct { SecgroupID string `width:"128" charset:"ascii" create:"required"` } +type SecurityGroupRuleSet []SSecurityGroupRule + +func (v SecurityGroupRuleSet) Len() int { + return len(v) +} + +func (v SecurityGroupRuleSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurityGroupRuleSet) Less(i, j int) bool { + if v[i].Priority < v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + func (manager *SSecurityGroupRuleManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return true } @@ -131,6 +152,8 @@ func (manager *SSecurityGroupRuleManager) ValidateCreateData( key += ":" } fields = append(fields, key) + } else if field == "cidr" { + data.Add(jsonutils.NewString("0.0.0.0/0"), "cidr") } } if _, err := secrules.ParseSecurityRule(strings.Join(fields, " ")); err != nil { @@ -167,7 +190,11 @@ func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred fields = append(fields, self.CIDR) } case "protocol": - fields = append(fields, self.Protocol) + protocol := self.Protocol + if protocol == "" { + protocol = secrules.PROTO_ANY + } + fields = append(fields, protocol) case "ports": if len(self.Ports) > 0 { fields = append(fields, self.Ports) @@ -181,7 +208,7 @@ func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred return self.SResourceBase.ValidateUpdateData(ctx, userCred, query, data) } -func (self *SSecurityGroupRule) GetRule() string { +func (self *SSecurityGroupRule) String() string { var fields []string for _, field := range []string{"direction", "action", "cidr", "protocol", "ports"} { switch field { @@ -193,11 +220,15 @@ func (self *SSecurityGroupRule) GetRule() string { case "action": fields = append(fields, self.Action) case "cidr": - if len(self.CIDR) > 0 { + if len(self.CIDR) > 0 && self.CIDR != "0.0.0.0/0" { fields = append(fields, self.CIDR) } case "protocol": - fields = append(fields, self.Protocol) + protocol := self.Protocol + if protocol == "" { + protocol = secrules.PROTO_ANY + } + fields = append(fields, protocol) case "ports": if len(self.Ports) > 0 { fields = append(fields, self.Ports) @@ -207,12 +238,49 @@ func (self *SSecurityGroupRule) GetRule() string { return fields[0] + strings.Join(fields[1:], " ") } +func (self *SSecurityGroupRule) SingleRules() ([]secrules.SecurityRule, error) { + rules := make([]secrules.SecurityRule, 0) + ruleStr := self.String() + if rule, err := secrules.ParseSecurityRule(ruleStr); err != nil { + return nil, err + } else if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + _rule := secrules.SecurityRule{ + Priority: int(self.Priority), + Action: rule.Action, + IPNet: rule.IPNet, + Protocol: rule.Protocol, + Direction: rule.Direction, + PortStart: -1, + PortEnd: -1, + Ports: []int{port}, + Description: self.Description, + } + rules = append(rules, _rule) + } + } else { + _rule := secrules.SecurityRule{ + Priority: int(self.Priority), + Action: rule.Action, + IPNet: rule.IPNet, + Protocol: rule.Protocol, + Direction: rule.Direction, + PortStart: rule.PortStart, + PortEnd: rule.PortEnd, + Ports: []int{}, + Description: self.Description, + } + rules = append(rules, _rule) + } + return rules, nil +} + func (self *SSecurityGroupRule) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) log.Debugf("POST Create %s", data) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } @@ -220,7 +288,7 @@ func (self *SSecurityGroupRule) PreDelete(ctx context.Context, userCred mcclient self.SResourceBase.PreDelete(ctx, userCred) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } @@ -229,6 +297,111 @@ func (self *SSecurityGroupRule) PostUpdate(ctx context.Context, userCred mcclien log.Debugf("POST Update %s", data) if secgroup := self.GetSecGroup(); secgroup != nil { - secgroup.DoSync() + secgroup.DoSync(ctx, userCred) } } + +func (manager *SSecurityGroupRuleManager) getRulesBySecurityGroup(secgroup *SSecurityGroup) ([]SSecurityGroupRule, error) { + rules := make([]SSecurityGroupRule, 0) + q := manager.Query().Equals("secgroup_id", secgroup.Id) + if err := db.FetchModelObjects(manager, q, &rules); err != nil { + return nil, err + } + return rules, nil +} + +func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, rules []secrules.SecurityRule) ([]SSecurityGroupRule, []SSecurityGroupRule, compare.SyncResult) { + syncResult := compare.SyncResult{} + + if dbRules, err := manager.getRulesBySecurityGroup(secgroup); err != nil { + return nil, nil, syncResult + } else { + + sort.Sort(SecurityGroupRuleSet(dbRules)) + sort.Sort(secrules.SecurityRuleSet(rules)) + + i, j := 0, 0 + for i < len(rules) || j < len(dbRules) { + if i < len(rules) && j < len(dbRules) { + dbStr := dbRules[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(dbStr, ruleStr) + if cmp == 0 { + if dbRules[j].Description != rules[i].Description { + if _, err := manager.TableSpec().Update(dbRules[j], func() error { + dbRules[j].Description = rules[i].Description + return nil + }); err != nil { + log.Errorf("Update SecurityGroupRule failed: %v", err) + } + } + i += 1 + j += 1 + } else if cmp > 0 { + if err := dbRules[j].Delete(ctx, userCred); err != nil { + syncResult.AddError(err) + } else { + syncResult.Delete() + } + j += 1 + } else { + if _, err := manager.newFromCloudSecurityGroup(rules[i], secgroup); err != nil { + syncResult.AddError(err) + } else { + syncResult.Add() + } + i += 1 + } + } else if i >= len(rules) { + if err := dbRules[j].Delete(ctx, userCred); err != nil { + syncResult.AddError(err) + } else { + syncResult.Delete() + } + j += 1 + } else if j >= len(dbRules) { + if _, err := manager.newFromCloudSecurityGroup(rules[i], secgroup); err != nil { + syncResult.AddError(err) + } else { + syncResult.Add() + } + i += 1 + } + } + } + return nil, nil, syncResult +} + +func (manager *SSecurityGroupRuleManager) newFromCloudSecurityGroup(rule secrules.SecurityRule, secgroup *SSecurityGroup) (*SSecurityGroupRule, error) { + protocol := rule.Protocol + if rule.Protocol == "any" { + protocol = "" + } + ports, _ports := "", make([]string, len(rule.Ports)) + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + _ports = append(_ports, fmt.Sprintf("%d", port)) + } + ports = strings.Join(_ports, ",") + } else if rule.PortStart != 0 || rule.PortEnd != 0 { + if rule.PortStart == rule.PortEnd { + ports = fmt.Sprintf("%d", rule.PortStart) + } else { + ports = fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd) + } + } + secrule := &SSecurityGroupRule{ + Priority: int64(rule.Priority), + Protocol: protocol, + Ports: ports, + Direction: string(rule.Direction), + CIDR: rule.IPNet.String(), + Action: string(rule.Action), + Description: rule.Description, + SecgroupID: secgroup.Id, + } + if err := manager.TableSpec().Insert(secrule); err != nil { + return nil, err + } + return secrule, nil +} diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index 1fc2ca24bd..e2418d3ed1 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -3,12 +3,17 @@ package models import ( "context" "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/sqlchemy" ) @@ -94,16 +99,28 @@ func (self *SSecurityGroup) getSecurityRules() (rules []SSecurityGroupRule) { sql := secgrouprules.Query().Filter(sqlchemy.Equals(secgrouprules.Field("secgroup_id"), self.Id)) if err := db.FetchModelObjects(SecurityGroupRuleManager, sql, &rules); err != nil { log.Errorf("GetGuests fail %s", err) - return nil + return } return } +func (self *SSecurityGroup) getSecRules() []secrules.SecurityRule { + rules := make([]secrules.SecurityRule, 0) + for _, _rule := range self.getSecurityRules() { + singleRules, err := _rule.SingleRules() + if err != nil { + log.Errorf(err.Error()) + } + rules = append(rules, singleRules...) + } + return rules +} + func (self *SSecurityGroup) getSecurityRuleString() string { secgrouprules := self.getSecurityRules() var rules []string for _, rule := range secgrouprules { - rules = append(rules, rule.GetRule()) + rules = append(rules, rule.String()) } return strings.Join(rules, SECURITY_GROUP_SEPARATOR) } @@ -158,11 +175,125 @@ func (self *SSecurityGroup) PerformClone(ctx context.Context, userCred mcclient. return nil, nil } -func (self *SSecurityGroup) DoSync() { +func (manager *SSecurityGroupManager) getSecurityGroups() ([]SSecurityGroup, error) { + secgroups := make([]SSecurityGroup, 0) + q := manager.Query() + if err := db.FetchModelObjects(manager, q, &secgroups); err != nil { + return nil, err + } else { + return secgroups, nil + } +} + +func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { + localSecgroups := make([]SSecurityGroup, 0) + remoteSecgroups := make([]cloudprovider.ICloudSecurityGroup, 0) + syncResult := compare.SyncResult{} + + if dbSecgroups, err := manager.getSecurityGroups(); err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } else { + removed := make([]SSecurityGroup, 0) + commondb := make([]SSecurityGroup, 0) + commonext := make([]cloudprovider.ICloudSecurityGroup, 0) + added := make([]cloudprovider.ICloudSecurityGroup, 0) + if err := compare.CompareSets(dbSecgroups, secgroups, &removed, &commondb, &commonext, &added); err != nil { + syncResult.Error(err) + return nil, nil, syncResult + } + + for i := 0; i < len(commondb); i += 1 { + if rules, err := commonext[i].GetRules(); err != nil { + syncResult.Error(err) + } else if len(rules) > 0 { + if err = commondb[i].SyncWithCloudSecurityGroup(userCred, commonext[i]); err != nil { + syncResult.UpdateError(err) + } else { + localSecgroups = append(localSecgroups, commondb[i]) + remoteSecgroups = append(remoteSecgroups, commonext[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, &commondb[i], rules) + syncResult.Update() + } + } + } + + for i := 0; i < len(added); i += 1 { + if rules, err := added[i].GetRules(); err != nil { + syncResult.AddError(err) + } else if len(rules) > 0 { + if new, err := manager.newFromCloudVpc(added[i]); err != nil { + syncResult.AddError(err) + } else { + localSecgroups = append(localSecgroups, *new) + remoteSecgroups = append(remoteSecgroups, added[i]) + SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules) + syncResult.Add() + } + } + } + } + return localSecgroups, remoteSecgroups, syncResult +} + +func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup) error { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extSec.Refresh() + self.Name = extSec.GetName() + self.Description = extSec.GetDescription() + return nil + }); err != nil { + log.Errorf("syncWithCloudSecurityGroup error %s", err) + return err + } + return nil +} + +func (manager *SSecurityGroupManager) newFromCloudVpc(extSec cloudprovider.ICloudSecurityGroup) (*SSecurityGroup, error) { + secgroup := SSecurityGroup{} + secgroup.SetModelManager(manager) + secgroup.Name = extSec.GetName() + secgroup.ExternalId = extSec.GetGlobalId() + secgroup.Description = extSec.GetDescription() + + if err := manager.TableSpec().Insert(&secgroup); err != nil { + return nil, err + } + return &secgroup, nil +} + +func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mcclient.TokenCredential, idStr string) { + if secgrp := manager.FetchSecgroupById(idStr); secgrp == nil { + log.Errorf("DelaySync secgroup failed") + } else { + needSync := false + lockman.LockObject(ctx, secgrp) + defer lockman.ReleaseObject(ctx, secgrp) + if secgrp.IsDirty { + if _, err := secgrp.GetModelManager().TableSpec().Update(secgrp, func() error { + secgrp.IsDirty = false + return nil + }); err != nil { + log.Errorf("Update Security Group error: %s", err.Error()) + } + needSync = true + } + if needSync { + for _, guest := range secgrp.GetGuests() { + guest.StartSyncTask(ctx, userCred, true, "") + } + } + } +} + +func (self *SSecurityGroup) DoSync(ctx context.Context, userCred mcclient.TokenCredential) { if _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.IsDirty = true return nil }); err != nil { log.Errorf("Update Security Group error: %s", err.Error()) } + time.AfterFunc(10*time.Second, func() { + SecurityGroupManager.DelaySync(ctx, userCred, self.Id) + }) } diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index eabf5a2a82..6e4d1db026 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -5,14 +5,16 @@ import ( "fmt" "time" + "github.com/serialx/hashring" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" ) const ( @@ -71,6 +73,47 @@ func (joint *SStoragecachedimage) Slave() db.IStandaloneModel { return db.JointSlave(joint) } +func (self *SStoragecachedimage) getStorageHostId() (string, error) { + var s SStorage + storage := StorageManager.Query() + err := storage.Filter(sqlchemy.Equals(storage.Field("storagecache_id"), self.StoragecacheId)).First(&s) + if err != nil { + return "", err + } + + hosts := s.GetAllAttachingHosts() + var hostIds = make([]string, 0) + for _, host := range hosts { + hostIds = append(hostIds, host.Id) + } + if len(hostIds) == 0 { + return "", nil + } + + ring := hashring.New(hostIds) + ret, _ := ring.GetNode(self.StoragecacheId) + return ret, nil +} + +func (self *SStoragecachedimage) GetHost() (*SHost, error) { + hostId, err := self.getStorageHostId() + if err != nil { + return nil, err + } else if len(hostId) == 0 { + return nil, nil + } + + host, err := HostManager.FetchById(hostId) + if err != nil { + return nil, err + } else if host == nil { + return nil, nil + } + h, _ := host.(*SHost) + return h, nil + +} + func (self *SStoragecachedimage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := self.SJointResourceBase.GetCustomizeColumns(ctx, userCred, query) extra = db.JointModelExtra(self, extra) @@ -252,6 +295,15 @@ func (self *SStoragecachedimage) SetStatus(userCred mcclient.TokenCredential, st return nil } +func (self *SStoragecachedimage) AddDownloadRefcount() error { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.DownloadRefcnt += 1 + self.LastDownload = time.Now() + return nil + }) + return err +} + func (self *SStoragecachedimage) SetExternalId(externalId string) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.ExternalId = externalId diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 643c5efec8..e0d71d9712 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -4,14 +4,16 @@ import ( "context" "database/sql" + "github.com/serialx/hashring" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/sqlchemy" ) type SStoragecacheManager struct { @@ -55,6 +57,56 @@ func (self *SStoragecache) getStorageNames() []string { return names } +func (self *SStoragecache) GetHost() (*SHost, error) { + hostId, err := self.getHostId() + if err != nil { + return nil, err + } + if len(hostId) == 0 { + return nil, nil + } + + host, err := HostManager.FetchById(hostId) + if err != nil { + return nil, err + } else if host == nil { + return nil, nil + } + h, _ := host.(*SHost) + return h, nil +} + +func (self *SStoragecache) getHostId() (string, error) { + hoststorages := HoststorageManager.Query().SubQuery() + storages := StorageManager.Query().SubQuery() + + hosts := make([]SHost, 0) + host := HostManager.Query().SubQuery() + q := host.Query(host.Field("id")) + err := q.Join(hoststorages, sqlchemy.AND(sqlchemy.Equals(hoststorages.Field("host_id"), host.Field("id")), + sqlchemy.Equals(host.Field("host_status"), HOST_ONLINE), + sqlchemy.IsTrue(host.Field("enabled")))). + Join(storages, sqlchemy.AND(sqlchemy.Equals(storages.Field("storagecache_id"), self.Id), + sqlchemy.Equals(storages.Field("status"), STORAGE_ONLINE), + sqlchemy.IsTrue(storages.Field("enabled")))). + Filter(sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id"))).All(&hosts) + if err != nil { + return "", err + } + + hostIds := make([]string, 0) + for _, h := range hosts { + hostIds = append(hostIds, h.Id) + } + + if len(hostIds) == 0 { + return "", nil + } + ring := hashring.New(hostIds) + ret, _ := ring.GetNode(self.Id) + return ret, nil +} + func (manager *SStoragecacheManager) SyncWithCloudStoragecache(cloudCache cloudprovider.ICloudStoragecache) (*SStoragecache, error) { localCacheObj, err := manager.FetchByExternalId(cloudCache.GetGlobalId()) if err != nil { diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 28650d29f1..4ac8ea7587 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -16,19 +16,22 @@ import ( ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_PUBLIC_CLOUD = "cloud" + STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + STORAGE_CLOUD_SSD = "cloud_ssd" + STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" STORAGE_ENABLED = "enabled" STORAGE_DISABLED = "disabled" STORAGE_OFFLINE = "offline" - STORAGE_ONLINE = "offline" + STORAGE_ONLINE = "online" DISK_TYPE_ROTATE = "rotate" DISK_TYPE_SSD = "ssd" @@ -235,9 +238,12 @@ func (self *SStorage) SyncStatusWithHosts() { } } -func (manager *SStorageManager) getStoragesByZoneId(zoneId string) ([]SStorage, error) { +func (manager *SStorageManager) getStoragesByZoneId(zoneId string, provider *SCloudprovider) ([]SStorage, error) { storages := make([]SStorage, 0) q := manager.Query().Equals("zone_id", zoneId) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &storages) if err != nil { log.Errorf("getStoragesByZoneId fail %s", err) @@ -261,7 +267,7 @@ func (manager *SStorageManager) scanLegacyStorages() error { return nil } -func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, zone *SZone, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { +func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { localStorages := make([]SStorage, 0) remoteStorages := make([]cloudprovider.ICloudStorage, 0) syncResult := compare.SyncResult{} @@ -272,7 +278,7 @@ func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mccli return nil, nil, syncResult } - dbStorages, err := manager.getStoragesByZoneId(zone.Id) + dbStorages, err := manager.getStoragesByZoneId(zone.Id, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index bba2aed865..368a4a963b 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -146,9 +146,12 @@ func (self *SVpc) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCr return self.getMoreDetails(extra) } -func (manager *SVpcManager) getVpcsByRegion(region *SCloudregion) ([]SVpc, error) { +func (manager *SVpcManager) getVpcsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SVpc, error) { vpcs := make([]SVpc, 0) q := manager.Query().Equals("cloudregion_id", region.Id) + if provider != nil { + q = q.Equals("manager_id", provider.Id) + } err := db.FetchModelObjects(manager, q, &vpcs) if err != nil { return nil, err @@ -167,12 +170,12 @@ func (self *SVpc) setDefault(def bool) error { return err } -func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpcs []cloudprovider.ICloudVpc) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { +func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, vpcs []cloudprovider.ICloudVpc) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { localVPCs := make([]SVpc, 0) remoteVPCs := make([]cloudprovider.ICloudVpc, 0) syncResult := compare.SyncResult{} - dbVPCs, err := manager.getVpcsByRegion(region) + dbVPCs, err := manager.getVpcsByRegion(region, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -237,6 +240,7 @@ func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.Toke func (self *SVpc) SyncWithCloudVpc(extVPC cloudprovider.ICloudVpc) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { + extVPC.Refresh() self.Name = extVPC.GetName() self.Status = extVPC.GetStatus() self.CidrBlock = extVPC.GetCidrBlock() diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index 227874f9d6..f8cac5dbae 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -4,9 +4,15 @@ import ( "os" "time" - _ "github.com/go-sql-driver/mysql" - "yunion.io/x/log" + "yunion.io/x/sqlchemy" + + _ "github.com/go-sql-driver/mysql" + _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" + _ "yunion.io/x/onecloud/pkg/compute/hostdrivers" + _ "yunion.io/x/onecloud/pkg/compute/tasks" + _ "yunion.io/x/onecloud/pkg/util/aliyun/provider" + _ "yunion.io/x/onecloud/pkg/util/esxi/provider" "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/cronman" @@ -14,16 +20,15 @@ import ( "yunion.io/x/onecloud/pkg/compute" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - - _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" - _ "yunion.io/x/onecloud/pkg/compute/tasks" - _ "yunion.io/x/onecloud/pkg/util/aliyun/provider" - _ "yunion.io/x/onecloud/pkg/util/esxi/provider" ) func StartService() { cloudcommon.ParseOptions(&options.Options, &options.Options.Options, os.Args, "region.conf") + if options.Options.DebugSqlchemy { + sqlchemy.DEBUG_SQLCHEMY = true + } + if options.Options.PortV2 > 0 { log.Infof("Port V2 %d is specified, use v2 port", options.Options.PortV2) options.Options.Port = options.Options.PortV2 diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index 84af4378a6..997f0792f3 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -130,7 +130,7 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task * return } - localVpcs, remoteVpcs, result := models.VpcManager.SyncVPCs(ctx, task.UserCred, localRegion, vpcs) + localVpcs, remoteVpcs, result := models.VpcManager.SyncVPCs(ctx, task.UserCred, provider, localRegion, vpcs) msg := result.Result() log.Infof("SyncVPCs for region %s result: %s", localRegion.Name, msg) if result.IsError() { @@ -141,13 +141,31 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task * for j := 0; j < len(localVpcs); j += 1 { syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) + syncVpcSecGroup(ctx, provider, task, &localVpcs[j], remoteVpcs[j]) + } +} + +func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc) { + if secgroups, err := remoteVpc.GetISecurityGroups(); err != nil { + msg := fmt.Sprintf("GetISecurityGroups for vpc %s failed %s", remoteVpc.GetId(), err) + log.Errorf(msg) + logSyncFailed(provider, task, msg) + return + } else { + _, _, result := models.SecurityGroupManager.SyncSecgroups(ctx, task.UserCred, secgroups) + msg := result.Result() + log.Infof("SyncSecurityGroup for VPC %s result: %s", localVpc.Name, msg) + if result.IsError() { + logSyncFailed(provider, task, msg) + return + } } } func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localVpc *models.SVpc, remoteVpc cloudprovider.ICloudVpc) { wires, err := remoteVpc.GetIWires() if err != nil { - msg := fmt.Sprintf("GetIWires for vps %s failed %s", remoteVpc.GetId(), err) + msg := fmt.Sprintf("GetIWires for vpc %s failed %s", remoteVpc.GetId(), err) log.Errorf(msg) logSyncFailed(provider, task, msg) return @@ -191,7 +209,7 @@ func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } - localStorages, remoteStorages, result := models.StorageManager.SyncStorages(ctx, task.UserCred, localZone, storages) + localStorages, remoteStorages, result := models.StorageManager.SyncStorages(ctx, task.UserCred, provider, localZone, storages) msg := result.Result() log.Infof("SyncZones for region %s result: %s", localZone.Name, msg) if result.IsError() { @@ -249,7 +267,7 @@ func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *C logSyncFailed(provider, task, msg) return } - localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, localZone, hosts) + localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, localZone, hosts) msg := result.Result() log.Infof("SyncHosts for zone %s result: %s", localZone.Name, msg) if result.IsError() { diff --git a/pkg/compute/tasks/disk_base_task.go b/pkg/compute/tasks/disk_base_task.go index ddd4445b20..083f78d85d 100644 --- a/pkg/compute/tasks/disk_base_task.go +++ b/pkg/compute/tasks/disk_base_task.go @@ -3,6 +3,8 @@ package tasks import ( "context" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -31,3 +33,16 @@ func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) { } } } + +func (self *SDiskBaseTask) CleanHostSchedCache(disk *models.SDisk) { + storage := disk.GetStorage() + if hosts := storage.GetAllAttachingHosts(); hosts == nil { + log.Errorf("get attaching host error") + } else { + for _, h := range hosts { + if err := h.ClearSchedDescCache(); err != nil { + log.Errorf("host CleanHostSchedCache error: %v", err) + } + } + } +} diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go new file mode 100644 index 0000000000..1c47d60d5a --- /dev/null +++ b/pkg/compute/tasks/disk_create_task.go @@ -0,0 +1,76 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type DiskCreateTask struct { + SDiskBaseTask +} + +func init() { + taskman.RegisterTask(DiskCreateTask{}) +} + +func (self *DiskCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + disk := obj.(*models.SDisk) + + storagecache := disk.GetStorage().GetStoragecache() + imageId := disk.GetTemplateId() + if len(imageId) > 0 { + self.SetStage("on_storage_cache_image_complete", nil) + storagecache.StartImageCacheTask(ctx, self.UserCred, imageId, false, self.GetTaskId()) + } else { + self.OnStorageCacheImageComplete(ctx, disk, nil) + } +} + +func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + rebuild, _ := self.GetParams().Bool("rebuild") + snapshot, _ := self.GetParams().GetString("snapshot") + if rebuild { + db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.GetUserCred()) + } + storage := disk.GetStorage() + host := storage.GetMasterHost() + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred()) + disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") + self.SetStage("on_disk_ready", nil) + if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { + self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) + } +} + +func (self *DiskCreateTask) OnStartAllocateFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) +} + +func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + diskSize, _ := data.Int("disk_size") + if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.DiskSize = int(diskSize) + disk.DiskFormat, _ = data.GetString("disk_format") + disk.AccessPath, _ = data.GetString("disk_path") + return nil + }); err != nil { + log.Errorf("update disk info error: %v", err) + } + + disk.SetStatus(self.UserCred, models.DISK_READY, "") + self.CleanHostSchedCache(disk) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.UserCred) + self.SetStageComplete(ctx, nil) +} + +func (self *DiskCreateTask) OnDiskReadyFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) +} diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 2ac629ecf9..ff6c4f76c1 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" ) type DiskDeleteTask struct { @@ -50,7 +51,12 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD if isPurge { self.OnGuestDiskDeleteSucc(ctx, disk, nil) } else { - // TODO + self.SetStage("on_guest_disk_delete_succ", nil) + if host == nil { + self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host")) + } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { + self.OnGuestDiskDeleteFailed(ctx, disk, err) + } } } @@ -65,8 +71,14 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteSucc(ctx context.Context, obj db.IS return } disk := obj.(*models.SDisk) - // self.clean_host_sched_cache(disk) + self.CleanHostSchedCache(disk) db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.UserCred) disk.RealDelete(ctx, self.UserCred) self.SetStageComplete(ctx, nil) } + +func (self *DiskDeleteTask) OnGuestDiskDeleteFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.GetUserCred(), models.DISK_DEALLOC_FAILED, resion.Error()) + self.SetStageFailed(ctx, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_DELOCATE_FAIL, disk.GetShortDesc(), self.GetUserCred()) +} diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go new file mode 100644 index 0000000000..584c43ab94 --- /dev/null +++ b/pkg/compute/tasks/disk_resize_task.go @@ -0,0 +1,108 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type DiskResizeTask struct { + SDiskBaseTask +} + +func init() { + taskman.RegisterTask(DiskResizeTask{}) +} + +func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + disk := obj.(*models.SDisk) + storage := disk.GetStorage() + host := storage.GetMasterHost() + online := disk.GetRuningGuestCount() > 0 + if online { + for _, guest := range disk.GetGuests() { + host = guest.GetHost() + } + } + resion := "Cannot find host for disk" + if host == nil || host.HostStatus != models.HOST_ONLINE { + disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion) + self.SetStageFailed(ctx, resion) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred()) + } else { + disk.SetStatus(self.GetUserCred(), models.DISK_START_RESIZE, "") + for _, guest := range disk.GetGuests() { + guest.SetStatus(self.GetUserCred(), models.VM_RESIZE_DISK, "") + } + self.StartResizeDisk(ctx, host, storage, disk, online) + } +} + +func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, online bool) { + log.Infof("Resizing disk on host %s ...", host.GetName()) + self.SetStage("on_disk_resize_complete", nil) + size, _ := self.GetParams().Int("size") + proc := host.GetHostDriver().RequestResizeDiskOnHost + if online { + proc = host.GetHostDriver().RequestResizeDiskOnHostOnline + } + if err := proc(host, storage, disk, size, self); err != nil { + log.Errorf("request_resize_disk_on_host: %v", err) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + self.OnStartResizeDiskSucc(ctx, disk) +} + +func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *models.SDisk) { + disk.SetStatus(self.GetUserCred(), models.DISK_RESIZING, "") +} + +func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error()) + self.SetStageFailed(ctx, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred()) +} + +func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + jSize, err := data.Get("disk_size") + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + size, err := jSize.Int() + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + oldStatus := disk.Status + _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.Status = models.DISK_READY + disk.DiskSize = int(size) + return nil + }) + if err != nil { + log.Errorf("OnDiskResizeComplete error: %s", err.Error()) + self.OnStartResizeDiskFailed(ctx, disk, err) + return + } + notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status) + db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred) + self.CleanHostSchedCache(disk) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE, disk.GetShortDesc(), self.UserCred) + self.SetStageComplete(ctx, disk.GetShortDesc()) + self.finalReleasePendingUsage(ctx) +} + +func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) { + disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error()) + db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) +} diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go new file mode 100644 index 0000000000..44ba532ae1 --- /dev/null +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -0,0 +1,208 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestChangeConfigTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestChangeConfigTask{}) +} + +func (self *GuestChangeConfigTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + _, err := self.Params.Get("resize") + if err == nil { + self.SetStage("on_disks_resize_complete", nil) + self.OnDisksResizeComplete(ctx, obj, data) + } else { + guest := obj.(*models.SGuest) + self.DoCreateDisksTask(ctx, guest) + } +} + +func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + iResizeDisks, err := self.Params.Get("resize") + if iResizeDisks == nil || err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + resizeDisks := iResizeDisks.(*jsonutils.JSONArray) + for i := 0; i < resizeDisks.Length(); i++ { + iResizeSet, err := resizeDisks.GetAt(i) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + resizeSet := iResizeSet.(*jsonutils.JSONArray) + diskId, err := resizeSet.GetAt(0) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + idStr, err := diskId.GetString() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + jSize, err := resizeSet.GetAt(1) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + size, err := jSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + iDisk, err := models.DiskManager.FetchById(idStr) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + disk := iDisk.(*models.SDisk) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if disk.DiskSize < int(size) { + var pendingUsage models.SQuota + err = self.GetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage) + return + } + } + guest := obj.(*models.SGuest) + self.DoCreateDisksTask(ctx, guest) +} + +func (self *GuestChangeConfigTask) DoCreateDisksTask(ctx context.Context, guest *models.SGuest) { + iCreateData, err := self.Params.Get("create") + if err != nil || iCreateData == nil { + self.OnCreateDisksComplete(ctx, guest, nil) + return + } + data := (iCreateData).(*jsonutils.JSONDict) + self.SetStage("on_create_disks_complete", nil) + guest.StartGuestCreateDiskTask(ctx, self.UserCred, data, self.GetTaskId()) + +} + +func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + iVcpuCount, errCpu := self.Params.Get("vcpu_count") + iVmemSize, errMem := self.Params.Get("vmem_size") + var vcpuCount, vmemSize int64 + var err error + guest := obj.(*models.SGuest) + if errCpu == nil || errMem == nil { + if iVcpuCount != nil { + vcpuCount, err = iVcpuCount.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + if iVmemSize != nil { + vmemSize, err = iVmemSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + err = guest.GetDriver().RequestChangeVmConfig(ctx, guest, self, vcpuCount, vmemSize) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + var addCpu, addMem = 0, 0 + if vcpuCount > 0 { + addCpu = int(vcpuCount - int64(guest.VcpuCount)) + if addCpu < 0 { + addCpu = 0 + } + } + if vmemSize > 0 { + addMem = int(vmemSize - int64(guest.VmemSize)) + if addMem < 0 { + addMem = 0 + } + } + _, err = guest.GetModelManager().TableSpec().Update(guest, func() error { + if vcpuCount > 0 { + guest.VcpuCount = int8(vcpuCount) + } + if vmemSize > 0 { + guest.VmemSize = int(vmemSize) + } + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + var pendingUsage models.SQuota + err = self.GetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + // ownerCred := guest.GetOwnerUserCred() + var cancelUsage models.SQuota + if addCpu > 0 { + cancelUsage.Cpu = addCpu + } + if addMem > 0 { + cancelUsage.Memory = addMem + } + lockman.LockClass(ctx, guest.GetModelManager(), guest.ProjectId) + defer lockman.ReleaseClass(ctx, guest.GetModelManager(), guest.ProjectId) + err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, guest.ProjectId, &pendingUsage, &cancelUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + err = self.SetPendingUsage(&pendingUsage) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + self.SetStage("on_sync_status_complete", nil) + err = guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } +} + +func (self *GuestChangeConfigTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) { + self.SetStage("on_guest_start_complete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + dt := jsonutils.NewDict() + dt.Add(jsonutils.NewString(guest.Id), "id") + self.SetStageComplete(ctx, dt) + } +} + +func (self *GuestChangeConfigTask) OnGuestStartComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + dt := jsonutils.NewDict() + dt.Add(jsonutils.NewString(guest.Id), "id") + self.SetStageComplete(ctx, dt) +} diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go new file mode 100644 index 0000000000..c7034bd735 --- /dev/null +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -0,0 +1,124 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *GuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_disk_prepared", nil) + guest := obj.(*models.SGuest) + err := guest.GetDriver().DoGuestCreateDisksTask(ctx, guest, self) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } +} + +func (self *GuestCreateDiskTask) OnDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *GuestCreateDiskTask) OnDiskPreparedFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data.String()) +} + +/* --------------------------------------------- */ +/* -----------KVMGuestCreateDiskTask------------ */ +/* --------------------------------------------- */ + +type KVMGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *KVMGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_kvm_disk_prepared", nil) + self.OnKvmDiskPrepared(ctx, obj, data) +} + +func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + var diskIndex = 0 + var diskReady = true + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status == models.DISK_INIT { + snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + if err != nil { + snapInfo = "" + } + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + diskReady = false + break + } + diskIndex += 1 + } + diskIndex = 0 + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status != models.DISK_READY { + diskReady = false + break + } + diskIndex += 1 + } + if diskReady { + guest := obj.(*models.SGuest) + if guest.Status == models.VM_RUNNING { + self.SetStage("on_config_sync_complete", nil) + err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + } else { + self.SetStageComplete(ctx, nil) + } + } +} + +func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func init() { + taskman.RegisterTask(GuestCreateDiskTask{}) + taskman.RegisterTask(KVMGuestCreateDiskTask{}) +} diff --git a/pkg/compute/tasks/guest_delete_task.go b/pkg/compute/tasks/guest_delete_task.go index 09666d5e52..b86a63d2ec 100644 --- a/pkg/compute/tasks/guest_delete_task.go +++ b/pkg/compute/tasks/guest_delete_task.go @@ -28,15 +28,14 @@ func (self *GuestDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) + guestStatus, _ := self.Params.GetString("guest_status") if options.Options.EnablePendingDelete && !guest.PendingDeleted && !jsonutils.QueryBoolean(self.Params, "purge", false) && - !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { - guestStatus, _ := self.Params.GetString("guest_status") - if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, + !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) && + !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) { - self.StartPendingDeleteGuest(ctx, guest) - return - } + self.StartPendingDeleteGuest(ctx, guest) + return } self.OnGuestStopCompleteFailed(ctx, guest, data) } @@ -64,6 +63,7 @@ func (self *GuestDeleteTask) OnPendingDeleteComplete(ctx context.Context, obj db } func (self *GuestDeleteTask) StartDeleteGuest(ctx context.Context, guest *models.SGuest) { + // No snapshot self.SetStage("on_guest_detach_disks_complete", nil) guest.GetDriver().RequestDetachDisksFromGuestForDelete(ctx, guest, self) } @@ -100,7 +100,6 @@ func (self *GuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.I } func (self *GuestDeleteTask) DeleteGuest(ctx context.Context, guest *models.SGuest) { - // host := guest.GetHost() guest.RealDelete(ctx, self.UserCred) guest.RemoveAllMetadata(ctx, self.UserCred) db.OpsLog.LogEvent(guest, db.ACT_DELOCATE, nil, self.UserCred) diff --git a/pkg/compute/tasks/guest_detach_all_disks_task.go b/pkg/compute/tasks/guest_detach_all_disks_task.go index ed2bc2529a..6be7da967f 100644 --- a/pkg/compute/tasks/guest_detach_all_disks_task.go +++ b/pkg/compute/tasks/guest_detach_all_disks_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/tasks/guest_detach_disk_task.go b/pkg/compute/tasks/guest_detach_disk_task.go index b8af7ce35c..0a6653750f 100644 --- a/pkg/compute/tasks/guest_detach_disk_task.go +++ b/pkg/compute/tasks/guest_detach_disk_task.go @@ -2,10 +2,15 @@ package tasks import ( "context" + "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" ) type GuestDetachDiskTask struct { @@ -17,5 +22,83 @@ func init() { } func (self *GuestDetachDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + diskId, _ := self.Params.GetString("disk_id") + objDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + disk := objDisk.(*models.SDisk) + if disk == nil { + self.OnTaskFail(ctx, guest, fmt.Errorf("Connot find disk %s", diskId)) + return + } + guest.DetachDisk(ctx, disk, self.UserCred) + if disk.Status == models.DISK_INIT { + self.OnSyncConfigComplete(ctx, guest, nil) + return + } + host := guest.GetHost() + purge := false + if host != nil && host.Status == models.HOST_DISABLED && jsonutils.QueryBoolean(self.Params, "purge", false) { + purge = true + } + detachStatus, err := guest.GetDriver().GetDetachDiskStatus() + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + if utils.IsInStringArray(guest.Status, detachStatus) && !purge { + self.SetStage("on_sync_config_complete", nil) + guest.GetDriver().RequestDetachDisk(ctx, guest, self) + disk.SetStatus(self.UserCred, models.DISK_READY, "Disk detach") + } else { + self.OnSyncConfigComplete(ctx, guest, nil) + } +} + +func (self *GuestDetachDiskTask) OnSyncConfigComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + diskId, _ := self.Params.GetString("disk_id") + objDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + disk := objDisk.(*models.SDisk) + if disk == nil { + self.OnTaskFail(ctx, guest, fmt.Errorf("Connot find disk %s", diskId)) + return + } + keepDisk := jsonutils.QueryBoolean(self.Params, "keep_disk", true) + host := guest.GetHost() + purge := false + if host != nil && host.Status == models.HOST_DISABLED && jsonutils.QueryBoolean(self.Params, "purge", false) { + purge = true + } + if disk.Status == models.DISK_INIT { + db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) + disk.RealDelete(ctx, self.UserCred) + self.SetStageComplete(ctx, nil) + } else if (disk.Status == models.DISK_READY || !keepDisk) && disk.GetGuestDiskCount() == 0 && disk.AutoDelete { + self.SetStage("on_disk_delete_complete", nil) + db.OpsLog.LogEvent(disk, db.ACT_DELETE, "", self.UserCred) + err := guest.GetDriver().RequestDeleteDetachedDisk(ctx, disk, self, purge) + if err != nil { + self.OnTaskFail(ctx, guest, err) + return + } + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestDetachDiskTask) OnTaskFail(ctx context.Context, guest *models.SGuest, err error) { + self.SetStageFailed(ctx, err.Error()) + log.Errorf("Guest %s GuestDetachDiskTask failed %s", guest.Id, err.Error()) +} + +func (self *GuestDetachDiskTask) OnDiskDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_insert_iso_task.go b/pkg/compute/tasks/guest_insert_iso_task.go index 21bb54b31d..e0b9816f52 100644 --- a/pkg/compute/tasks/guest_insert_iso_task.go +++ b/pkg/compute/tasks/guest_insert_iso_task.go @@ -4,8 +4,10 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" ) type GuestInsertIsoTask struct { @@ -17,5 +19,59 @@ func init() { } func (self *GuestInsertIsoTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - // TODO + self.prepareIsoImage(ctx, obj) +} + +func (self *GuestInsertIsoTask) prepareIsoImage(ctx context.Context, obj db.IStandaloneModel) { + imageId, _ := self.Params.GetString("image_id") + db.OpsLog.LogEvent(obj, db.ACT_ISO_PREPARING, imageId, self.UserCred) + var host *models.SHost + if self.Params.Contains("host_id") { + hostId, _ := self.Params.GetString("host_id") + iHost, _ := models.HostManager.FetchById(hostId) + host = iHost.(*models.SHost) + } else { + guest := obj.(*models.SGuest) + host = guest.GetHost() + } + self.SetStage("OnIsoPrepareComplete", nil) + host.StartImageCacheTask(ctx, self.UserCred, imageId, self.GetTaskId(), false) +} + +func (self *GuestInsertIsoTask) OnIsoPrepareCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + imageId, _ := self.Params.GetString("image_id") + db.OpsLog.LogEvent(obj, db.ACT_ISO_PREPARE_FAIL, imageId, self.UserCred) + guest := obj.(*models.SGuest) + guest.EjectIso(self.UserCred) + self.SetStageFailed(ctx, "OnIsoPrepareCompleteFailed") +} + +func (self *GuestInsertIsoTask) OnIsoPrepareComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + imageId, _ := data.GetString("image_id") + jSize, err := data.Get("size") + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + size, err := jSize.Int() + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + name, _ := data.GetString("name") + path, _ := data.GetString("path") + guest := obj.(*models.SGuest) + if guest.InsertIsoSucc(imageId, path, int(size), name) { + db.OpsLog.LogEvent(guest, db.ACT_ISO_ATTACH, guest.GetDetailsIso(self.UserCred), self.UserCred) + if guest.Status == models.VM_RUNNING { + self.SetStage("OnConfigSyncComplete", nil) + guest.GetDriver().RequestGuestHotAddIso(ctx, guest, path, self) + } else { + self.SetStageComplete(ctx, nil) + } + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestInsertIsoTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go new file mode 100644 index 0000000000..03345b85ac --- /dev/null +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -0,0 +1,144 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/osprofile" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestRebuildRootTask{}) + taskman.RegisterTask(KVMGuestRebuildRootTask{}) +} + +type GuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if jsonutils.QueryBoolean(self.Params, "need_stop", false) { + self.SetStage("OnStopServerComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) + } else { + self.StartRebuildRootDisk(ctx, guest) + } +} + +func (self *GuestRebuildRootTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartRebuildRootDisk(ctx, guest) +} + +func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred) + gds := guest.CategorizeDisks() + imageId, _ := self.Params.GetString("image_id") + oldStatus := gds.Root.Status + _, err := gds.Root.GetModelManager().TableSpec().Update(gds.Root, func() error { + gds.Root.TemplateId = imageId + gds.Root.Status = models.DISK_REBUILD + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } else { + db.OpsLog.LogEvent(gds.Root, db.ACT_UPDATE_STATUS, + fmt.Sprintf("%s=>%s", oldStatus, models.DISK_REBUILD), self.UserCred) + } + + self.SetStage("OnRebuildRootDiskComplete", nil) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT, "") + guest.GetDriver().RequestRebuildRootDisk(ctx, guest, self) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + imgId, _ := self.Params.GetString("image_id") + imginfo, err := models.CachedimageManager.GetImageById(ctx, self.UserCred, imgId, false) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + osprof, err := osprofile.GetOSProfileFromImageProperties(imginfo.Properties, guest.Hypervisor) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + err = guest.SetMetadata(ctx, "__os_profile__", osprof, self.UserCred) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if guest.OsType != osprof.OSType { + _, err := guest.GetModelManager().TableSpec().Update(guest, func() error { + guest.OsType = osprof.OSType + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT, "", self.UserCred) + // TODO: logclient + guest.NotifyServerEvent(notifyclient.SERVER_REBUILD_ROOT, notifyclient.PRIORITY_IMPORTANT, true) + self.SetStage("OnSyncStatusComplete", nil) + guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT_FAIL, data.String(), self.UserCred) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT_FAIL, "") + // TODO: logclient +} + +func (self *GuestRebuildRootTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) { + self.SetStage("OnGuestStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestRebuildRootTask) OnGuestStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +/* -------------------------------------------------- */ +/* ------------ KVMGuestRebuildRootTask ------------- */ +/* -------------------------------------------------- */ + +type KVMGuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *KVMGuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + gds := guest.CategorizeDisks() + self.SetStage("OnRebuildRootDiskComplete", nil) + gds.Root.StartDiskCreateTask(ctx, self.UserCred, true, "", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStage("OnGuestDeployComplete", nil) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") + params := jsonutils.NewDict() + params.Set("reset_password", jsonutils.JSONTrue) + guest.StartGuestDeployTask(ctx, self.UserCred, params, "deploy", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data.String()) +} + +func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_reset_task.go b/pkg/compute/tasks/guest_reset_task.go new file mode 100644 index 0000000000..72a5273e2d --- /dev/null +++ b/pkg/compute/tasks/guest_reset_task.go @@ -0,0 +1,58 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestSoftResetTask{}) + taskman.RegisterTask(GuestHardResetTask{}) +} + +type GuestSoftResetTask struct { + SGuestBaseTask +} + +func (self *GuestSoftResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + err := guest.GetDriver().RequestSoftReset(ctx, guest, self) + if err == nil { + self.SetStageComplete(ctx, nil) + } else { + self.SetStageFailed(ctx, err.Error()) + } +} + +type GuestHardResetTask struct { + SGuestBaseTask +} + +func (self *GuestHardResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.StopServer(ctx, guest) +} + +func (self *GuestHardResetTask) StopServer(ctx context.Context, guest *models.SGuest) { + guest.SetStatus(self.UserCred, models.VM_STOPPING, "") + self.SetStage("OnServerStopComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.StartServer(ctx, guest) +} + +func (self *GuestHardResetTask) StartServer(ctx context.Context, guest *models.SGuest) { + self.SetStage("OnServerStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) +} + +func (self *GuestHardResetTask) OnServerStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 3d910543a6..7bbb2ca2dc 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -42,7 +43,7 @@ func (self *GuestStartTask) startStart(ctx context.Context, guest *models.SGuest self.SetStage("on_start_complete", nil) host := guest.GetHost() guest.SetStatus(self.UserCred, models.VM_STARTING, "") - result, err := guest.GetDriver().RequestStartOnHost(guest, host, self.UserCred, self) + result, err := guest.GetDriver().RequestStartOnHost(ctx, guest, host, self.UserCred, self) if err != nil { self.onStartGuestFailed(ctx, guest, err) } else { diff --git a/pkg/compute/tasks/guest_stop_task.go b/pkg/compute/tasks/guest_stop_task.go index ddffa32032..2ed0f4aa6c 100644 --- a/pkg/compute/tasks/guest_stop_task.go +++ b/pkg/compute/tasks/guest_stop_task.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/tasks/guest_suspend_task.go b/pkg/compute/tasks/guest_suspend_task.go new file mode 100644 index 0000000000..13023f12f3 --- /dev/null +++ b/pkg/compute/tasks/guest_suspend_task.go @@ -0,0 +1,47 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestSuspendTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestSuspendTask{}) +} + +func (self *GuestSuspendTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_STOPPING, "", self.UserCred) + guest.SetStatus(self.UserCred, models.VM_SUSPENDING, "GuestSusPendTask") + self.SetStage("on_suspend_complete", nil) + err := guest.GetDriver().RqeuestSuspendOnHost(ctx, guest, self) + if err != nil { + self.OnSuspendGuestFail(guest, err.Error()) + } +} + +func (self *GuestSuspendTask) OnSuspendComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.UserCred, models.VM_SUSPEND, "") + db.OpsLog.LogEvent(guest, db.ACT_STOP, "", self.UserCred) + self.SetStageComplete(ctx, nil) +} + +func (self *GuestSuspendTask) OnSuspendCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.UserCred, models.VM_RUNNING, "") + db.OpsLog.LogEvent(guest, db.ACT_STOP_FAIL, err.String(), self.UserCred) +} + +func (self *GuestSuspendTask) OnSuspendGuestFail(guest *models.SGuest, reason string) { + guest.SetStatus(self.UserCred, models.VM_SUSPEND_FAILED, reason) +} diff --git a/pkg/compute/tasks/guest_sync_task.go b/pkg/compute/tasks/guest_sync_task.go new file mode 100644 index 0000000000..1daeb9e058 --- /dev/null +++ b/pkg/compute/tasks/guest_sync_task.go @@ -0,0 +1,69 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type GuestSyncConfTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestSyncConfTask{}) +} + +func (self *GuestSyncConfTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred) + if host := guest.GetHost(); host == nil { + self.SetStageFailed(ctx, "No host for sync") + return + } else { + self.SetStage("on_sync_complete", nil) + if err := guest.GetDriver().RequestSyncConfigOnHost(ctx, guest, host, self); err != nil { + self.SetStageFailed(ctx, err.Error()) + log.Errorf("SyncConfTask faled %v", err) + } + } +} + +func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if fw_only, _ := self.GetParams().Bool("fw_only"); fw_only { + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred) + self.SetStageComplete(ctx, guest.GetShortDesc()) + } else if data.Contains("task") { + self.SetStage("on_disk_sync_complete", nil) + } else { + self.OnDiskSyncComplete(ctx, guest, data) + } +} + +func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStage("on_sync_status_complete", nil) + guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId()) +} + +func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) + log.Errorf("Guest sync config failed: %v", data.String()) +} + +func (self *GuestSyncConfTask) OnSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.GetUserCred(), models.VM_SYNC_FAIL, data.String()) + log.Errorf("Guest sync config failed: %v", data.String()) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) +} + +func (self *GuestSyncConfTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_syncstatus_task.go b/pkg/compute/tasks/guest_syncstatus_task.go index 6c4b5f5af4..478f4afe6d 100644 --- a/pkg/compute/tasks/guest_syncstatus_task.go +++ b/pkg/compute/tasks/guest_syncstatus_task.go @@ -49,7 +49,9 @@ func (self *GuestSyncstatusTask) OnGetStatusSucc(ctx context.Context, guest *mod default: statusStr = models.VM_UNKNOWN } - guest.SetStatus(self.UserCred, statusStr, "syncstatus") + statusData := jsonutils.NewDict() + statusData.Add(jsonutils.NewString(statusStr), "status") + guest.PerformStatus(ctx, self.UserCred, nil, statusData) self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_undeploy_task.go b/pkg/compute/tasks/guest_undeploy_task.go index 1cb128ec53..eb3f789e8a 100644 --- a/pkg/compute/tasks/guest_undeploy_task.go +++ b/pkg/compute/tasks/guest_undeploy_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -33,8 +34,6 @@ func (self *GuestUndeployTask) OnInit(ctx context.Context, obj db.IStandaloneMod err := guest.GetDriver().RequestUndeployGuestOnHost(ctx, guest, host, self) if err != nil { self.OnStartDeleteGuestFail(ctx, err) - } else { - // do nothing } } else { self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/storage_cache_image_task.go b/pkg/compute/tasks/storage_cache_image_task.go index ffd44c6c62..fc24e13ea0 100644 --- a/pkg/compute/tasks/storage_cache_image_task.go +++ b/pkg/compute/tasks/storage_cache_image_task.go @@ -5,6 +5,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -20,7 +21,7 @@ func init() { func (self *StorageCacheImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { imageId, _ := self.Params.GetString("image_id") - isForce := jsonutils.QueryBoolean(self.Params, "is_force", false) + // isForce := jsonutils.QueryBoolean(self.Params, "is_force", false) storageCache := obj.(*models.SStoragecache) scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) @@ -32,22 +33,13 @@ func (self *StorageCacheImageTask) OnInit(ctx context.Context, obj db.IStandalon self.SetStage("on_image_cache_complete", nil) - taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) { - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } + host, _ := storageCache.GetHost() + err := host.GetHostDriver().CheckAndSetCacheImage(ctx, host, storageCache, scimg, self) + if err != nil { + errData := taskman.Error2TaskData(err) + self.OnImageCacheCompleteFailed(ctx, storageCache, errData) + } - extImgId, err := iStorageCache.UploadImage(self.UserCred, imageId, scimg.ExternalId, isForce) - - if err != nil { - return nil, err - } else { - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - } - }) } func (self *StorageCacheImageTask) OnImageCacheComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -80,7 +72,7 @@ func (self *StorageCacheImageTask) OnCacheFailed(ctx context.Context, cache *mod func (self *StorageCacheImageTask) OnCacheSucc(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage, extImgId string) { scimg.SetStatus(self.UserCred, models.CACHED_IMAGE_STATUS_READY, "cached") - if len(extImgId) > 0 && scimg.ExternalId != extImgId { + if len(cache.ExternalId) > 0 && len(extImgId) > 0 && scimg.ExternalId != extImgId { scimg.SetExternalId(extImgId) } models.CachedimageManager.ImageAddRefCount(imageId) diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index 77d0e62d67..d57bf8ff79 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -39,6 +39,13 @@ func NewResourceNotReadyError(msg string, params ...interface{}) *httputils.JSON return NewJsonClientError(500, "ResourceNotReadyError", msg) } +func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError { + if len(params) > 0 { + msg = fmt.Sprintf(msg, params...) + } + return NewJsonClientError(500, "NewOutOfResourceError", msg) +} + func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError { if len(params) > 0 { msg = fmt.Sprintf(msg, params...) diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 92bf5b4bb9..4df0561160 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -53,6 +53,10 @@ func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client { return &client } +func (this *Client) SetDebug(debug bool) { + this.debug = debug +} + func (this *Client) AuthVersion() string { pos := strings.LastIndexByte(this.authUrl, '/') if pos > 0 { @@ -62,6 +66,13 @@ func (this *Client) AuthVersion() string { } } +func (this *Client) NewAuthTokenCredential() TokenCredential { + if this.AuthVersion() == "v3" { + return &TokenCredentialV3{} + } + return &TokenCredentialV2{} +} + func getDefaultHeader(header http.Header, token string) http.Header { if len(token) > 0 { if header == nil { diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index d458d65038..3526f6eea4 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -63,6 +63,10 @@ func SplitVersionedURL(url string) (string, string) { return base }*/ +func (this *ClientSession) GetClient() *Client { + return this.client +} + func (this *ClientSession) GetServiceURL(service, endpointType string) (string, error) { if len(this.endpointType) > 0 { // session specific endpoint type should override the input endpointType, which is supplied by manager diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 4d3a039793..aeeca8096c 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -96,6 +96,14 @@ func (self *SDisk) GetId() string { return self.DiskId } +func (self *SDisk) Delete() error { + return self.storage.zone.region.deleteDisk(self.DiskId) +} + +func (self *SDisk) Resize(size int64) error { + return self.storage.zone.region.resizeDisk(self.DiskId, size) +} + func (self *SDisk) GetName() string { return self.DiskId } @@ -214,3 +222,12 @@ func (self *SRegion) deleteDisk(diskId string) error { _, err := self.ecsRequest("DeleteDisk", params) return err } + +func (self *SRegion) resizeDisk(diskId string, size int64) error { + params := make(map[string]string) + params["DiskId"] = diskId + params["NewSize"] = fmt.Sprintf("%d", size) + + _, err := self.ecsRequest("ResizeDisk", params) + return err +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index c5b61dde3b..3ee4a40236 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -222,7 +222,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int return "", fmt.Errorf("vsiwtch's wire's vpc is empty") } - secgroups, err := net.wire.vpc.GetSecurityGroups() + secgroups, err := net.wire.vpc.GetISecurityGroups() if err != nil { return "", fmt.Errorf("get security group error %s", err) } @@ -236,7 +236,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int secgroupId = secId } } else { - secgroupId = secgroups[0].SecurityGroupId + secgroupId = secgroups[0].GetId() } keypair := "" diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 292a231193..80a2e9c267 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/seclib" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -185,6 +186,10 @@ func (self *SInstance) IsEmulated() bool { return false } +func (self *SInstance) getVpc() (*SVpc, error) { + return self.host.zone.region.getVpc(self.VpcAttributes.VpcId) +} + func (self *SInstance) fetchDisks() error { disks, total, err := self.host.zone.region.GetDisks(self.InstanceId, "", "", nil, 0, 50) if err != nil { @@ -508,3 +513,29 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") return ret, nil } + +func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { + if vpc, err := self.getVpc(); err != nil { + return err + } else if len(secgroupId) == 0 { + for index, secgrpId := range self.SecurityGroupIds.SecurityGroupId { + if err := vpc.revokeSecurityGroup(secgrpId, self.InstanceId, index == 0); err != nil { + return err + } + } + } else if secgrpId, err := vpc.syncSecurityGroup(secgroupId, name, rules); err != nil { + return err + } else if err := vpc.assignSecurityGroup(secgrpId, self.InstanceId); err != nil { + return err + } else { + for _, secgroupId := range self.SecurityGroupIds.SecurityGroupId { + if secgroupId != secgrpId { + if err := vpc.revokeSecurityGroup(secgroupId, self.InstanceId, false); err != nil { + return err + } + } + } + self.SecurityGroupIds.SecurityGroupId = []string{secgrpId} + } + return nil +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index cdea189749..78efb6a653 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -292,6 +292,7 @@ func (self *SRegion) getVpc(vpcId string) (*SVpc, error) { if total != 1 { return nil, cloudprovider.ErrNotFound } + vpcs[0].region = self return &vpcs[0], nil } diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 85d9518b32..69515ae79d 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -2,9 +2,13 @@ package aliyun import ( "fmt" + "sort" + "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" ) @@ -43,6 +47,7 @@ type SPermissions struct { } type SSecurityGroup struct { + vpc *SVpc CreationTime time.Time Description string SecurityGroupId string @@ -53,6 +58,82 @@ type SSecurityGroup struct { RegionId string } +type PermissionSet []SPermission + +func (v PermissionSet) Len() int { + return len(v) +} + +func (v PermissionSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v PermissionSet) Less(i, j int) bool { + if v[i].Priority < v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + +func (self *SSecurityGroup) GetId() string { + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetGlobalId() string { + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetDescription() string { + return self.Description +} + +func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { + rules := make([]secrules.SecurityRule, 0) + if secgrp, err := self.vpc.region.GetSecurityGroupDetails(self.SecurityGroupId); err != nil { + return rules, err + } else { + for _, permission := range secgrp.Permissions.Permission { + if rule, err := secrules.ParseSecurityRule(permission.String()); err != nil { + return rules, err + } else { + priority := permission.Priority + if priority > 100 { + priority = 100 + } + rule.Priority = 101 - priority + rule.Description = permission.Description + rules = append(rules, *rule) + } + } + } + return rules, nil +} + +func (self *SSecurityGroup) GetName() string { + if len(self.SecurityGroupName) > 0 { + return self.SecurityGroupName + } + return self.SecurityGroupId +} + +func (self *SSecurityGroup) GetStatus() string { + return "" +} + +func (self *SSecurityGroup) IsEmulated() bool { + return false +} + +func (self *SSecurityGroup) Refresh() error { + if new, err := self.vpc.region.GetSecurityGroupDetails(self.SecurityGroupId); err != nil { + return err + } else { + return jsonutils.Update(self, new) + } +} + func (self *SRegion) GetSecurityGroups(vpcId string, offset int, limit int) ([]SSecurityGroup, int, error) { if limit > 50 || limit <= 0 { limit = 50 @@ -122,15 +203,25 @@ func (self *SRegion) createSecurityGroup(vpcId string, name string, desc string) return body.GetString("SecurityGroupId") } -func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { +func (self *SRegion) modifySecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { params := make(map[string]string) + params["RegionId"] = self.RegionId params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["Description"] = rule.Description + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) protocol := rule.Protocol - if len(rule.Protocol) == 0 { + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { protocol = "all" } params["IpProtocol"] = protocol - params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } if rule.Action == secrules.SecurityRuleAllow { params["Policy"] = "accept" } else { @@ -143,7 +234,78 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi } else { params["SourceCidrIp"] = "0.0.0.0/0" } - params["DestCidrIp"] = "0.0.0.0/0" + _, err := self.ecsRequest("ModifySecurityGroupRule", params) + return err + } else { // rule.Direction == secrules.SecurityRuleEgress { + //阿里云不支持出方向API接口调用 + return nil + // if rule.IPNet != nil { + // params["DestCidrIp"] = rule.IPNet.String() + // } else { + // params["DestCidrIp"] = "0.0.0.0/0" + // } + // _, err := self.ecsRequest("ModifySecurityGroupRule", params) + // return err + } +} + +func (self *SRegion) modifySecurityGroup(secGrpId string, name string, desc string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["SecurityGroupName"] = name + if len(desc) > 0 { + params["Description"] = desc + } + _, err := self.ecsRequest("ModifySecurityGroupAttribute", params) + return err +} + +func (self *SRegion) addSecurityGroupRules(secGrpId string, rule *secrules.SecurityRule) error { + if len(rule.Ports) != 0 { + for _, port := range rule.Ports { + rule.PortStart, rule.PortEnd = port, port + if err := self.addSecurityGroupRule(secGrpId, rule); err != nil { + return err + } + } + } else { + return self.addSecurityGroupRule(secGrpId, rule) + } + return nil +} + +func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["Description"] = rule.Description + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + protocol := rule.Protocol + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = "all" + } + params["IpProtocol"] = protocol + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } + if rule.Action == secrules.SecurityRuleAllow { + params["Policy"] = "accept" + } else { + params["Policy"] = "drop" + } + params["Priority"] = fmt.Sprintf("%d", 101-rule.Priority) + if rule.Direction == secrules.SecurityRuleIngress { + if rule.IPNet != nil { + params["SourceCidrIp"] = rule.IPNet.String() + } else { + params["SourceCidrIp"] = "0.0.0.0/0" + } _, err := self.ecsRequest("AuthorizeSecurityGroup", params) return err } else { // rule.Direction == secrules.SecurityRuleEgress { @@ -152,12 +314,54 @@ func (self *SRegion) addSecurityGroupRule(secGrpId string, rule *secrules.Securi } else { params["DestCidrIp"] = "0.0.0.0/0" } - params["SourceCidrIp"] = "0.0.0.0/0" _, err := self.ecsRequest("AuthorizeSecurityGroupEgress", params) return err } } +func (self *SRegion) delSecurityGroupRule(secGrpId string, rule *secrules.SecurityRule) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["SecurityGroupId"] = secGrpId + params["NicType"] = string(IntranetNicType) + params["PortRange"] = fmt.Sprintf("%d/%d", rule.PortStart, rule.PortEnd) + protocol := rule.Protocol + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = "all" + } + params["IpProtocol"] = protocol + if rule.PortStart == 0 && rule.PortEnd == 0 { + if protocol == "udp" || protocol == "tcp" { + params["PortRange"] = "1/65535" + } else { + params["PortRange"] = "-1/-1" + } + } + if rule.Action == secrules.SecurityRuleAllow { + params["Policy"] = "accept" + } else { + params["Policy"] = "drop" + } + params["Priority"] = fmt.Sprintf("%d", rule.Priority) + if rule.Direction == secrules.SecurityRuleIngress { + if rule.IPNet != nil { + params["SourceCidrIp"] = rule.IPNet.String() + } else { + params["SourceCidrIp"] = "0.0.0.0/0" + } + _, err := self.ecsRequest("RevokeSecurityGroup", params) + return err + } else { // rule.Direction == secrules.SecurityRuleEgress { + if rule.IPNet != nil { + params["DestCidrIp"] = rule.IPNet.String() + } else { + params["DestCidrIp"] = "0.0.0.0/0" + } + _, err := self.ecsRequest("RevokeSecurityGroupEgress", params) + return err + } +} + func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { secId, err := self.createSecurityGroup(vpcId, "", "") if err != nil { @@ -171,7 +375,7 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { PortStart: -1, PortEnd: -1, } - err = self.addSecurityGroupRule(secId, &inRule) + err = self.addSecurityGroupRules(secId, &inRule) if err != nil { return "", err } @@ -183,9 +387,190 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { PortStart: -1, PortEnd: -1, } - err = self.addSecurityGroupRule(secId, &outRule) + err = self.addSecurityGroupRules(secId, &outRule) if err != nil { return "", err } return secId, nil } + +func (self *SRegion) getSecurityGroupByTag(vpcId, secgroupId string) (*SSecurityGroup, error) { + params := make(map[string]string) + params["RegionId"] = self.RegionId + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + params["Tag.1.Key"] = "id" + params["Tag.1.Value"] = secgroupId + + secgrps := make([]SSecurityGroup, 0) + if body, err := self.ecsRequest("DescribeSecurityGroups", params); err != nil { + return nil, err + } else if err := body.Unmarshal(&secgrps, "SecurityGroups", "SecurityGroup"); err != nil { + return nil, err + } else if len(secgrps) != 1 { + return nil, httperrors.NewNotFoundError("failed to find SecurityGroup %s", secgroupId) + } + return &secgrps[0], nil +} + +func (self *SPermission) String() string { + action := secrules.SecurityRuleDeny + if strings.ToLower(self.Policy) == "accept" { + action = secrules.SecurityRuleAllow + } + direction := "in" + if self.Direction == "egress" { + direction = "out" + } + cidr := self.SourceCidrIp + if direction == "out" { + cidr = self.DestCidrIp + } + if cidr == "0.0.0.0/0" { + cidr = "" + } + protocol := strings.ToLower(self.IpProtocol) + if protocol == "all" { + protocol = "any" + } + port, ports := "", strings.Split(self.PortRange, "/") + if ports[0] == ports[1] { + if ports[0] != "-1" { + port = ports[0] + } + } else if ports[0] != "1" && ports[1] != "65535" { + port = fmt.Sprintf("%s-%s", ports[0], ports[1]) + } + result := fmt.Sprintf("%s:%s", direction, string(action)) + if len(cidr) > 0 { + result += fmt.Sprintf(" %s", cidr) + } + result += fmt.Sprintf(" %s", protocol) + if len(port) > 0 { + result += fmt.Sprintf(" %s", port) + } + return result +} + +func (self *SRegion) addTagToSecurityGroup(secgroupId, key, value string, index int) error { + if index > 5 || index < 1 { + index = 1 + } + params := map[string]string{"ResourceType": "securitygroup", "ResourceId": secgroupId} + params[fmt.Sprintf("Tag.%d.Key", index)] = key + params[fmt.Sprintf("Tag.%d.Value", index)] = value + _, err := self.ecsRequest("AddTags", params) + return err +} + +func (self *SRegion) revokeSecurityGroup(secgroupId, instanceId string, keep bool) error { + if !keep { + return self.leaveSecurityGroup(secgroupId, instanceId) + } + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return err + } else { + for _, permission := range secgroup.Permissions.Permission { + if rule, err := secrules.ParseSecurityRule(permission.String()); err != nil { + return err + } else { + rule.Priority = permission.Priority + if err := self.delSecurityGroupRule(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + } + if rule, err := secrules.ParseSecurityRule("in:allow any"); err != nil { + rule.Priority = 100 + if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + if rule, err := secrules.ParseSecurityRule("out:allow any"); err != nil { + rule.Priority = 100 + if err := self.addSecurityGroupRules(secgroup.SecurityGroupId, rule); err != nil { + return err + } + } + } + return nil +} + +func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error { + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return err + } else { + + sort.Sort(secrules.SecurityRuleSet(rules)) + sort.Sort(PermissionSet(secgroup.Permissions.Permission)) + + i, j := 0, 0 + for i < len(rules) || j < len(secgroup.Permissions.Permission) { + if i < len(rules) && j < len(secgroup.Permissions.Permission) { + permissionStr := secgroup.Permissions.Permission[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(permissionStr, ruleStr) + if cmp == 0 { + if secgroup.Permissions.Permission[j].Description != rules[i].Description { + rules[i].Priority = secgroup.Permissions.Permission[j].Priority + if err := self.modifySecurityGroupRule(secgroupId, &rules[i]); err != nil { + log.Errorf("modifySecurityGroupRule error %v", rules[i]) + return err + } + } + i += 1 + j += 1 + } else if cmp > 0 { + if rule, err := secrules.ParseSecurityRule(permissionStr); err != nil { + return err + } else { + rule.Priority = secgroup.Permissions.Permission[j].Priority + if err := self.delSecurityGroupRule(secgroupId, rule); err != nil { + log.Errorf("delSecurityGroupRule error %v", rule) + return err + } + } + j += 1 + } else { + if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil { + log.Errorf("addSecurityGroupRule error %v", rules[i]) + return err + } + i += 1 + } + } else if i >= len(rules) { + permissionStr := secgroup.Permissions.Permission[j].String() + if rule, err := secrules.ParseSecurityRule(permissionStr); err != nil { + return err + } else { + rule.Priority = secgroup.Permissions.Permission[j].Priority + if err := self.delSecurityGroupRule(secgroupId, rule); err != nil { + log.Errorf("delSecurityGroupRule error %v", rule) + return err + } + } + j += 1 + } else if j >= len(secgroup.Permissions.Permission) { + if err := self.addSecurityGroupRules(secgroupId, &rules[i]); err != nil { + log.Errorf("addSecurityGroupRule error %v", rules[i]) + return err + } + i += 1 + } + } + } + return nil +} + +func (self *SRegion) assignSecurityGroup(secgroupId, instanceId string) error { + params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId} + _, err := self.ecsRequest("JoinSecurityGroup", params) + return err +} + +func (self *SRegion) leaveSecurityGroup(secgroupId, instanceId string) error { + params := map[string]string{"InstanceId": instanceId, "SecurityGroupId": secgroupId} + _, err := self.ecsRequest("LeaveSecurityGroup", params) + return err +} diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index 0376b3b5f7..b54700b1e6 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -113,3 +113,12 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr disk.storage = self return disk, nil } + +func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { + if disk, err := self.zone.region.getDisk(idStr); err != nil { + return nil, err + } else { + disk.storage = self + return disk, nil + } +} diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index d61028a555..9751039c39 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -5,7 +5,9 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/pkg/util/secrules" ) const ( @@ -28,7 +30,7 @@ type SVpc struct { iwires []cloudprovider.ICloudWire - secgroups []SSecurityGroup + secgroups []cloudprovider.ICloudSecurityGroup CidrBlock string CreationTime time.Time @@ -159,11 +161,15 @@ func (self *SVpc) fetchSecurityGroups() error { break } } - self.secgroups = secgroups + self.secgroups = make([]cloudprovider.ICloudSecurityGroup, len(secgroups)) + for index, secgroup := range secgroups { + secgroup.vpc = self + self.secgroups[index] = &secgroup + } return nil } -func (self *SVpc) GetSecurityGroups() ([]SSecurityGroup, error) { +func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { if self.secgroups == nil { err := self.fetchSecurityGroups() if err != nil { @@ -180,3 +186,39 @@ func (self *SVpc) GetManagerId() string { func (self *SVpc) Delete() error { return self.region.DeleteVpc(self.VpcId) } + +func (self *SVpc) syncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) (string, error) { + secgrpId := "" + if secgroup, err := self.region.getSecurityGroupByTag(self.VpcId, secgroupId); err != nil { + if secgrpId, err = self.region.createSecurityGroup(self.VpcId, name, ""); err != nil { + return "", err + } else if err := self.region.addTagToSecurityGroup(secgrpId, "id", secgroupId, 1); err != nil { + return "", err + } + //addRules + log.Debugf("Add Rules for %s", secgrpId) + for _, rule := range rules { + if err := self.region.addSecurityGroupRule(secgrpId, &rule); err != nil { + return "", err + } + } + } else { + //syncRules + secgrpId = secgroup.SecurityGroupId + log.Debugf("Sync Rules for %s", secgroup.GetName()) + if secgroup.GetName() != name { + if err := self.region.modifySecurityGroup(secgrpId, name, ""); err != nil { + log.Errorf("Change SecurityGroup name to %s failed: %v", name, err) + } + } + self.region.syncSecgroupRules(secgrpId, rules) + } + return secgrpId, nil +} +func (self *SVpc) assignSecurityGroup(secgroupId string, instanceId string) error { + return self.region.assignSecurityGroup(secgroupId, instanceId) +} + +func (self *SVpc) revokeSecurityGroup(secgroupId string, instanceId string, keep bool) error { + return self.region.revokeSecurityGroup(secgroupId, instanceId, keep) +} diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 1c7fdd3f7e..90846ed5a9 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -54,6 +54,10 @@ func (self *SDatastore) GetIZone() cloudprovider.ICloudZone { return nil } +func (self *SDatastore) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { + return nil, cloudprovider.ErrNotImplemented +} + func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 11b1f7231a..2005bbc9bc 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -1,18 +1,18 @@ package esxi import ( + "fmt" "time" + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/secrules" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - - "fmt" - "github.com/vmware/govmomi/object" ) var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary"} @@ -35,6 +35,10 @@ func (self *SVirtualMachine) GetGlobalId() string { return self.getUuid() } +func (self *SVirtualMachine) SyncSecurityGroup(secgroupId, name string, rules []secrules.SecurityRule) error { + return nil +} + func (self *SVirtualMachine) GetStatus() string { vm := object.NewVirtualMachine(self.manager.client.Client, self.getVirtualMachine().Self) state, err := vm.PowerState(self.manager.context) diff --git a/vendor/github.com/serialx/hashring/LICENSE b/vendor/github.com/serialx/hashring/LICENSE new file mode 100644 index 0000000000..cbc54295ad --- /dev/null +++ b/vendor/github.com/serialx/hashring/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sung-jin Hong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/serialx/hashring/README.md b/vendor/github.com/serialx/hashring/README.md new file mode 100644 index 0000000000..0077c4d703 --- /dev/null +++ b/vendor/github.com/serialx/hashring/README.md @@ -0,0 +1,70 @@ +hashring +============================ + +Implements consistent hashing that can be used when +the number of server nodes can increase or decrease (like in memcached). +The hashing ring is built using the same algorithm as libketama. + +This is a port of Python hash_ring library +in Go with the extra methods to add and remove nodes. + + +Using +============================ + +Importing :: + +```go +import "github.com/serialx/hashring" +``` + +Basic example usage :: + +```go +memcacheServers := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.249:11212"} + +ring := hashring.New(memcacheServers) +server, _ := ring.GetNode("my_key") +``` + +To fulfill replication requirements, you can also get a list of servers that should store your key. +```go +serversInRing := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.248:11212", + "192.168.0.249:11212", + "192.168.0.250:11212", + "192.168.0.251:11212", + "192.168.0.252:11212"} + +replicaCount := 3 +ring := hashring.New(serversInRing) +server, _ := ring.GetNodes("my_key", replicaCount) +``` + +Using weights example :: + +```go +weights := make(map[string]int) +weights["192.168.0.246:11212"] = 1 +weights["192.168.0.247:11212"] = 2 +weights["192.168.0.249:11212"] = 1 + +ring := hashring.NewWithWeights(weights) +server, _ := ring.GetNode("my_key") +``` + +Adding and removing nodes example :: + +```go +memcacheServers := []string{"192.168.0.246:11212", + "192.168.0.247:11212", + "192.168.0.249:11212"} + +ring := hashring.New(memcacheServers) +ring = ring.RemoveNode("192.168.0.246:11212") +ring = ring.AddNode("192.168.0.250:11212") +server, _ := ring.GetNode("my_key") +``` diff --git a/vendor/github.com/serialx/hashring/hashring.go b/vendor/github.com/serialx/hashring/hashring.go new file mode 100644 index 0000000000..4f4f1d59dd --- /dev/null +++ b/vendor/github.com/serialx/hashring/hashring.go @@ -0,0 +1,268 @@ +package hashring + +import ( + "crypto/md5" + "fmt" + "math" + "sort" +) + +type HashKey uint32 +type HashKeyOrder []HashKey + +func (h HashKeyOrder) Len() int { return len(h) } +func (h HashKeyOrder) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h HashKeyOrder) Less(i, j int) bool { return h[i] < h[j] } + +type HashRing struct { + ring map[HashKey]string + sortedKeys []HashKey + nodes []string + weights map[string]int +} + +func New(nodes []string) *HashRing { + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: make(map[string]int), + } + hashRing.generateCircle() + return hashRing +} + +func NewWithWeights(weights map[string]int) *HashRing { + nodes := make([]string, 0, len(weights)) + for node, _ := range weights { + nodes = append(nodes, node) + } + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func (h *HashRing) Size() int { + return len(h.nodes) +} + +func (h *HashRing) UpdateWithWeights(weights map[string]int) { + nodesChgFlg := false + if len(weights) != len(h.weights) { + nodesChgFlg = true + } else { + for node, newWeight := range weights { + oldWeight, ok := h.weights[node] + if !ok || oldWeight != newWeight { + nodesChgFlg = true + break + } + } + } + + if nodesChgFlg { + newhring := NewWithWeights(weights) + h.weights = newhring.weights + h.nodes = newhring.nodes + h.ring = newhring.ring + h.sortedKeys = newhring.sortedKeys + } +} + +func (h *HashRing) generateCircle() { + totalWeight := 0 + for _, node := range h.nodes { + if weight, ok := h.weights[node]; ok { + totalWeight += weight + } else { + totalWeight += 1 + h.weights[node] = 1 + } + } + + for _, node := range h.nodes { + weight := h.weights[node] + + factor := math.Floor(float64(40*len(h.nodes)*weight) / float64(totalWeight)) + + for j := 0; j < int(factor); j++ { + nodeKey := fmt.Sprintf("%s-%d", node, j) + bKey := hashDigest(nodeKey) + + for i := 0; i < 3; i++ { + key := hashVal(bKey[i*4 : i*4+4]) + h.ring[key] = node + h.sortedKeys = append(h.sortedKeys, key) + } + } + } + + sort.Sort(HashKeyOrder(h.sortedKeys)) +} + +func (h *HashRing) GetNode(stringKey string) (node string, ok bool) { + pos, ok := h.GetNodePos(stringKey) + if !ok { + return "", false + } + return h.ring[h.sortedKeys[pos]], true +} + +func (h *HashRing) GetNodePos(stringKey string) (pos int, ok bool) { + if len(h.ring) == 0 { + return 0, false + } + + key := h.GenKey(stringKey) + + nodes := h.sortedKeys + pos = sort.Search(len(nodes), func(i int) bool { return nodes[i] > key }) + + if pos == len(nodes) { + // Wrap the search, should return first node + return 0, true + } else { + return pos, true + } +} + +func (h *HashRing) GenKey(key string) HashKey { + bKey := hashDigest(key) + return hashVal(bKey[0:4]) +} + +func (h *HashRing) GetNodes(stringKey string, size int) (nodes []string, ok bool) { + pos, ok := h.GetNodePos(stringKey) + if !ok { + return nil, false + } + + if size > len(h.nodes) { + return nil, false + } + + returnedValues := make(map[string]bool, size) + //mergedSortedKeys := append(h.sortedKeys[pos:], h.sortedKeys[:pos]...) + resultSlice := make([]string, 0, size) + + for i := pos; i < pos+len(h.sortedKeys); i++ { + key := h.sortedKeys[i%len(h.sortedKeys)] + val := h.ring[key] + if !returnedValues[val] { + returnedValues[val] = true + resultSlice = append(resultSlice, val) + } + if len(returnedValues) == size { + break + } + } + + return resultSlice, len(resultSlice) == size +} + +func (h *HashRing) AddNode(node string) *HashRing { + return h.AddWeightedNode(node, 1) +} + +func (h *HashRing) AddWeightedNode(node string, weight int) *HashRing { + if weight <= 0 { + return h + } + + if _, ok := h.weights[node]; ok { + return h + } + + nodes := make([]string, len(h.nodes), len(h.nodes)+1) + copy(nodes, h.nodes) + nodes = append(nodes, node) + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + weights[eNode] = eWeight + } + weights[node] = weight + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func (h *HashRing) UpdateWeightedNode(node string, weight int) *HashRing { + if weight <= 0 { + return h + } + + /* node is not need to update for node is not existed or weight is not changed */ + if oldWeight, ok := h.weights[node]; (!ok) || (ok && oldWeight == weight) { + return h + } + + nodes := make([]string, len(h.nodes), len(h.nodes)) + copy(nodes, h.nodes) + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + weights[eNode] = eWeight + } + weights[node] = weight + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} +func (h *HashRing) RemoveNode(node string) *HashRing { + /* if node isn't exist in hashring, don't refresh hashring */ + if _, ok := h.weights[node]; !ok { + return h + } + + nodes := make([]string, 0) + for _, eNode := range h.nodes { + if eNode != node { + nodes = append(nodes, eNode) + } + } + + weights := make(map[string]int) + for eNode, eWeight := range h.weights { + if eNode != node { + weights[eNode] = eWeight + } + } + + hashRing := &HashRing{ + ring: make(map[HashKey]string), + sortedKeys: make([]HashKey, 0), + nodes: nodes, + weights: weights, + } + hashRing.generateCircle() + return hashRing +} + +func hashVal(bKey []byte) HashKey { + return ((HashKey(bKey[3]) << 24) | + (HashKey(bKey[2]) << 16) | + (HashKey(bKey[1]) << 8) | + (HashKey(bKey[0]))) +} + +func hashDigest(key string) [md5.Size]byte { + return md5.Sum([]byte(key)) +} From fd69dbac820419340cdca01575f2a8ba6189cde1 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 22 Aug 2018 01:06:10 +0800 Subject: [PATCH 27/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3:=20=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E5=99=A8=20baremetal=20raid=20config=20range=20=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E6=AD=A3=E7=A1=AE=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/scheduler/api/sched.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index a36ea74ab3..85a7017066 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -551,7 +551,15 @@ func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*BaremetalDis rangeArray := make([]int64, 0) ranges := sjson.Get("range").MustArray() for _, size := range ranges { - rangeArray = append(rangeArray, size.(int64)) + sizeNum, ok := size.(json.Number) + if !ok { + return nil, fmt.Errorf("Invalid range size, not json.Number: %v", size) + } + rangeSize, err := sizeNum.Int64() + if err != nil { + return nil, err + } + rangeArray = append(rangeArray, rangeSize) } baremetalDiskConfig.Range = rangeArray From 95e63d9f87b5bee6f836053c122434085cb9cdff Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 22 Aug 2018 03:14:57 +0800 Subject: [PATCH 28/71] climc: kube cluster, node and service resource rename --- pkg/mcclient/modules/k8s/cluster.go | 2 +- pkg/mcclient/modules/k8s/node.go | 2 +- pkg/mcclient/modules/k8s/service.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/mcclient/modules/k8s/cluster.go b/pkg/mcclient/modules/k8s/cluster.go index 42ee41d6b1..46b7bb4fea 100644 --- a/pkg/mcclient/modules/k8s/cluster.go +++ b/pkg/mcclient/modules/k8s/cluster.go @@ -9,7 +9,7 @@ var ( ) func init() { - Clusters = NewManager("cluster", "clusters", + Clusters = NewManager("kube_cluster", "kube_clusters", NewResourceCols("mode", "k8s_version", "status", "api_endpoint"), NewColumns()) modules.Register(Clusters) diff --git a/pkg/mcclient/modules/k8s/node.go b/pkg/mcclient/modules/k8s/node.go index 60f5bd6ade..630c0ffb8c 100644 --- a/pkg/mcclient/modules/k8s/node.go +++ b/pkg/mcclient/modules/k8s/node.go @@ -9,6 +9,6 @@ var ( ) func init() { - Nodes = NewManager("node", "nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns()) + Nodes = NewManager("kube_node", "kube_nodes", NewResourceCols("cluster", "roles", "address", "status"), NewColumns()) modules.Register(Nodes) } diff --git a/pkg/mcclient/modules/k8s/service.go b/pkg/mcclient/modules/k8s/service.go index e66f38dda3..7a153c4bc4 100644 --- a/pkg/mcclient/modules/k8s/service.go +++ b/pkg/mcclient/modules/k8s/service.go @@ -13,7 +13,7 @@ type ServiceManager struct { func init() { Services = &ServiceManager{ ResourceManager: *NewManager( - "service", "services", + "k8s_service", "k8s_services", NewNamespaceCols("clusterIP", "selector", "internalEndpoint", "externalEndpoints"), NewClusterCols())} modules.Register(Services) From af61f13df6383b21354337bfe01f4d09a81959a2 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 22 Aug 2018 11:05:39 +0800 Subject: [PATCH 29/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3:=20=E4=B8=8D=E5=90=8C?= =?UTF-8?q?=20ServiceType=20=E7=9A=84=20module=20=E6=B3=A8=E5=86=8C?= =?UTF-8?q?=E4=B8=8D=E4=BC=9A=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/mcclient/modules/modules.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/mcclient/modules/modules.go b/pkg/mcclient/modules/modules.go index 2381f50232..1ba4171f98 100644 --- a/pkg/mcclient/modules/modules.go +++ b/pkg/mcclient/modules/modules.go @@ -2,10 +2,10 @@ package modules import ( "fmt" - "log" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -142,6 +142,17 @@ func _getJointKey(mod1 Manager, mod2 Manager) string { return fmt.Sprintf("%s-%s", mod1.KeyString(), mod2.KeyString()) } +func ensureModuleNotRegistered(mod, newMod BaseManagerInterface) { + modSvcType := mod.ServiceType() + newModSvcType := newMod.ServiceType() + if mod == newMod { + log.Fatalf("Module %#v duplicate registered, service type: %q", mod, modSvcType) + } + if modSvcType != newModSvcType { + log.Fatalf("Module %#v already registered, service type is %q.\nSo new module %#v can't be registered, service type is %q", mod, modSvcType, newMod, newModSvcType) + } +} + func _register(version string, mod BaseManagerInterface) { if modules == nil { modules = make(map[string]map[string][]BaseManagerInterface) @@ -156,9 +167,7 @@ func _register(version string, mod BaseManagerInterface) { mods = make([]BaseManagerInterface, 0) } for _, m := range mods { - if m == mod { - log.Fatal("Module already registered", mod) - } + ensureModuleNotRegistered(m, mod) } modtable[mod.KeyString()] = append(mods, mod) } @@ -179,7 +188,7 @@ func _registerJointModule(version string, mod BaseManagerInterface) { } for _, m := range jointMods { if m == jointMod { - log.Fatal("Module already registered", jointMod) + ensureModuleNotRegistered(m, jointMod) } } modtable[jointKey] = append(jointMods, jointMod) From 80571f480f3d4b4dca88954db9836de09d1eb0a3 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 22 Aug 2018 12:36:28 +0800 Subject: [PATCH 30/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Asimpletoken?= =?UTF-8?q?=E6=B2=A1=E6=9C=89servicecatalog=EF=BC=8C=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=9F=BA=E4=BA=8Esimpletoken=E7=9A=84session=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E8=8E=B7=E5=BE=97service=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- pkg/mcclient/catalog.go | 6 +++++ pkg/mcclient/mcclient.go | 51 +++++++++++++++++-------------------- pkg/mcclient/session.go | 21 ++++++++++++++- pkg/mcclient/token.go | 6 ++++- pkg/mcclient/token2.go | 20 +++++++++++++-- pkg/mcclient/token3.go | 20 +++++++++++++-- pkg/mcclient/tokensimple.go | 8 ++++++ 8 files changed, 99 insertions(+), 35 deletions(-) create mode 100644 pkg/mcclient/catalog.go diff --git a/Makefile b/Makefile index 6537fbd733..5dadd4f7bb 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell readlink -f `pwd`) +ROOT_DIR := $(shell pwd -P) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh diff --git a/pkg/mcclient/catalog.go b/pkg/mcclient/catalog.go new file mode 100644 index 0000000000..1238bf6723 --- /dev/null +++ b/pkg/mcclient/catalog.go @@ -0,0 +1,6 @@ +package mcclient + +type IServiceCatalog interface { + GetServiceURL(service, region, zone, endpointType string) (string, error) + GetServiceURLs(service, region, zone, endpointType string) ([]string, error) +} diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 4df0561160..45ee26bc1c 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -10,36 +10,16 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/log" ) -/*const ( - USER_AGENT = "yunioncloud-go/201708" -) - -var ( - red = color.New(color.FgRed, color.Bold).PrintlnFunc() - green = color.New(color.FgGreen, color.Bold).PrintlnFunc() - yellow = color.New(color.FgYellow, color.Bold).PrintlnFunc() - cyan = color.New(color.FgHiCyan, color.Bold).PrintlnFunc() -)*/ - -/*type JSONClientError struct { - Code int - Class string - Details string -} - -func (e *JSONClientError) Error() string { - return fmt.Sprintf("JSONClientError: %s %d %s", e.Details, e.Code, e.Class) -}*/ - type Client struct { authUrl string timeout int debug bool httpconn *http.Client - // serviceCatalog *KeystoneServiceCatalog + serviceCatalog IServiceCatalog } func NewClient(authUrl string, timeout int, debug bool, insecure bool) *Client { @@ -222,7 +202,7 @@ func (this *Client) _authV3(domainName, uname, passwd, projectId, projectName, t if len(tokenId) == 0 { return nil, fmt.Errorf("No X-Subject-Token in header") } - ret, err := UnmarshalV3Token(rbody, tokenId) + ret, err := this.unmarshalV3Token(rbody, tokenId) return ret, err } @@ -245,7 +225,7 @@ func (this *Client) _authV2(uname, passwd, tenantId, tenantName, token string) ( if err != nil { return nil, err } - return UnmarshalV2Token(rbody) + return this.unmarshalV2Token(rbody) } func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) (TokenCredential, error) { @@ -255,16 +235,21 @@ func (this *Client) Authenticate(uname, passwd, domainName, tenantName string) ( return this._authV2(uname, passwd, "", tenantName, "") } -func UnmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) { +func (this *Client) unmarshalV3Token(rbody jsonutils.JSONObject, tokenId string) (cred TokenCredential, err error) { cred = &TokenCredentialV3{Id: tokenId} err = rbody.Unmarshal(cred) if err != nil { err = fmt.Errorf("Invalid response when unmarshal V3 Token: %v", err) } + cata := cred.GetServiceCatalog() + if cata == nil { + log.Fatalf("No srvice catalog avaiable") + } + this.serviceCatalog = cata return } -func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) { +func (this *Client) unmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err error) { access, err := rbody.Get("access") if err == nil { cred = &TokenCredentialV2{} @@ -272,6 +257,11 @@ func UnmarshalV2Token(rbody jsonutils.JSONObject) (cred TokenCredential, err err if err != nil { err = fmt.Errorf("Invalid response when unmarshal V2 Token: %s", err) } + cata := cred.GetServiceCatalog() + if cata == nil { + log.Fatalf("No srvice catalog avaiable") + } + this.serviceCatalog = cata return } err = fmt.Errorf("Invalid response: no access object") @@ -286,7 +276,7 @@ func (this *Client) verifyV3(adminToken, token string) (TokenCredential, error) if err != nil { return nil, err } - return UnmarshalV3Token(rbody, token) + return this.unmarshalV3Token(rbody, token) } func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) { @@ -297,7 +287,7 @@ func (this *Client) verifyV2(adminToken, token string) (TokenCredential, error) if err != nil { return nil, err } - return UnmarshalV2Token(rbody) + return this.unmarshalV2Token(rbody) } func (this *Client) Verify(adminToken, token string) (cred TokenCredential, err error) { @@ -320,6 +310,11 @@ func (this *Client) SetProject(tenantId, tenantName string, token TokenCredentia } func (this *Client) NewSession(region, zone, endpointType string, token TokenCredential, apiVersion string) *ClientSession { + cata := token.GetServiceCatalog() + if cata == nil { + log.Fatalf("Missing service catalog in token") + } + this.serviceCatalog = cata return &ClientSession{client: this, region: region, zone: zone, endpointType: endpointType, token: token, apiVersion: apiVersion, diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index 3526f6eea4..527e05c4e4 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -75,7 +75,26 @@ func (this *ClientSession) GetServiceURL(service, endpointType string) (string, if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { service = fmt.Sprintf("%s_%s", service, this.apiVersion) } - return this.token.GetServiceURL(service, this.region, this.zone, endpointType) + url, err := this.token.GetServiceURL(service, this.region, this.zone, endpointType) + if err != nil { + url, err = this.client.serviceCatalog.GetServiceURL(service, this.region, this.zone, endpointType) + } + return url, err +} + +func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]string, error) { + if len(this.endpointType) > 0 { + // session specific endpoint type should override the input endpointType, which is supplied by manager + endpointType = this.endpointType + } + if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { + service = fmt.Sprintf("%s_%s", service, this.apiVersion) + } + urls, err := this.token.GetServiceURLs(service, this.region, this.zone, endpointType) + if err != nil { + urls, err = this.client.serviceCatalog.GetServiceURLs(service, this.region, this.zone, endpointType) + } + return urls, err } func (this *ClientSession) getBaseUrl(service, endpointType string) (string, error) { diff --git a/pkg/mcclient/token.go b/pkg/mcclient/token.go index 7a9ba2751b..8c0e5e00ed 100644 --- a/pkg/mcclient/token.go +++ b/pkg/mcclient/token.go @@ -23,6 +23,8 @@ type Endpoint struct { type TokenCredential interface { gotypes.ISerializable + IServiceCatalog + GetTokenString() string GetDomainId() string GetDomainName() string @@ -39,7 +41,9 @@ type TokenCredential interface { IsAdmin() bool IsSystemAdmin() bool GetRegions() []string - GetServiceURL(service, region, zone, endpointType string) (string, error) + + GetServiceCatalog() IServiceCatalog + GetInternalServices(region string) []string GetExternalServices(region string) []ExternalService GetEndpoints(region string, endpointType string) []Endpoint diff --git a/pkg/mcclient/token2.go b/pkg/mcclient/token2.go index d38bd052b9..daca173d17 100644 --- a/pkg/mcclient/token2.go +++ b/pkg/mcclient/token2.go @@ -133,7 +133,11 @@ func (this *TokenCredentialV2) IsSystemAdmin() bool { } func (this *TokenCredentialV2) GetServiceURL(service, region, zone, endpointType string) (string, error) { - return this.ServiceCatalog.getServiceURL(service, region, zone, endpointType) + return this.ServiceCatalog.GetServiceURL(service, region, zone, endpointType) +} + +func (this *TokenCredentialV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return this.ServiceCatalog.GetServiceURLs(service, region, zone, endpointType) } func (this *TokenCredentialV2) GetInternalServices(region string) []string { @@ -148,6 +152,10 @@ func (this *TokenCredentialV2) GetEndpoints(region string, endpointType string) return nil } +func (this *TokenCredentialV2) GetServiceCatalog() IServiceCatalog { + return this.ServiceCatalog +} + func stringArrayContains(arr []string, needle string) bool { for i := 0; i < len(arr); i++ { if arr[i] == needle { @@ -211,7 +219,7 @@ func (catalog KeystoneServiceCatalogV2) getServiceEndpoint(service, region, zone return selected, fmt.Errorf("No such service %s", service) } -func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, endpointType string) (string, error) { +func (catalog KeystoneServiceCatalogV2) GetServiceURL(service, region, zone, endpointType string) (string, error) { ep, err := catalog.getServiceEndpoint(service, region, zone) if err != nil { return "", err @@ -219,6 +227,14 @@ func (catalog KeystoneServiceCatalogV2) getServiceURL(service, region, zone, end return ep.getURL(endpointType), nil } +func (catalog KeystoneServiceCatalogV2) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + url, err := catalog.GetServiceURL(service, region, zone, endpointType) + if err != nil { + return nil, err + } + return []string{url}, nil +} + func (ep KeystoneEndpointV2) getURL(epType string) string { switch epType { case "publicURL": diff --git a/pkg/mcclient/token3.go b/pkg/mcclient/token3.go index f7265da44e..09d49b870b 100644 --- a/pkg/mcclient/token3.go +++ b/pkg/mcclient/token3.go @@ -141,7 +141,11 @@ func (this *TokenCredentialV3) GetRegions() []string { } func (this *TokenCredentialV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { - return this.Token.Catalog.getServiceURL(service, region, zone, endpointType) + return this.Token.Catalog.GetServiceURL(service, region, zone, endpointType) +} + +func (this *TokenCredentialV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return this.Token.Catalog.GetServiceURLs(service, region, zone, endpointType) } func (this *TokenCredentialV3) GetInternalServices(region string) []string { @@ -156,6 +160,10 @@ func (this *TokenCredentialV3) GetEndpoints(region string, endpointType string) return this.Token.Catalog.getEndpoints(region, endpointType) } +func (this *TokenCredentialV3) GetServiceCatalog() IServiceCatalog { + return this.Token.Catalog +} + func (catalog KeystoneServiceCatalogV3) getInternalServices(region string) []string { services := make([]string, 0) for i := 0; i < len(catalog); i++ { @@ -242,7 +250,15 @@ func Id2RegionZone(id string) (string, string) { } } -func (catalog KeystoneServiceCatalogV3) getServiceURL(service, region, zone, endpointType string) (string, error) { +func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + url, err := catalog.GetServiceURL(service, region, zone, endpointType) + if err != nil { + return nil, err + } + return []string{url}, nil +} + +func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { if endpointType == "" { endpointType = "internalURL" } diff --git a/pkg/mcclient/tokensimple.go b/pkg/mcclient/tokensimple.go index 2054560440..5be331fbd9 100644 --- a/pkg/mcclient/tokensimple.go +++ b/pkg/mcclient/tokensimple.go @@ -96,6 +96,10 @@ func (self *SSimpleToken) GetServiceURL(service, region, zone, endpointType stri return "", fmt.Errorf("Not available") } +func (self *SSimpleToken) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { + return nil, fmt.Errorf("Not available") +} + func (self *SSimpleToken) GetInternalServices(region string) []string { return nil } @@ -108,6 +112,10 @@ func (this *SSimpleToken) GetEndpoints(region string, endpointType string) []End return nil } +func (this *SSimpleToken) GetServiceCatalog() IServiceCatalog { + return nil +} + func SimplifyToken(token TokenCredential) TokenCredential { simToken, ok := token.(*SSimpleToken) if ok { From b77f39d194077c6721d47e9e79da7e8095116afd Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 22 Aug 2018 16:11:18 +0800 Subject: [PATCH 31/71] minor updates --- Makefile | 2 +- cmd/climc/climc.go | 29 +++++++++++++++++------------ pkg/mcclient/token3.go | 18 +++++++++--------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 5dadd4f7bb..923270969f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell pwd -P) +ROOT_DIR := $(CURDIR) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index 1d4bdec473..a8cc818dd9 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -5,28 +5,32 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "time" prompt "github.com/c-bata/go-prompt" + "yunion.io/x/log" "yunion.io/x/pkg/util/version" "yunion.io/x/structarg" "yunion.io/x/onecloud/cmd/climc/promputils" "yunion.io/x/onecloud/cmd/climc/shell" - _ "yunion.io/x/onecloud/cmd/climc/shell/k8s" "yunion.io/x/onecloud/pkg/mcclient" + + _ "yunion.io/x/onecloud/cmd/climc/shell/k8s" ) type BaseOptions struct { - Help bool `help:"Show help" short-token:"h"` - Debug bool `help:"Show debug information"` - Version bool `help:"Show version"` - Timeout int `default:"600" help:"Number of seconds to wait for a response"` - Insecure bool `default:"False" help:"Allow skip server cert verification if URL is https" short-token:"k"` - OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` - OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` + Help bool `help:"Show help" short-token:"h"` + Debug bool `help:"Show debug information"` + Version bool `help:"Show version"` + Timeout int `default:"600" help:"Number of seconds to wait for a response"` + Insecure bool `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"` + NoCachedToken bool `default:"false" help:"Force not use cached token"` + OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` + OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` // OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"` OsProjectName string `default:"$OS_PROJECT_NAME" help:"Project name, defaults to env[OS_PROJECT_NAME]"` OsDomainName string `default:"$OS_DOMAIN_NAME" help:"Domain name, defaults to env[OS_DOMAIN_NAME]"` @@ -122,8 +126,9 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { options.Insecure) var cacheToken mcclient.TokenCredential - cacheFile, err := os.Open("/tmp/OS_AUTH_CACHE_TOKEN") - if err == nil && cacheFile != nil { + tokenCachePath := filepath.Join(os.TempDir(), "OS_AUTH_CACHE_TOKEN") + cacheFile, err := os.Open(tokenCachePath) + if err == nil && cacheFile != nil && !options.NoCachedToken { fileInfo, _ := cacheFile.Stat() dur, err := time.ParseDuration("-24h") if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { @@ -153,12 +158,12 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { if err != nil { fmt.Printf("Marshal token error:%s", err) } else { - fo, _ := os.Create("/tmp/OS_AUTH_CACHE_TOKEN") + fo, _ := os.Create(tokenCachePath) fo.Write(bytesCacheToken) fo.Close() } } else { - fmt.Println("******** Use Token Cache At /tmp/OS_AUTH_CACHE_TOKEN ********") + fmt.Printf("******** Use Token Cache At %s ********\n", tokenCachePath) } session := client.NewSession(options.OsRegionName, diff --git a/pkg/mcclient/token3.go b/pkg/mcclient/token3.go index 09d49b870b..233c336be2 100644 --- a/pkg/mcclient/token3.go +++ b/pkg/mcclient/token3.go @@ -250,15 +250,15 @@ func Id2RegionZone(id string) (string, string) { } } -func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { - url, err := catalog.GetServiceURL(service, region, zone, endpointType) +func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { + urls, err := catalog.GetServiceURLs(service, region, zone, endpointType) if err != nil { - return nil, err + return "", err } - return []string{url}, nil + return urls[rand.Intn(len(urls))], nil } -func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, endpointType string) (string, error) { +func (catalog KeystoneServiceCatalogV3) GetServiceURLs(service, region, zone, endpointType string) ([]string, error) { if endpointType == "" { endpointType = "internalURL" } @@ -289,7 +289,7 @@ func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, end break } } else { - return "", fmt.Errorf("No default region") + return nil, fmt.Errorf("No default region") } } else { _, ok := regeps[regionzone] @@ -298,14 +298,14 @@ func (catalog KeystoneServiceCatalogV3) GetServiceURL(service, region, zone, end } else { selected, ok = regeps[region] if !ok { - return "", fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone)) + return nil, fmt.Errorf("No valid %s endpoints for %s in region %s", endpointType, service, RegionID(region, zone)) } } } - return selected[rand.Intn(len(selected))], nil + return selected, nil } } - return "", fmt.Errorf("No such service %s", service) + return nil, fmt.Errorf("No such service %s", service) } func (self *TokenCredentialV3) String() string { From 0d6a9f33c60e69ec3733d2fa6cf0d7b07571a2f3 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 13:32:11 +0800 Subject: [PATCH 32/71] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=EF=BC=9A=E9=80=9A?= =?UTF-8?q?=E8=BF=87cloud=20provider=E6=9D=A5=E8=BF=87=E6=BB=A4=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E6=9C=BA=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/servers.go | 4 +++ pkg/compute/models/guests.go | 61 ++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index fe663182b6..f0d74b5d59 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -64,6 +64,7 @@ func init() { Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun"` + Manager string `help:"Show servers imported from manager"` BaseListOptions } R(&ServerListOptions{}, "server-list", "List virtual servers", func(s *mcclient.ClientSession, args *ServerListOptions) error { @@ -98,6 +99,9 @@ func init() { if len(args.Hypervisor) > 0 { params.Add(jsonutils.NewString(args.Hypervisor), "hypervisor") } + if len(args.Manager) > 0 { + params.Add(jsonutils.NewString(args.Manager), "manager") + } result, err := modules.Servers.List(s, params) if err != nil { return err diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 401bdbd1b0..7b91be0760 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -202,49 +202,60 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ if len(hypervisor) > 0 { q = q.Equals("hypervisor", hypervisor) } + hostFilter, _ := queryDict.GetString("host") - zoneFilter, _ := queryDict.GetString("zone") - wireFilter, _ := queryDict.GetString("wire") - networkFilter, _ := queryDict.GetString("network") - diskFilter, _ := queryDict.GetString("disk") - var sq *sqlchemy.SSubQuery if len(hostFilter) > 0 { host, _ := HostManager.FetchByIdOrName("", hostFilter) if host == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("host %s not found", hostFilter)) + return nil, httperrors.NewResourceNotFoundError("host %s not found", hostFilter) } - sq = HostManager.Query("id").Equals("id", host.GetId()).SubQuery() - } else if len(zoneFilter) > 0 { + q = q.Equals("host_id", host.GetId()) + } + + zoneFilter, _ := queryDict.GetString("zone") + if len(zoneFilter) > 0 { zone, _ := ZoneManager.FetchByIdOrName("", zoneFilter) if zone == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("zone %s not found", zoneFilter)) + return nil, httperrors.NewResourceNotFoundError("zone %s not found", zoneFilter) } hostTable := HostManager.Query().SubQuery() zoneTable := ZoneManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(zoneTable, + sq := hostTable.Query(hostTable.Field("id")).Join(zoneTable, sqlchemy.Equals(zoneTable.Field("id"), hostTable.Field("zone_id"))).Filter(sqlchemy.Equals(zoneTable.Field("id"), zone.GetId())).SubQuery() - } else if len(wireFilter) > 0 { + q = q.In("host_id", sq) + } + + wireFilter, _ := queryDict.GetString("wire") + if len(wireFilter) > 0 { wire, _ := WireManager.FetchByIdOrName("", wireFilter) if wire == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("wire %s not found", wireFilter)) + return nil, httperrors.NewResourceNotFoundError("wire %s not found", wireFilter) } hostTable := HostManager.Query().SubQuery() hostWire := HostwireManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery() - } else if len(networkFilter) > 0 { + sq := hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), wire.GetId())).SubQuery() + q = q.In("host_id", sq) + } + + networkFilter, _ := queryDict.GetString("network") + if len(networkFilter) > 0 { netI, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), networkFilter) if netI == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("network %s not found", networkFilter)) + return nil, httperrors.NewResourceNotFoundError("network %s not found", networkFilter) } net := netI.(*SNetwork) hostTable := HostManager.Query().SubQuery() hostWire := HostwireManager.Query().SubQuery() - sq = hostTable.Query(hostTable.Field("id")).Join(hostWire, + sq := hostTable.Query(hostTable.Field("id")).Join(hostWire, sqlchemy.Equals(hostWire.Field("host_id"), hostTable.Field("id"))).Filter(sqlchemy.Equals(hostWire.Field("wire_id"), net.WireId)).SubQuery() - } else if len(diskFilter) > 0 { + q = q.In("host_id", sq) + } + + diskFilter, _ := queryDict.GetString("disk") + if len(diskFilter) > 0 { diskI, _ := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskFilter) if diskI == nil { - return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("disk %s not found", diskFilter)) + return nil, httperrors.NewResourceNotFoundError("disk %s not found", diskFilter) } disk := diskI.(*SDisk) guestdisks := GuestdiskManager.Query().SubQuery() @@ -261,7 +272,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ hosts := HostManager.Query().SubQuery() hoststorages := HoststorageManager.Query().SubQuery() storages := StorageManager.Query().SubQuery() - sq = hosts.Query(hosts.Field("id")). + sq := hosts.Query(hosts.Field("id")). Join(hoststorages, sqlchemy.AND( sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id")), sqlchemy.IsFalse(hoststorages.Field("deleted")))). @@ -269,11 +280,21 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), sqlchemy.IsFalse(storages.Field("deleted")))). Filter(sqlchemy.Equals(storages.Field("id"), disk.StorageId)).SubQuery() + q = q.In("host_id", sq) } } - if sq != nil { + + managerFilter, _ := queryDict.GetString("manager") + if len(managerFilter) > 0 { + managerI, _ := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter) + if managerI == nil { + return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter) + } + hosts := HostManager.Query().SubQuery() + sq := hosts.Query(hosts.Field("id")).Equals("manager_id", managerI.GetId()).SubQuery() q = q.In("host_id", sq) } + gpu, _ := queryDict.GetString("gpu") if len(gpu) != 0 { isodev := IsolatedDeviceManager.Query().SubQuery() From e681012fd0d160f9d567887a12d3351de8f795b7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 18:22:47 +0800 Subject: [PATCH 33/71] =?UTF-8?q?=E8=99=9A=E6=8B=9F=E6=9C=BA=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0region=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 7b91be0760..142cd8688f 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -910,6 +910,12 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T if zone != nil { extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") + + region := zone.GetRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.Id), "region_id") + extra.Add(jsonutils.NewString(region.Name), "region") + } } extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") @@ -951,6 +957,12 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token if zone != nil { extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") extra.Add(jsonutils.NewString(zone.GetName()), "zone") + + region := zone.GetRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.Id), "region_id") + extra.Add(jsonutils.NewString(region.Name), "region") + } } return extra } From ac4275f48afcdc0b3bfb61cd99aecbff7b49fd53 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 20:11:10 +0800 Subject: [PATCH 34/71] =?UTF-8?q?=E5=A2=9E=E5=8A=A0guest=E7=9A=84region=5F?= =?UTF-8?q?external=5Fid=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 45 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 142cd8688f..d47ff0e5f8 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -906,17 +906,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T // extra.Add(jsonutils.NewString(self.getFlavorName()), "flavor") extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") - zone := self.getZone() - if zone != nil { - extra.Add(jsonutils.NewString(zone.Id), "zone_id") - extra.Add(jsonutils.NewString(zone.Name), "zone") - region := zone.GetRegion() - if region != nil { - extra.Add(jsonutils.NewString(region.Id), "region_id") - extra.Add(jsonutils.NewString(region.Name), "region") - } - } extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") if self.PendingDeleted { @@ -924,6 +914,28 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") } + return self.moreExtraInfo(extra) +} + +func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + zone := self.getZone() + if zone != nil { + extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") + extra.Add(jsonutils.NewString(zone.GetName()), "zone") + if len(zone.ExternalId) > 0 { + extra.Add(jsonutils.NewString(zone.ExternalId), "zone_external_id") + } + + region := zone.GetRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.Id), "region_id") + extra.Add(jsonutils.NewString(region.Name), "region") + + if len(region.ExternalId) > 0 { + extra.Add(jsonutils.NewString(region.ExternalId), "region_external_id") + } + } + } return extra } @@ -953,18 +965,7 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token } extra.Add(jsonutils.NewString(self.getAdminSecurityRules()), "admin_security_rules") } - zone := self.getZone() - if zone != nil { - extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") - extra.Add(jsonutils.NewString(zone.GetName()), "zone") - - region := zone.GetRegion() - if region != nil { - extra.Add(jsonutils.NewString(region.Id), "region_id") - extra.Add(jsonutils.NewString(region.Name), "region") - } - } - return extra + return self.moreExtraInfo(extra) } func (self *SGuest) getNetworksDetails() string { From 8ead2a64711c01c2fb56b9443337bbae8cef47a4 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 20:20:00 +0800 Subject: [PATCH 35/71] minor fixes --- pkg/compute/models/guests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 2297cba2ca..8ecba69d47 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -928,7 +928,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T extra.Add(jsonutils.NewString(self.getKeypairName()), "keypair") extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw") - extra.Add(jsonutils.NewString(self.getSecgroupName()), "secgroup") + extra.Add(jsonutils.NewString(self.GetSecgroupName()), "secgroup") if self.PendingDeleted { pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) From b23562b019cc80c5d8ae1862f60c41429f612ad5 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 20:39:10 +0800 Subject: [PATCH 36/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E9=98=BF=E9=87=8C=E4=BA=91VPC=E6=97=B6=E7=94=B1?= =?UTF-8?q?=E4=BA=8E=E6=9C=89securitygroup=E5=AF=BC=E8=87=B4=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/aliyun/securitygroup.go | 12 ++++++++++++ pkg/util/aliyun/vpc.go | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 85d9518b32..969bce79c8 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -189,3 +189,15 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { } return secId, nil } + +func (self *SRegion) deleteSecurityGroup(secGrpId string) error { + params := make(map[string]string) + params["SecurityGroupId"] = secGrpId + + _, err := self.ecsRequest("DeleteSecurityGroup", params) + if err != nil { + log.Errorf("Delete security group fail %s", err) + return err + } + return nil +} diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index d61028a555..196aaddb42 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/log" ) const ( @@ -178,5 +179,17 @@ func (self *SVpc) GetManagerId() string { } func (self *SVpc) Delete() error { + err := self.fetchSecurityGroups() + if err != nil { + log.Errorf("fetchSecurityGroup for VPC delete fail %s", err) + return err + } + for i := 0; i < len(self.secgroups); i += 1 { + err := self.region.deleteSecurityGroup(self.secgroups[i].SecurityGroupId) + if err != nil { + log.Errorf("deleteSecurityGroup for VPC delete fail %s", err) + return err + } + } return self.region.DeleteVpc(self.VpcId) } From 644e19e23f8fde1e758f54541f5cd549b1ed33fc Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 11:47:46 +0800 Subject: [PATCH 37/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Aclient=20newse?= =?UTF-8?q?ssion=E6=97=B6=E5=80=99=EF=BC=8C=E5=A6=82=E6=9E=9Ctoken?= =?UTF-8?q?=E6=B2=A1=E6=9C=89service=20catalog=EF=BC=8C=E5=AF=BC=E8=87=B4f?= =?UTF-8?q?atal=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/mcclient/mcclient.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 45ee26bc1c..b537ae1a40 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -311,10 +311,12 @@ func (this *Client) SetProject(tenantId, tenantName string, token TokenCredentia func (this *Client) NewSession(region, zone, endpointType string, token TokenCredential, apiVersion string) *ClientSession { cata := token.GetServiceCatalog() - if cata == nil { - log.Fatalf("Missing service catalog in token") + if this.serviceCatalog == nil { + if cata == nil { + log.Fatalf("Missing service catalog in token") + } + this.serviceCatalog = cata } - this.serviceCatalog = cata return &ClientSession{client: this, region: region, zone: zone, endpointType: endpointType, token: token, apiVersion: apiVersion, From cc63b9ff9374e1b3fcd97e771c1c26fb0decfd86 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 14:38:56 +0800 Subject: [PATCH 38/71] =?UTF-8?q?=E6=9B=B4=E6=96=B0Makefile=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0make=20dep=E5=91=BD=E4=BB=A4=EF=BC=8C?= =?UTF-8?q?=E5=8F=AA=E6=9B=B4=E6=96=B0vendor/yunion.io/x/=E4=B8=8B?= =?UTF-8?q?=E7=9A=84=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 228037d447..5984634fd0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell pwd) +ROOT_DIR := $(CURDIR) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh @@ -95,5 +95,8 @@ fmt: find . -type f -name "*.go" -not -path "./_output/*" \ -not -path "./vendor/*" | xargs gofmt -s -w +dep: + cd $(ROOT_DIR) && dep ensure -v -update $(shell for p in $$(ls vendor/yunion.io/x/); do echo "yunion.io/x/$$p"; done | xargs) + %: @: From f539aa53d8400a77f65e269a9eb0c7eef4ebe180 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 14:39:40 +0800 Subject: [PATCH 39/71] Update vendor & Gopkg.lock --- Gopkg.lock | 16 ++++++++-------- vendor/yunion.io/x/sqlchemy/field_update.go | 2 +- vendor/yunion.io/x/sqlchemy/insert.go | 2 +- vendor/yunion.io/x/sqlchemy/update.go | 2 +- vendor/yunion.io/x/structarg/structarg.go | 3 +++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2499f89c7f..aba3be928b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1026,11 +1026,11 @@ [[projects]] branch = "master" - digest = "1:5ede93047a3e04f6ede8fbc011f13ee94c397b727613fd9712cc99ebc47ff906" + digest = "1:d0257638bb52243f9fa293ef07544081259d54edd8e53f084e522b6c22e92637" name = "yunion.io/x/jsonutils" packages = ["."] pruneopts = "UT" - revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" + revision = "38477c9cceb895816fe21507d73da30376d358b7" [[projects]] branch = "master" @@ -1045,7 +1045,7 @@ [[projects]] branch = "master" - digest = "1:13da776b435ec7873c6f9309191296d920a110145372d2a9e1a58ab9e8b34b0d" + digest = "1:47670c70d0326f624de21c8f29d0f73850062f0752f27e146308db62fbb540d3" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1080,23 +1080,23 @@ "utils", ] pruneopts = "UT" - revision = "56b426f0ca15288cc26442f5802c949f137ade60" + revision = "6c349ecbdb90d5f6ecc9b86596428176c1937b32" [[projects]] branch = "master" - digest = "1:3d3e1ecb41c63448df5d66eeafa5f6da720ca6e32543e89faa82b5bc4b9adebc" + digest = "1:e23019db2cda58480528738c86bc7abbff09d10488c2a51dd5d500f8686815f0" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "a74ef73e555a1ed2e19b1a47e0567814c289eaf9" + revision = "19a3c90524d3638f8e3580231396dda46088aebc" [[projects]] branch = "master" - digest = "1:6baf7b4ee14e156a9dd6c5c850bb8201af47665e3267e3cdee26edf2a90eaf73" + digest = "1:71e1b62648868a9083f9e4693145a0154a600591ca548aaf64cae32a10e956e5" name = "yunion.io/x/structarg" packages = ["."] pruneopts = "UT" - revision = "d5e5d87357b9bc2164215117763f6b15b2a3e75d" + revision = "4a5eb8e2cdfbf7f6511561a3c710c116b171f405" [solve-meta] analyzer-name = "dep" diff --git a/vendor/yunion.io/x/sqlchemy/field_update.go b/vendor/yunion.io/x/sqlchemy/field_update.go index 2087dbb6c0..b6b32203a5 100644 --- a/vendor/yunion.io/x/sqlchemy/field_update.go +++ b/vendor/yunion.io/x/sqlchemy/field_update.go @@ -102,7 +102,7 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index d9f10406c1..c7442588f0 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -36,7 +36,7 @@ func (t *STableSpec) insertSqlPrep(dataFields map[string]interface{}) (string, [ if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) - format = append(format, "NOW()") + format = append(format, "UTC_TIMESTAMP()") } else if ov != nil && !c.IsZero(ov) && !isAutoInc { v := c.ConvertFromValue(ov) values = append(values, v) diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index d99ef0b95e..e40e0f6eff 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -126,7 +126,7 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, er buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 9eb223f272..7f7f6c6414 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -305,6 +305,9 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) return fmt.Errorf("positional %s must not have default value", token) } } + if !positional && use_default && required { + return fmt.Errorf("non-positional argument with default value should not have required:true set") + } subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) if err != nil { subcommand = false From 038d1c731ba14390ec319b4c28605afef98f1b59 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 24 Aug 2018 15:08:13 +0800 Subject: [PATCH 40/71] bugfix --- pkg/mcclient/modules/mod_domains.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/mcclient/modules/mod_domains.go b/pkg/mcclient/modules/mod_domains.go index 84d57ffb34..e7be85cd5e 100644 --- a/pkg/mcclient/modules/mod_domains.go +++ b/pkg/mcclient/modules/mod_domains.go @@ -240,12 +240,7 @@ func (this *DomainManager) DoDomainConfigDelete(s *mcclient.ClientSession, param return ret, httperrors.NewResourceNotFoundError("找不到该认证域") } - driver, err := detail.GetString("driver") - if err != nil { - log.Errorf("got driver from domain detail error: %v", err) - return ret, httperrors.NewInternalServerError("服务器错误,获取认证协议失败,不允许删除") - } - + driver, _ := detail.GetString("driver") if driver != "ldap" { if result, err := UsersV3.List(s, params); err != nil { log.Errorf("user list got error: %v", err) From 58aecb256d43b765f69d6060236cb5268577827b Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:15:13 +0800 Subject: [PATCH 41/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E4=B8=BB=E6=9C=BA=E6=97=B6=E5=8F=AA=E9=80=89=E6=8B=A9?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E4=B8=BA=20available=20=E7=9A=84=E7=BD=91?= =?UTF-8?q?=E7=BB=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/wires.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 22fa6d9a6f..d522c53e58 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -347,6 +347,7 @@ func (self *SWire) getNetworks() ([]SNetwork, error) { func (self *SWire) getGatewayNetworkQuery() *sqlchemy.SQuery { q := self.getNetworkQuery() q = q.IsNotNull("guest_gateway").IsNotEmpty("guest_gateway") + q = q.Equals("status", NETWORK_STATUS_AVAILABLE) return q } From 931ee5cef64fe6407f8067739f449a62fa96ca7e Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:30:56 +0800 Subject: [PATCH 42/71] =?UTF-8?q?=E5=A2=9E=E5=8A=A0VM=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/aliyun/instance.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 292a231193..5d90dedd32 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -422,6 +422,7 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout func (self *SRegion) StartVM(instanceId string) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusStopped { + log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doStartVM(instanceId) @@ -434,6 +435,7 @@ func (self *SRegion) StartVM(instanceId string) error { func (self *SRegion) StopVM(instanceId string, isForce bool) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusRunning { + log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doStopVM(instanceId, isForce) @@ -451,6 +453,7 @@ func (self *SRegion) DeleteVM(instanceId string) error { return err } } else if status != InstanceStatusStopped { + log.Errorf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doDeleteVM(instanceId) From 32d56101dfe81f42f44d29e585422512c50f230f Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:32:08 +0800 Subject: [PATCH 43/71] minor updates --- pkg/util/aliyun/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 5d90dedd32..12cc379e68 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -435,7 +435,7 @@ func (self *SRegion) StartVM(instanceId string) error { func (self *SRegion) StopVM(instanceId string, isForce bool) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusRunning { - log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusStopped) + log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning) return cloudprovider.ErrInvalidStatus } return self.doStopVM(instanceId, isForce) From f23c0257cd613b25beac60b9bb6036b60d5c57f8 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:10:32 +0000 Subject: [PATCH 44/71] dnsrecords: do not log NotFound as error --- pkg/compute/models/dnsrecords.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/compute/models/dnsrecords.go b/pkg/compute/models/dnsrecords.go index a1177a3401..5da570532c 100644 --- a/pkg/compute/models/dnsrecords.go +++ b/pkg/compute/models/dnsrecords.go @@ -6,7 +6,6 @@ import ( "strings" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" @@ -205,7 +204,6 @@ func (man *SDnsRecordManager) QueryDns(projectId, name string) *SDnsRecord { err := q.First(rec) if err != nil { - log.Errorf("QueryDns %q fail: %s", name, err) return nil } return rec From 745f97e912eda8811b05892f81e637ce34a9cdb2 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:11:16 +0000 Subject: [PATCH 45/71] guestnetworks: GetGuestByAddress(): optimize by using a single join query --- pkg/compute/models/guestnetworks.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..2f3defdf37 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -263,17 +263,17 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { - gn := SGuestnetwork{} - gn.SetModelManager(GuestnetworkManager) - - err := manager.Query().Equals("ip_addr", address).First(&gn) - if err != nil { - log.Errorf("GetGuestByAddress query fail: %s", err) - return nil - } - guest, _ := GuestManager.FetchById(gn.GuestId) - if guest != nil { - return guest.(*SGuest) + networks := manager.Query().SubQuery() + guests := GuestManager.Query() + q := guests.Join(networks, sqlchemy.AND( + sqlchemy.Equals(networks.Field("ip_addr"), address), + sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")), + )) + guest := &SGuest{} + guest.SetModelManager(GuestManager) + err := q.First(guest) + if err == nil { + return guest } return nil } From 96ceaafd2ff27a544d11b3960a18849e250b2185 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:30:12 +0000 Subject: [PATCH 46/71] hostnetworks: GetHostNetworkByAddress(): fix logic invert --- pkg/compute/models/hostnetworks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index abf31b3ed6..09875cce28 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -92,7 +92,7 @@ func (man *SHostnetworkManager) QueryByAddress(addr string) *sqlchemy.SQuery { func (man *SHostnetworkManager) GetHostNetworkByAddress(addr string) *SHostnetwork { network := SHostnetwork{} err := man.QueryByAddress(addr).First(&network) - if err != nil { + if err == nil { return &network } return nil From 54ee08ef494a81c4876a178436a4c0d3ce17a3da Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:33:56 +0000 Subject: [PATCH 47/71] hostnetworks: GetHostByAddress(): use a single join --- pkg/compute/models/hostnetworks.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index 09875cce28..88433fb90a 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -107,9 +107,17 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork { } func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost { - net := man.GetHostNetworkByAddress(addr) - if net == nil { - return nil + networks := man.Query().SubQuery() + hosts := HostManager.Query() + q := hosts.Join(networks, sqlchemy.AND( + sqlchemy.Equals(networks.Field("ip_addr"), addr), + sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")), + )) + host := &SHost{} + host.SetModelManager(HostManager) + err := q.First(host) + if err == nil { + return host } - return net.GetHost() + return nil } From 7679056f550eb14d466e353c838f41fcd831d708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 24 Aug 2018 20:58:54 +0800 Subject: [PATCH 48/71] =?UTF-8?q?=E6=B7=BB=E5=8A=A0storage=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 4ac8ea7587..7990188dd8 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/sysutils" "yunion.io/x/sqlchemy" ) @@ -654,6 +655,23 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. q = q.Filter(sqlchemy.In(q.Field("zone_id"), sq.SubQuery())) } + if jsonutils.QueryBoolean(query, "share", false) { + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "local", false) { + q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "usable", false) { + hostStorage := HoststorageManager.Query().SubQuery() + q = q.Join(hostStorage, sqlchemy.AND( + sqlchemy.Equals(hostStorage.Field("storage_id"), q.Field("id")), + sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE}), + sqlchemy.IsTrue(q.Field("enabled")), + )) + } + managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) if len(managerStr) > 0 { provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) From 8df37525c033bdcc59c914d04c833c3d9fc8168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 24 Aug 2018 20:31:20 +0800 Subject: [PATCH 49/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9B=A0=E8=BE=93?= =?UTF-8?q?=E5=85=A5disk=20name=E5=AF=BC=E8=87=B4=E6=89=BE=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E7=A3=81=E7=9B=98,=E5=8D=B8=E8=BD=BD=E7=A3=81?= =?UTF-8?q?=E7=9B=98=E5=A4=B1=E8=B4=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8ecba69d47..fe551cf561 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2337,7 +2337,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok disk.SetStatus(userCred, DISK_DETACHING, "") } taskData := jsonutils.NewDict() - taskData.Add(jsonutils.NewString(diskId), "disk_id") + taskData.Add(jsonutils.NewString(disk.Id), "disk_id") taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk") self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "") return nil, nil From 5de6220a745d025ab27c80734b478dd4ff35f567 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 24 Aug 2018 22:25:33 +0800 Subject: [PATCH 50/71] server shot desc --- pkg/compute/models/baremetalstatus.go | 22 +++++ pkg/compute/models/disks.go | 21 +++++ pkg/compute/models/guests.go | 125 +++++++++++++++++++++++++ pkg/compute/models/hosts.go | 83 ++++++++++++++++ pkg/compute/models/isolated_devices.go | 18 ++++ 5 files changed, 269 insertions(+) create mode 100644 pkg/compute/models/baremetalstatus.go diff --git a/pkg/compute/models/baremetalstatus.go b/pkg/compute/models/baremetalstatus.go new file mode 100644 index 0000000000..5c4561f14c --- /dev/null +++ b/pkg/compute/models/baremetalstatus.go @@ -0,0 +1,22 @@ +package models + +const ( + BAREMETAL_INIT = "init" + BAREMETAL_PREPARE = "prepare" + BAREMETAL_PREPARE_FAIL = "prepare_fail" + BAREMETAL_READY = "ready" + BAREMETAL_RUNNING = "running" + BAREMETAL_MAINTAINING = "maintaining" + BAREMETAL_START_MAINTAIN = "start_maintain" + BAREMETAL_DELETING = "deleting" + BAREMETAL_DELETE = "delete" + BAREMETAL_DELETE_FAIL = "delete_fail" + BAREMETAL_UNKNOWN = "unknown" + BAREMETAL_SYNCING_STATUS = "syncing_status" + BAREMETAL_SYNC = "sync" + BAREMETAL_SYNC_FAIL = "sync_fail" + BAREMETAL_START_CONVERT = "start_convert" + BAREMETAL_CONVERTING = "converting" + BAREMETAL_START_FAIL = "start_fail" + BAREMETAL_STOP_FAIL = "stop_fail" +) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index a2e827b7b9..94dd5972d1 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -701,6 +701,27 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info s return image.Id, nil } +// def get_disk_spec_v2(conf): +// def _get_spec(storages): +// spec = {} +// for adapter, ss in group_by_adapter(storages).items(): +// if len(ss) == 0: +// continue +// spec[adapter] = get_disk_spec(ss) +// return spec + +// spec = {} +// for driver in DISK_DRIVERS: +// storages = [s for s in conf if s['driver'] == driver] +// if len(storages) != 0: +// spec[driver] = _get_spec(storages) +// return spec + +func GetDiskSpecV2(storageInfo jsonutils.JSONObject) *jsonutils.JSONDict { + // ToDo + return nil +} + func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8ecba69d47..aef8301dc4 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3027,6 +3027,131 @@ func (manager *SGuestManager) FetchGuestById(guestId string) *SGuest { return guest.(*SGuest) } +func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if !utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + + // get disk spec + guestdisks := self.GetDisks() + diskSpecs := jsonutils.NewArray() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + diskSpec := jsonutils.NewDict() + diskSpec.Set("size", jsonutils.NewInt(int64(disk.DiskSize))) + s := disk.GetStorage() + diskSpec.Set("backend", jsonutils.NewString(s.StorageType)) + diskSpec.Set("medium_type", jsonutils.NewString(s.MediumType)) + diskSpecs.Add(diskSpec) + } + spec.Set("disk", diskSpecs) + + // get nic spec + guestnics := self.GetNetworks() + nicSpecs := jsonutils.NewArray() + for _, guestnic := range guestnics { + nicSpec := jsonutils.NewDict() + nicSpec.Set("bandwidth", jsonutils.NewInt(int64(guestnic.getBandwidth()))) + t := "int" + if guestnic.IsExit() { + t = "ext" + } + nicSpec.Set("type", jsonutils.NewString(t)) + nicSpecs.Add(nicSpec) + } + spec.Set("nic", nicSpecs) + + // get isolate device spec + guestgpus := self.GetIsolatedDevices() + gpuSpecs := jsonutils.NewArray() + for _, guestgpu := range guestgpus { + if strings.HasPrefix(guestgpu.DevType, "GPU") { + gs := guestgpu.GetSpec(false) + if gs != nil { + gpuSpecs.Add(gs) + } + } + } + spec.Set("gpu", gpuSpecs) + return spec +} + +func (self *SGuest) GetTemplateId() string { + guestdisks := self.GetDisks() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + if disk != nil { + templateId := disk.GetTemplateId() + if len(templateId) > 0 { + return templateId + } + } + } + return "" +} + +// def get_short_desc(self, user_cred=None): +// desc = super(Guests, self).get_short_desc() +// desc['mem'] = self.vmem_size +// desc['cpu'] = self.vcpu_count +// template_id = self.get_template_id() +// if template_id is not None: +// desc['template_id'] = template_id +// ext_bw, int_bw = self.get_bandwidth() +// if ext_bw > 0: +// desc['ext_bandwidth'] = ext_bw +// if int_bw > 0: +// desc['int_bandwidth'] = int_bw +// hypervisor = self.get_hypervisor() +// desc['hypervisor'] = hypervisor +// spec = self.get_spec(user_cred, status_check=False) +// if hypervisor == self.HYPERVISOR_BAREMETAL: +// host = self.get_host() +// if host is not None: +// host_spec = '/'.join(host.get_spec_ident( +// host.get_spec(user_cred, status_check=False))) +// spec['host_spec'] = host_spec +// if spec is not None: +// desc.update(spec) +// return desc + +func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { + desc := self.SStandaloneResourceBase.GetShortDesc() + desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + desc.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + templateId := self.GetTemplateId() + if len(templateId) > 0 { + desc.Set("cpu", jsonutils.NewString(templateId)) + } + extBw := self.getBandwidth(true) + intBw := self.getBandwidth(false) + if extBw > 0 { + desc.Set("ext_bandwidth", jsonutils.NewInt(int64(extBw))) + } + if intBw > 0 { + desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw))) + } + desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) + spec := self.GetSpec(false) + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + host := self.GetHost() + if host != nil { + hostSpec := host.GetSpec(false) + hostSpecIdent := host.GetSpecIdent(hostSpec) + spec.Set("host_spec", jsonutils.NewString(strings.Join(hostSpecIdent, "/"))) + } + } + if spec != nil { + desc.Update(spec) + } + return desc +} + func (self *SGuest) saveOsType(osType string) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.OsType = osType diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 6b47c4e370..97ccecb6ef 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -506,6 +506,89 @@ func (self *SHost) ClearSchedDescCache() error { return HostManager.ClearSchedDescCache(self.Id) } +// def get_spec(self, user_cred, status_check=True, **kwargs): +// if status_check: +// if self.status in [baremetalstatus.INIT, baremetalstatus.PREPARE_FAIL, +// baremetalstatus.PREPARE] or \ +// self.get_baremetal_server() is not None: +// return None +// if self.mem_size is None or self.cpu_count is None: +// return None +// from clouds.baremetal import nictypes +// spec = self.get_hardware_specification() +// spec.pop('storage_info', None) +// net_info = [] +// for netif in self.get_netifs(): +// net_desc = netif.get_baremetal_json_desc() +// if net_desc.get('nic_type', None) != nictypes.NIC_TYPE_IPMI: +// net_info.append(net_desc) +// spec['nic_count'] = len(net_info) +// spec['manufacture'] = self.sys_info.get('manufacture', 'Unknown') +// spec['model'] = self.sys_info.get('model', 'Unknown') +// return spec +// def get_hardware_specification(self): +// spec = {} +// spec['cpu'] = self.cpu_count +// spec['mem'] = self.mem_size +// if self.storage_info: +// from clouds.baremetal import diskconfig +// spec['disk'] = diskconfig.get_disk_spec_v2(self.storage_info) +// spec['driver'] = self.storage_driver +// spec['storage_info'] = self.storage_info +// return spec + +func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { + if statusCheck { + if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) || + self.getBaremetalServer() != nil { + return nil + } + if self.MemSize == 0 || self.CpuCount == 0 { + return nil + } + } + spec := self.GetHardwareSpecification() + spec.Remove("storage_info") + netInfo := jsonutils.NewArray() + nifs := self.GetNetInterfaces() + for _, nif := range nifs { + netDesc := nif.getBaremetalJsonDesc() + nicType, err := netDesc.GetString("nic_type") + if err != nil && nicType != NIC_TYPE_IPMI { + netInfo.Add(netDesc) + } + } + spec.Set("nic_count", jsonutils.NewInt(int64(netInfo.Length()))) + manufacture, err := self.SysInfo.Get("manufacture") + if err != nil { + manufacture = jsonutils.NewString("Unknown") + } + spec.Set("manufacture", manufacture) + model, err := self.SysInfo.Get("model") + if err != nil { + model = jsonutils.NewString("Unknown") + } + spec.Set("model", model) + return nil +} + +func (self *SHost) GetSpecIdent(spec *jsonutils.JSONDict) []string { + // Todo + return []string{} +} + +func (self *SHost) GetHardwareSpecification() *jsonutils.JSONDict { + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.CpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.MemSize))) + if self.StorageInfo != nil { + spec.Set("disk", GetDiskSpecV2(self.StorageInfo)) + spec.Set("driver", jsonutils.NewString(self.StorageDriver)) + spec.Set("storage_info", self.StorageInfo) + } + return spec +} + type SStorageCapacity struct { Capacity int Used int diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index f3182d3f67..d1c7c26c21 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -380,6 +380,24 @@ func (self *SIsolatedDevice) getDesc() *jsonutils.JSONDict { return desc } +func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if len(self.GuestId) > 0 { + return nil + } + host := self.getHost() + if host.Status != BAREMETAL_RUNNING || !host.Enabled { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("dev_type", jsonutils.NewString(self.DevType)) + spec.Set("model", jsonutils.NewString(self.Model)) + spec.Set("pci_id", jsonutils.NewString(self.VendorDeviceId)) + spec.Set("vendor", jsonutils.NewString(self.getVendor())) + return spec +} + func (self *SIsolatedDevice) GetShortDesc() *jsonutils.JSONDict { desc := self.getDesc() desc.Add(jsonutils.NewString(self.Keyword()), "res_name") From d113017c338c3a0b7bf68960e624aa4d45b5f606 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 24 Aug 2018 22:41:24 +0800 Subject: [PATCH 51/71] remove temp code --- pkg/compute/models/guests.go | 25 ------------------------- pkg/compute/models/hosts.go | 31 ------------------------------- 2 files changed, 56 deletions(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index aef8301dc4..713f5e88c9 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3095,31 +3095,6 @@ func (self *SGuest) GetTemplateId() string { return "" } -// def get_short_desc(self, user_cred=None): -// desc = super(Guests, self).get_short_desc() -// desc['mem'] = self.vmem_size -// desc['cpu'] = self.vcpu_count -// template_id = self.get_template_id() -// if template_id is not None: -// desc['template_id'] = template_id -// ext_bw, int_bw = self.get_bandwidth() -// if ext_bw > 0: -// desc['ext_bandwidth'] = ext_bw -// if int_bw > 0: -// desc['int_bandwidth'] = int_bw -// hypervisor = self.get_hypervisor() -// desc['hypervisor'] = hypervisor -// spec = self.get_spec(user_cred, status_check=False) -// if hypervisor == self.HYPERVISOR_BAREMETAL: -// host = self.get_host() -// if host is not None: -// host_spec = '/'.join(host.get_spec_ident( -// host.get_spec(user_cred, status_check=False))) -// spec['host_spec'] = host_spec -// if spec is not None: -// desc.update(spec) -// return desc - func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { desc := self.SStandaloneResourceBase.GetShortDesc() desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 97ccecb6ef..0874133ba1 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -506,37 +506,6 @@ func (self *SHost) ClearSchedDescCache() error { return HostManager.ClearSchedDescCache(self.Id) } -// def get_spec(self, user_cred, status_check=True, **kwargs): -// if status_check: -// if self.status in [baremetalstatus.INIT, baremetalstatus.PREPARE_FAIL, -// baremetalstatus.PREPARE] or \ -// self.get_baremetal_server() is not None: -// return None -// if self.mem_size is None or self.cpu_count is None: -// return None -// from clouds.baremetal import nictypes -// spec = self.get_hardware_specification() -// spec.pop('storage_info', None) -// net_info = [] -// for netif in self.get_netifs(): -// net_desc = netif.get_baremetal_json_desc() -// if net_desc.get('nic_type', None) != nictypes.NIC_TYPE_IPMI: -// net_info.append(net_desc) -// spec['nic_count'] = len(net_info) -// spec['manufacture'] = self.sys_info.get('manufacture', 'Unknown') -// spec['model'] = self.sys_info.get('model', 'Unknown') -// return spec -// def get_hardware_specification(self): -// spec = {} -// spec['cpu'] = self.cpu_count -// spec['mem'] = self.mem_size -// if self.storage_info: -// from clouds.baremetal import diskconfig -// spec['disk'] = diskconfig.get_disk_spec_v2(self.storage_info) -// spec['driver'] = self.storage_driver -// spec['storage_info'] = self.storage_info -// return spec - func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { if statusCheck { if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) || From b16613a0e6de1fcacf9c42e438133ca9d3b2f05f Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 09:24:05 +0800 Subject: [PATCH 52/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E9=98=BF?= =?UTF-8?q?=E9=87=8C=E4=BA=91=E4=B8=BB=E6=9C=BA=E5=88=9B=E5=BB=BA=E5=90=8E?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 9 +++++---- pkg/compute/tasks/guest_create_task.go | 4 ++-- pkg/compute/tasks/guest_start_task.go | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 50ebab0f8d..673c4f1b1b 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -125,10 +125,11 @@ type SDiskInfo struct { func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) - onfinish, err := config.GetString("on_finish") + /* onfinish, err := config.GetString("on_finish") if err != nil { return err - } + } */ + action, err := config.GetString("action") if err != nil { return err @@ -170,12 +171,12 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil, err } - if onfinish == "none" { + /* if onfinish == "none" { err = iVM.StartVM() if err != nil { return nil, err } - } + } */ encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) if err != nil { diff --git a/pkg/compute/tasks/guest_create_task.go b/pkg/compute/tasks/guest_create_task.go index a57f49afca..1248fe287e 100644 --- a/pkg/compute/tasks/guest_create_task.go +++ b/pkg/compute/tasks/guest_create_task.go @@ -105,7 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String()) } -func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { +/*func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) } @@ -113,4 +113,4 @@ func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStand func (self *GuestCreateTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) -} +}*/ diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 3d910543a6..8be672f918 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -72,6 +72,7 @@ func (self *GuestStartTask) OnStartCompleteFailed(ctx context.Context, obj db.IS } func (self *GuestStartTask) onStartGuestFailed(ctx context.Context, guest *models.SGuest, err error) { + guest.SetStatus(self.UserCred, models.VM_START_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(err.Error())) } From ed2bf2eba5817476dbccbf17d68cb18a454a2804 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:11:45 +0000 Subject: [PATCH 53/71] dns: policy, log, db query optimization Fixes CLOUD-451 TODO - timeout bound for k8s query - k8s client pools --- pkg/dns/dns.go | 161 +++++++++++++++++++++------------------------ pkg/dns/parse.go | 49 +++++++------- pkg/dns/reverse.go | 12 +--- pkg/dns/setup.go | 9 +-- 4 files changed, 107 insertions(+), 124 deletions(-) diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index b60ab73c51..9485a2557f 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -18,25 +18,27 @@ import ( "github.com/mholt/caddy" "github.com/miekg/dns" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ylog "yunion.io/x/log" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/k8s" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" ) const ( PluginName string = "yunion" // defaultTTL to apply to all answers - defaultTTL = 10 + defaultTTL = 10 + defaultDbMaxOpenConn = 32 + defaultDbMaxIdleConn = 32 ) var ( @@ -74,15 +76,17 @@ func (r *SRegionDNS) initDB(c *caddy.Controller) error { if err != nil { return err } - dbConn, err := sql.Open(dialect, sqlStr) + sqlDb, err := sql.Open(dialect, sqlStr) if err != nil { return err } - sqlchemy.SetDB(dbConn) + sqlDb.SetMaxOpenConns(defaultDbMaxOpenConn) + sqlDb.SetMaxIdleConns(defaultDbMaxIdleConn) + sqlchemy.SetDB(sqlDb) db.InitAllManagers() c.OnShutdown(func() error { - r.CloseDB() + sqlchemy.CloseDB() return nil }) return nil @@ -103,10 +107,6 @@ func (r *SRegionDNS) initK8s(c *caddy.Controller) { ylog.Infof("Init k8s client success, %d pods in the cluster", len(pods.Items)) } -func (r *SRegionDNS) CloseDB() { - sqlchemy.CloseDB() -} - func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *dns.Msg) (int, error) { var ( records []dns.RR @@ -116,36 +116,26 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d opt := plugin.Options{} state := request.Request{W: w, Req: rmsg, Context: ctx} - zone := plugin.Zones(r.Zones).Matches(state.Name()) - switch state.QType() { case dns.TypeA: - ylog.Debugf("A question: %#v", state) records, err = plugin.A(r, zone, state, nil, opt) case dns.TypeAAAA: - ylog.Debugf("AAAA question: %#v", state) + // TODO fallthrough to next records, err = plugin.AAAA(r, zone, state, nil, opt) case dns.TypeTXT: - ylog.Debugf("TXT question: %#v", state) records, err = plugin.TXT(r, zone, state, opt) case dns.TypeCNAME: - ylog.Debugf("CNAME question: %#v", state) records, err = plugin.CNAME(r, zone, state, opt) case dns.TypePTR: - ylog.Debugf("PTR question: %#v", state) records, err = plugin.PTR(r, zone, state, opt) case dns.TypeMX: - ylog.Debugf("MX question: %#v", state) records, extra, err = plugin.MX(r, zone, state, opt) case dns.TypeSRV: - ylog.Debugf("SRV question: %#v", state) records, extra, err = plugin.SRV(r, zone, state, opt) case dns.TypeSOA: - ylog.Debugf("SOA question: %#v", state) records, err = plugin.SOA(r, zone, state, opt) case dns.TypeNS: - ylog.Debugf("NS question: %#v", state) if state.Name() == zone { records, extra, err = plugin.NS(r, zone, state, opt) break @@ -157,18 +147,19 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d _, err = plugin.A(r, zone, state, nil, opt) } - if r.IsNameError(err) { + if err == errCallNext { if r.Fall.Through(state.Name()) { return plugin.NextOrFailure(r.Name(), r.Next, ctx, w, rmsg) } return plugin.BackendError(r, zone, dns.RcodeNameError, state, nil /* err */, opt) - } - if err != nil { - return plugin.BackendError(r, zone, dns.RcodeServerFailure, state, err, opt) + } else if err == errRefused { + return plugin.BackendError(r, zone, dns.RcodeRefused, state, err, opt) + } else if err == errNotFound { + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } if len(records) == 0 { - return plugin.BackendError(r, zone, dns.RcodeSuccess, state, err, opt) + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } m := new(dns.Msg) @@ -184,13 +175,14 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d } var ( - errNoItems = errors.New("no items found") + errRefused = errors.New("refused the query") + errNotFound = errors.New("not found") + errCallNext = errors.New("continue to next") ) // Services implements the ServiceBackend interface func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) { switch state.QType() { - case dns.TypeTXT: t, _ := dnsutil.TrimZone(state.Name(), state.Zone) @@ -203,7 +195,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } svc := msg.Service{Text: "0.0.1", TTL: 28800, Key: msg.Path(state.QName(), "coredns")} return []msg.Service{svc}, nil - case dns.TypeNS: ns := r.nsAddr() svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")} @@ -219,7 +210,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } services, err = r.Records(state, false) - ylog.Debugf("Get records: %#v, error: %v", services, err) return } @@ -230,7 +220,7 @@ func (r *SRegionDNS) Lookup(state request.Request, name string, typ uint16) (*dn // IsNameError implements the ServiceBackend interface func (r *SRegionDNS) IsNameError(err error) bool { - return err == errNoItems + return err == errCallNext } // Records looks up records in region mysql @@ -242,25 +232,22 @@ func (r *SRegionDNS) Records(state request.Request, exact bool) ([]msg.Service, return r.findRecords(req) } -func (r *SRegionDNS) getHostIpWithName(req *recordRequest) []string { +func (r *SRegionDNS) getHostIpWithName(req *recordRequest) string { name := req.QueryName() host, _ := models.HostManager.FetchByName("", name) if host == nil { - return nil + return "" } ip := host.(*models.SHost).AccessIp - if len(ip) == 0 { - return nil - } - return []string{ip} + return ip } func (r *SRegionDNS) getGuestIpWithName(req *recordRequest) []string { ips := []string{} name := req.QueryName() projectId := req.ProjectId() - isExitOnly := req.IsExitOnly() - ips = models.GuestManager.GetIpInProjectWithName(projectId, name, isExitOnly) + wantOnlyExit := false + ips = models.GuestManager.GetIpInProjectWithName(projectId, name, wantOnlyExit) return ips } @@ -268,6 +255,9 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) queryInfo := req.GetK8sQueryInfo() pods, err := r.getK8sServicePods(queryInfo.Namespace, queryInfo.ServiceName) if err != nil { + if k8serrors.IsNotFound(err) { + err = nil + } return nil, err } ips := make([]string, 0) @@ -301,10 +291,9 @@ func (r *SRegionDNS) Name() string { return PluginName } -func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service) { ips := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type()) if len(ips) == 0 { - err = errNoItems return } @@ -317,12 +306,12 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic if req.IsSRV() { parts := strings.SplitN(ip.Addr, ":", 2) if len(parts) != 2 { - err = fmt.Errorf("Invalid SRV records: %q", ip.Addr) + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } port, e := strconv.Atoi(parts[1]) if e != nil { - err = e + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } s = msg.Service{Host: parts[0], Port: port, TTL: ttl} @@ -334,17 +323,6 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic return } -func (r *SRegionDNS) IsCloudNetworkIp(req *recordRequest) bool { - if req.network != nil { - return true - } - return false -} - -func (r *SRegionDNS) IsK8sClientReady() bool { - return r.K8sClient != nil -} - func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)} zone := plugin.Zones(zones).Matches(req.state.Name()) @@ -354,54 +332,65 @@ func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { return false } -func (r *SRegionDNS) findRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) findRecords(req *recordRequest) ([]msg.Service, error) { // 1. try local dns records table - recs, err = r.queryLocalDnsRecords(req) - if len(recs) != 0 { - return + rrs := r.queryLocalDnsRecords(req) + if len(rrs) > 0 { + return rrs, nil } + isPlainName := req.IsPlainName() isMyDomain := r.isMyDomain(req) - - isCloudIp := r.IsCloudNetworkIp(req) - - // 2. not my domain and src ip not in cloud network table - // query from upstream - if !isMyDomain && !isCloudIp { - err = errNoItems - return + if isPlainName { + isCloudIp := req.SrcInCloud() + if isCloudIp { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errRefused + } + } else if isMyDomain { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errCallNext } - - // 3. internal query - ips, err := r.findInternalRecordIps(req) - return ips2DnsRecords(ips), err } -func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) ([]string, error) { - // 1. try host table - ip := r.getHostIpWithName(req) - if len(ip) != 0 { - return ip, nil +func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) []string { + { + // 1. try host table + ip := r.getHostIpWithName(req) + if len(ip) > 0 { + return []string{ip} + } } - // 2. try guest table - ip = r.getGuestIpWithName(req) - if len(ip) != 0 { - return ip, nil + { + // 2. try guest table + ips := r.getGuestIpWithName(req) + if len(ips) > 0 { + return ips + } } - if !r.IsK8sClientReady() { + if r.K8sClient == nil { ylog.Warningf("K8s client not ready, skip it.") - return nil, errNoItems + return nil } // 3. try k8s service backends ips, err := r.getK8sServiceBackends(req) - if len(ips) != 0 { - return ips, nil - } if err != nil { ylog.Errorf("Get k8s service backends error: %v", err) } - return nil, errNoItems + return ips } func ips2DnsRecords(ips []string) []msg.Service { diff --git a/pkg/dns/parse.go b/pkg/dns/parse.go index 5cf836b646..568a773567 100644 --- a/pkg/dns/parse.go +++ b/pkg/dns/parse.go @@ -7,17 +7,16 @@ import ( "github.com/coredns/coredns/request" "github.com/miekg/dns" - "yunion.io/x/pkg/tristate" - "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/tristate" ) type recordRequest struct { - state request.Request - domainSegs []string - guest *models.SGuest - host *models.SHost - network *models.SNetwork + state request.Request + domainSegs []string + srcProjectId string + srcInCloud bool + network *models.SNetwork } func parseRequest(state request.Request) (r *recordRequest, err error) { @@ -28,9 +27,19 @@ func parseRequest(state request.Request) (r *recordRequest, err error) { domainSegs: segs, } srcIP := r.SrcIP4() - r.guest = models.GuestnetworkManager.GetGuestByAddress(srcIP) - r.host = models.HostnetworkManager.GetHostByAddress(srcIP) - r.network, _ = models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None) + // NOTE the check on networks_tbl is a hack, we should be more specific + // by querying only guest networks, host networks, and others like + // loadbalancer network the to come. + // + // Order matters here, we want to find the srcIP project as accurately + // as possible + if guest := models.GuestnetworkManager.GetGuestByAddress(srcIP); guest != nil { + r.srcProjectId = guest.ProjectId + r.srcInCloud = true + } else if network, _ := models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None); network != nil { + r.srcProjectId = network.ProjectId + r.srcInCloud = true + } return } @@ -41,6 +50,11 @@ func (r recordRequest) Name() string { return name } +func (r recordRequest) IsPlainName() bool { + nl := dns.CountLabel(r.Name()) + return nl == 1 +} + func (r recordRequest) QueryName() string { seps := strings.Split(r.Name(), ".") if len(seps) == 0 { @@ -63,20 +77,11 @@ func (r recordRequest) SrcIP4() string { } func (r recordRequest) ProjectId() string { - if r.guest != nil { - return r.guest.ProjectId - } - if r.network != nil { - return r.network.ProjectId - } - return "" + return r.srcProjectId } -func (r recordRequest) IsExitOnly() bool { - if r.guest == nil { - return false - } - return r.guest.IsExitOnly() +func (r recordRequest) SrcInCloud() bool { + return r.srcInCloud } type K8sQueryInfo struct { diff --git a/pkg/dns/reverse.go b/pkg/dns/reverse.go index e6cf7e2276..f6df515fdc 100644 --- a/pkg/dns/reverse.go +++ b/pkg/dns/reverse.go @@ -8,8 +8,6 @@ import ( "github.com/coredns/coredns/plugin/pkg/dnsutil" "github.com/coredns/coredns/request" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/models" ) @@ -21,13 +19,7 @@ func (r *SRegionDNS) Reverse(state request.Request, exact bool, opt plugin.Optio return nil, e } records, err := r.getNameForIp(ip, state) - if err != nil { - log.Errorf("Reverse get name for ip: %v", err) - } - if len(records) == 0 { - return records, errNoItems - } - return records, nil + return records, err } func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Service, error) { @@ -53,7 +45,7 @@ func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Servi if guest != nil { return []msg.Service{{Host: r.joinDomain(guest.Name), TTL: defaultTTL}}, nil } - return nil, errNoItems + return nil, errNotFound } func (r *SRegionDNS) joinDomain(name string) string { diff --git a/pkg/dns/setup.go b/pkg/dns/setup.go index 1a6a1ff673..77a9cb1c4b 100644 --- a/pkg/dns/setup.go +++ b/pkg/dns/setup.go @@ -2,7 +2,6 @@ package dns import ( "fmt" - "os" "github.com/coredns/coredns/core/dnsserver" "github.com/coredns/coredns/plugin" @@ -20,18 +19,16 @@ func init() { } func setup(c *caddy.Controller) error { - os.Stderr = os.Stdout - rDNS, err := regionDNSParse(c) if err != nil { return plugin.Error(PluginName, err) } - if rDNS.PrimaryZone == "" { - return fmt.Errorf("dns_domain must provided") + if len(rDNS.PrimaryZone) == 0 { + return fmt.Errorf("dns_domain missing") } if !regutils.MatchDomainName(rDNS.PrimaryZone) { - return fmt.Errorf("dns_domain %q not match domain format", rDNS.PrimaryZone) + return fmt.Errorf("dns_domain %q invalid", rDNS.PrimaryZone) } err = rDNS.initDB(c) From 5b12bdc7e75f769f078eaf2942132fd2e14bf1a0 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 10:12:43 +0000 Subject: [PATCH 54/71] dns: hack: new k8s client for each query - rest.Config Timeout setting does not work right now [1] - number of connections will not increase along with newed client: they use a single connection always... [1] https://github.com/kubernetes/client-go/issues/374#issuecomment-413842964 --- pkg/dns/dns.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 9485a2557f..248504b9f6 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -271,7 +271,11 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) } func (r *SRegionDNS) getK8sServicePods(namespace, name string) ([]v1.Pod, error) { - cli := r.K8sClient + cli, err := k8s.NewClientByFile(r.K8sConfigFile, nil) + if err != nil { + ylog.Errorf("Init kubernetes client error: %v", err) + return nil, err + } svc, err := cli.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { return nil, err From 96d52d1061bc8d573b154a97e217acc641429885 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 11:02:03 +0800 Subject: [PATCH 55/71] =?UTF-8?q?=E4=BF=AE=E6=AD=A3:=20task=20=E5=9B=9E?= =?UTF-8?q?=E8=B0=83=E4=B8=8D=E5=88=B0=20stage=20tailed=20=E7=9A=84?= =?UTF-8?q?=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/appsrv/dispatcher/dispatcher.go | 2 +- pkg/cloudcommon/db/taskman/tasks.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index 56cb359b66..89048c175f 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -203,7 +203,7 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht manager, params, query, body := fetchEnv(ctx, w, r) data, _ := body.Get(manager.KeywordPlural()) if data == nil { - data = jsonutils.NewDict() + data = body.(*jsonutils.JSONDict) } results, err := manager.PerformClassAction(ctx, params[""], query, data) if err != nil { diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..277032c453 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -296,7 +296,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, taskStatus, _ := data.GetString("__status__") if len(taskStatus) > 0 && taskStatus != "OK" { taskFailed = true - data, err = data.Get("reason") + data, err = data.Get("__reason__") if err != nil { data = jsonutils.NewString("Task failed due to unknown remote errors!") } From 43d8b74d175144ac086f6bf3849727b292a77726 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:00:18 +0000 Subject: [PATCH 56/71] dns: add a README.md for memo purposes --- pkg/dns/README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 pkg/dns/README.md diff --git a/pkg/dns/README.md b/pkg/dns/README.md new file mode 100644 index 0000000000..79bfeb66f3 --- /dev/null +++ b/pkg/dns/README.md @@ -0,0 +1,56 @@ +# 行为约定 + +A + +```sh +names='' +names="$names titan" #ok, guest PlainName +names="$names titan.hq.cloud.yunionyun.com" #ok, guest CloudZoneFQDN +names="$names kubenode" #ok, host PlainName +names="$names kubenode.hq.cloud.yunionyun.com" #ok, host CloudZoneFQDN +names="$names whoever-the-ether" #NXDOMAIN, nonexistent PlainName +names="$names whoever-the-ether.titan.hq.cloud.yunionyun.com" #NXDOMAIN, nonexistent CloudZoneFQDN +names="$names mail.google.com" #ok, dnsrecords +names="$names www.douban.com" #ok, pub +names="$names app" #ok, k8s svc PlainName in "default" ns +names="$names app.default" #NXDOMAIN, k8s svc name.namespace +names="$names app.default.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN +names="$names mon-kafka.system" #NXDOMAIN, k8s svc name.namespace +names="$names mon-kafka.system.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN + +# TODO source IP +# TODO other TYPEs +for name in $names; do + echo "############### $name" + #dig @192.168.222.171 $name + #dig -p 54 @10.168.222.136 $name + dig -p 54 @192.168.222.171 $name +done +``` + +PTR + +```sh +names='' +names="$names " #ok, guest ip +names="$names " #ok, host ip +names="$names " #ok, ptr records in db +names="$names " #NXDOMAIN, others + +for name in $names; do + echo "############### $name" + dig -p 54 @192.168.222.171 -x $name +done +``` + +# 配置 + + log { + # note that apart from rcode like NXDOMAIN, SERVFAIL, coredns will also + # log NOERROR response when it's NoData as defined by coredns itself + # + # > NoData indicates name found, but not the type: NOERROR in header, SOA in auth. + # + class denial + class error + } From 3be82ebd540e75a4c690dae9e686c6777f527851 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Sat, 25 Aug 2018 04:37:48 +0000 Subject: [PATCH 57/71] guestnetworks, hostnetworks: avoid subquery in get by address call --- pkg/compute/models/guestnetworks.go | 3 ++- pkg/compute/models/hostnetworks.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 2f3defdf37..223d6f4d4d 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -263,9 +263,10 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { - networks := manager.Query().SubQuery() + networks := manager.TableSpec().Instance() guests := GuestManager.Query() q := guests.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), sqlchemy.Equals(networks.Field("ip_addr"), address), sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")), )) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index 88433fb90a..5a94918e32 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -107,9 +107,10 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork { } func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost { - networks := man.Query().SubQuery() + networks := man.TableSpec().Instance() hosts := HostManager.Query() q := hosts.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), sqlchemy.Equals(networks.Field("ip_addr"), addr), sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")), )) From 40b99ef250b41dfc0f8b0853f18aa76210bac93b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 12:50:41 +0800 Subject: [PATCH 58/71] performClassActionHandler: handle body is nil --- pkg/appsrv/dispatcher/dispatcher.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index de313f451c..ea95a458f9 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -209,9 +209,8 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht if data == nil { data = body.(*jsonutils.JSONDict) } - } - if data == nil { - data = body.(*jsonutils.JSONDict) + } else { + data = jsonutils.NewDict() } results, err := manager.PerformClassAction(ctx, params[""], query, data) if err != nil { From a39eea4b646da92b2e4df6ceb05980c4849164e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 12:57:42 +0800 Subject: [PATCH 59/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E5=8F=AF=E7=94=A8storage=E6=97=B6=EF=BC=8C=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E6=95=B0=E6=8D=AEbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 7990188dd8..769a76612d 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -664,12 +664,15 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. } if jsonutils.QueryBoolean(query, "usable", false) { - hostStorage := HoststorageManager.Query().SubQuery() - q = q.Join(hostStorage, sqlchemy.AND( - sqlchemy.Equals(hostStorage.Field("storage_id"), q.Field("id")), - sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE}), - sqlchemy.IsTrue(q.Field("enabled")), - )) + hostStorageTable := HoststorageManager.Query().SubQuery() + hostTable := HostManager.Query().SubQuery() + sq := hostStorageTable.Query(hostStorageTable.Field("storage_id")).Join(hostTable, + sqlchemy.Equals(hostTable.Field("id"), hostStorageTable.Field("host_id"))). + Filter(sqlchemy.Equals(hostTable.Field("host_status"), HOST_ONLINE)) + + q = q.Filter(sqlchemy.In(q.Field("id"), sq)). + Filter(sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})). + Filter(sqlchemy.IsTrue(q.Field("enabled"))) } managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) From 4f24cbe615573ecb0b61ad59726c03349ec020e7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 13:07:25 +0800 Subject: [PATCH 60/71] remove some debug log --- pkg/cloudcommon/db/db_dispatcher.go | 4 ++-- pkg/cloudcommon/db/virtualresource.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index fe06fc2f9d..4541621ca1 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -524,7 +524,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr } func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - log.Debugf("Get %s", idStr) + // log.Debugf("Get %s", idStr) userCred := fetchUserCredential(ctx) data, err := dispatcher.tryGetModelProperty(ctx, idStr, query) @@ -541,7 +541,7 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer } else if err != nil { return nil, err } - log.Debugf("Get found %s", model) + // log.Debugf("Get found %s", model) if !model.AllowGetDetails(ctx, userCred, query) { return nil, httperrors.NewForbiddenError("Not allow to get details") } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 28360ccb55..f811e434a0 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -161,16 +161,16 @@ func (model *SVirtualResourceBase) AllowPerformMetadata(ctx context.Context, use } func (model *SVirtualResourceBase) GetTenantCache(ctx context.Context) (*STenant, error) { - log.Debugf("Get tenant by Id %s", model.ProjectId) + // log.Debugf("Get tenant by Id %s", model.ProjectId) return TenantCacheManager.FetchTenantById(ctx, model.ProjectId) } func (model *SVirtualResourceBase) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) *jsonutils.JSONDict { if userCred.IsSystemAdmin() { - log.Debugf("GetCustomizeColumns") + // log.Debugf("GetCustomizeColumns") tobj, err := model.GetTenantCache(ctx) if err == nil { - log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) + // log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) extra.Add(jsonutils.NewString(tobj.GetName()), "tenant") } else { log.Errorf("GetTenantCache fail %s", err) From 291cf2547e87440a14bc67c04af76e67b01ba2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 13:36:27 +0800 Subject: [PATCH 61/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddisk-resize=E5=90=8E?= =?UTF-8?q?=E6=9C=AA=E6=AD=A3=E7=A1=AE=E8=AE=BE=E7=BD=AEguest=E7=8A=B6?= =?UTF-8?q?=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/tasks/disk_resize_task.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go index 584c43ab94..dd25da5879 100644 --- a/pkg/compute/tasks/disk_resize_task.go +++ b/pkg/compute/tasks/disk_resize_task.go @@ -32,7 +32,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, } resion := "Cannot find host for disk" if host == nil || host.HostStatus != models.HOST_ONLINE { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion) + disk.SetDiskReady(ctx, self.GetUserCred(), resion) self.SetStageFailed(ctx, resion) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred()) } else { @@ -65,7 +65,7 @@ func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *mod } func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) self.SetStageFailed(ctx, resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred()) } @@ -94,6 +94,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode self.OnStartResizeDiskFailed(ctx, disk, err) return } + disk.SetDiskReady(ctx, self.GetUserCred(), "") notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status) db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred) self.CleanHostSchedCache(disk) @@ -103,6 +104,6 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode } func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) } From 4014c55b69a817cee771ec3415d96330c2263a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 13:39:54 +0800 Subject: [PATCH 62/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9C=AA=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96guestmanager=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/disks.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 047863499a..198586a6ee 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -560,9 +560,8 @@ func (self *SDisk) GetAttachedGuests() []SGuest { q = q.Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) ret := make([]SGuest, 0) - err := q.All(&ret) - if err != nil { - log.Errorf("%s", err) + if err := db.FetchModelObjects(GuestManager, q, &ret); err != nil { + log.Errorf("Fetch Geusts Objects %v", err) return nil } return ret From 2914e880af453cf59f793216268dbc5f0ddfe562 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 14:31:05 +0800 Subject: [PATCH 63/71] fix: NotifyParentTaskFailure reason --- pkg/cloudcommon/db/taskman/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 277032c453..de893d3476 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -538,7 +538,7 @@ func (self *STask) NotifyParentTaskFailure(ctx context.Context, reason string) { if len(reason) > 100 { reason = reason[:100] + "..." } - body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason))) + body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason)), "__reason__") self.NotifyParentTaskComplete(ctx, body, true) } From 618f183428212af860a4e6b62446f82214af5254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 14:45:45 +0800 Subject: [PATCH 64/71] =?UTF-8?q?release/2.0.0=E7=89=88=E6=9C=AC=E9=9C=80?= =?UTF-8?q?=E8=A6=81disk=E8=AF=A6=E6=83=85=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/disks.go | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 198586a6ee..699556c1bd 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -151,6 +152,21 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk { } return guestdisks } +func (self *SDisk) GetGuests() []SGuest { + result := make([]SGuest, 0) + query := GuestManager.Query() + guestdisks := GuestdiskManager.Query().SubQuery() + q := query.Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), query.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) + // q.DebugQuery() + err := db.FetchModelObjects(GuestManager, q, &result) + if err != nil { + log.Errorf(err.Error()) + return nil + } + return result +} func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { if self.GetGuestDiskCount() > 0 { @@ -183,6 +199,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -536,6 +559,49 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + func (self *SDisk) StartDiskDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, isPurge bool) error { params := jsonutils.NewDict() if isPurge { From 122c1d63f298da1dd8800a29e3e83d48b946e021 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 16:02:26 +0800 Subject: [PATCH 65/71] bugfixs for aliyun server create/delete --- pkg/cloudcommon/cronman/cronman.go | 2 ++ pkg/cloudcommon/db/taskman/tasks.go | 31 ++++++++++++----- pkg/compute/guestdrivers/base.go | 14 ++++---- pkg/compute/models/guestnetworks.go | 17 ++++++++++ pkg/compute/models/guests.go | 21 +++++++++--- pkg/compute/tasks/guest_create_task.go | 4 +-- pkg/compute/tasks/guest_delete_task.go | 6 +++- pkg/mcclient/auth/auth.go | 2 +- pkg/util/aliyun/aliyun.go | 1 + pkg/util/aliyun/errors.go | 13 +++++++ pkg/util/aliyun/instance.go | 47 ++++++++++++++++++-------- 11 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 pkg/util/aliyun/errors.go diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index d01cc096d5..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -89,6 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..3182d47138 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -285,20 +289,21 @@ func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) } } -func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, isMulti bool) { +func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject, isMulti bool) { var err error ctxData := task.GetRequestContext() ctx := ctxData.GetContext() taskFailed := false + data := odata if data != nil { taskStatus, _ := data.GetString("__status__") if len(taskStatus) > 0 && taskStatus != "OK" { taskFailed = true data, err = data.Get("reason") if err != nil { - data = jsonutils.NewString("Task failed due to unknown remote errors!") + data = jsonutils.NewString(fmt.Sprintf("Task failed due to unknown remote errors! %s", odata)) } } } else { @@ -320,7 +325,13 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, if !funcValue.IsValid() || funcValue.IsNil() { msg := fmt.Sprintf("Stage %s not found", stageName) - log.Errorf(msg) + if taskFailed { + // failed handler is optional, ignore the error + log.Warningf(msg) + msg, _ = data.GetString() + } else { + log.Errorf(msg) + } task.SetStageFailed(ctx, msg) task.SaveRequestContext(&ctxData) return @@ -408,9 +419,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 6d10f94aa5..93ab7183cc 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -54,13 +54,13 @@ func (self *SBaseGuestDriver) StartGuestCreateTask(guest *models.SGuest, ctx con } func (self *SBaseGuestDriver) OnGuestCreateTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - //if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { - // task.SetStage("on_auto_start_guest", nil) - // return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) - //} else { - task.SetStage("on_sync_status_complete", nil) - return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) - //} + if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { + task.SetStage("on_auto_start_guest", nil) + return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) + } else { + task.SetStage("on_sync_status_complete", nil) + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) + } } func (self *SBaseGuestDriver) StartDeleteGuestTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..514aff944d 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" + "database/sql" ) const ( @@ -331,6 +332,22 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest return nil } +func (manager *SGuestnetworkManager) getGuestNicByIP(ip string) (*SGuestnetwork, error) { + gn := SGuestnetwork{} + q := manager.Query() + q = q.Equals("ip_addr", ip) + err := q.First(&gn) + if err != nil { + if err != sql.ErrNoRows { + log.Errorf("getGuestNicByIP fail %s", err) + return nil, err + } + return nil, nil + } + gn.SetModelManager(manager) + return &gn, nil +} + func (self *SGuestnetwork) LogDetachEvent(userCred mcclient.TokenCredential, guest *SGuest, network *SNetwork) { if network == nil { netTmp, _ := NetworkManager.FetchById(self.NetworkId) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d47ff0e5f8..7676350d30 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1429,7 +1429,20 @@ func (self *SGuest) SyncVMNics(ctx context.Context, userCred mcclient.TokenCrede if add.net == nil { continue // cannot determine which network it attached to } - err := self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), + // check if the IP has been occupied, if yes, release the IP + gn, err := GuestnetworkManager.getGuestNicByIP(add.nic.GetIP()) + if err != nil { + result.AddError(err) + continue + } + if gn != nil { + err = gn.Detach(ctx, userCred) + if err != nil { + result.AddError(err) + continue + } + } + err = self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), add.nic.GetMAC(), add.nic.GetDriver(), 0, false, -1, add.reserve, IPAllocationDefault, true) if err != nil { result.AddError(err) @@ -2110,9 +2123,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } diff --git a/pkg/compute/tasks/guest_create_task.go b/pkg/compute/tasks/guest_create_task.go index 1248fe287e..a57f49afca 100644 --- a/pkg/compute/tasks/guest_create_task.go +++ b/pkg/compute/tasks/guest_create_task.go @@ -105,7 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String()) } -/*func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { +func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) } @@ -113,4 +113,4 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context func (self *GuestCreateTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) -}*/ +} diff --git a/pkg/compute/tasks/guest_delete_task.go b/pkg/compute/tasks/guest_delete_task.go index 09666d5e52..7b21109cda 100644 --- a/pkg/compute/tasks/guest_delete_task.go +++ b/pkg/compute/tasks/guest_delete_task.go @@ -4,12 +4,14 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/pkg/utils" ) type GuestDeleteTask struct { @@ -31,6 +33,7 @@ func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.ISt if options.Options.EnablePendingDelete && !guest.PendingDeleted && !jsonutils.QueryBoolean(self.Params, "purge", false) && !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { + log.Debugf("XXXXXXX Do guest pending delete... XXXXXXX") guestStatus, _ := self.Params.GetString("guest_status") if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) { @@ -38,6 +41,7 @@ func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.ISt return } } + log.Debugf("XXXXXXX Do real delete on guest ... XXXXXXX") self.OnGuestStopCompleteFailed(ctx, guest, data) } diff --git a/pkg/mcclient/auth/auth.go b/pkg/mcclient/auth/auth.go index 214be4b0ad..4e0957e66a 100644 --- a/pkg/mcclient/auth/auth.go +++ b/pkg/mcclient/auth/auth.go @@ -103,7 +103,7 @@ func (c *TokenCacheVerify) Verify(cli *mcclient.Client, adminToken, token string if err != nil { return nil, fmt.Errorf("Add %s credential to cache: %#v", cred.GetTokenString(), err) } - log.Infof("Add token: %s", cred) + // log.Debugf("Add token: %s", cred) return cred, nil } diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index 26b9efce28..9d002a46d1 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -3,6 +3,7 @@ package aliyun import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" diff --git a/pkg/util/aliyun/errors.go b/pkg/util/aliyun/errors.go new file mode 100644 index 0000000000..da618ab69d --- /dev/null +++ b/pkg/util/aliyun/errors.go @@ -0,0 +1,13 @@ +package aliyun + +import ( + aliyunerrors "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" +) + +func isError(err error, code string) bool { + aliyunErr, ok := err.(aliyunerrors.Error) + if ! ok { + return false + } + return aliyunErr.ErrorCode() == code +} diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 12cc379e68..2ad1e5f4fc 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -397,7 +397,10 @@ func (self *SRegion) doStopVM(instanceId string, isForce bool) error { } func (self *SRegion) doDeleteVM(instanceId string) error { - return self.instanceOperation(instanceId, "DeleteInstance", nil) + params := make(map[string]string) + params["TerminateSubscription"] = "false" + params["Force"] = "true" + return self.instanceOperation(instanceId, "DeleteInstance", params) } /*func (self *SRegion) waitInstanceStatus(instanceId string, target string, interval time.Duration, timeout time.Duration) error { @@ -420,7 +423,11 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout }*/ func (self *SRegion) StartVM(instanceId string) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StartVM: %s", err) + return err + } if status != InstanceStatusStopped { log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus @@ -433,7 +440,11 @@ func (self *SRegion) StartVM(instanceId string) error { } func (self *SRegion) StopVM(instanceId string, isForce bool) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StopVM: %s", err) + return err + } if status != InstanceStatusRunning { log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning) return cloudprovider.ErrInvalidStatus @@ -447,14 +458,13 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error { func (self *SRegion) DeleteVM(instanceId string) error { status, err := self.GetInstanceStatus(instanceId) - if status == InstanceStatusRunning { - err = self.StopVM(instanceId, true) - if err != nil { - return err - } - } else if status != InstanceStatusStopped { - log.Errorf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) - return cloudprovider.ErrInvalidStatus + if err != nil { + log.Errorf("Fail to get instance status on DeleteVM: %s", err) + return err + } + log.Debugf("Instance status on delete is %s", status) + if status != InstanceStatusStopped { + log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) } return self.doDeleteVM(instanceId) // if err != nil { @@ -487,9 +497,18 @@ func (self *SInstance) StopVM(isForce bool) error { } func (self *SInstance) DeleteVM() error { - err := self.host.zone.region.DeleteVM(self.InstanceId) - if err != nil { - return err + for { + err := self.host.zone.region.DeleteVM(self.InstanceId) + if err != nil { + if isError(err, "IncorrectInstanceStatus.Initializing") { + log.Infof("The instance is initializing, try later ...") + time.Sleep(10*time.Second) + } else { + return err + } + }else { + break + } } return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes } From c073766ccce803e5938bb32d33561aaffca08df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 16:49:48 +0800 Subject: [PATCH 66/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9B=B4=E6=96=B0secru?= =?UTF-8?q?les=E5=A4=B1=E8=B4=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/secgrouprules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 4347ea48c9..46b032e195 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -328,7 +328,7 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre cmp := strings.Compare(dbStr, ruleStr) if cmp == 0 { if dbRules[j].Description != rules[i].Description { - if _, err := manager.TableSpec().Update(dbRules[j], func() error { + if _, err := manager.TableSpec().Update(&dbRules[j], func() error { dbRules[j].Description = rules[i].Description return nil }); err != nil { From d1607e7be6b7080ca05cc8ef1ae846a2849e70ca Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Sat, 25 Aug 2018 18:58:24 +0800 Subject: [PATCH 67/71] support joint filter --- Gopkg.lock | 15 ++- pkg/cloudcommon/db/db_dispatcher.go | 30 ++++- vendor/yunion.io/x/log/hooks/file.go | 108 ++++++++++++++++++ vendor/yunion.io/x/log/hooks/stdio.go | 7 +- .../x/pkg/util/filterclause/filterclause.go | 36 +++++- .../x/pkg/util/sysutils/storagetypes.go | 8 +- 6 files changed, 190 insertions(+), 14 deletions(-) create mode 100644 vendor/yunion.io/x/log/hooks/file.go diff --git a/Gopkg.lock b/Gopkg.lock index 953506dd3e..dd0d306013 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -666,13 +666,15 @@ version = "v1.2.0" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" branch = "master" + digest = "1:c1f242537d2898c84fe9bfd8bd94b177cd5ff98a10c397dbbc494f4e2b173be6" name = "github.com/serialx/hashring" packages = ["."] + pruneopts = "UT" revision = "49a4782e9908fe098c907022a1bd7519c79803d6" [[projects]] + digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" @@ -1040,18 +1042,18 @@ [[projects]] branch = "master" - digest = "1:b707a5d591b21dad9ebdb264281632ed43d34e2b980dce967acb2c7adcc93ad3" + digest = "1:636db180d3fc734536a5437d93bb6fe937feb036c1ed062c25872a47c4283163" name = "yunion.io/x/log" packages = [ ".", "hooks", ] pruneopts = "UT" - revision = "a10b94c5038480920262e0f344656328434955a2" + revision = "5185c49f6d361f61e561fbccba1bedcc7c62748b" [[projects]] branch = "master" - digest = "1:47670c70d0326f624de21c8f29d0f73850062f0752f27e146308db62fbb540d3" + digest = "1:8e2a4d71738e5e3bffc415b23f100b0a5033dbf88154783122f5147d4fc213b5" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1086,7 +1088,7 @@ "utils", ] pruneopts = "UT" - revision = "6c349ecbdb90d5f6ecc9b86596428176c1937b32" + revision = "7348c84865abac3fcdec618591b7eee7c2e0669b" [[projects]] branch = "master" @@ -1109,6 +1111,7 @@ analyzer-version = 1 input-imports = [ "github.com/aliyun/alibaba-cloud-sdk-go/sdk", + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors", "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests", "github.com/aliyun/aliyun-oss-go-sdk/oss", "github.com/aokoli/goutils", @@ -1147,6 +1150,7 @@ "github.com/mholt/caddy/startupshutdown", "github.com/miekg/dns", "github.com/moul/http2curl", + "github.com/serialx/hashring", "github.com/stretchr/testify/assert", "github.com/vmware/govmomi", "github.com/vmware/govmomi/object", @@ -1158,6 +1162,7 @@ "golang.org/x/crypto/ssh", "gopkg.in/gin-gonic/gin.v1", "k8s.io/api/core/v1", + "k8s.io/apimachinery/pkg/api/errors", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/fields", "k8s.io/apimachinery/pkg/labels", diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 97df816bde..57b6ea13c9 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -251,6 +251,30 @@ func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery, return q, nil } +func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, jointFilters []string, filterAny bool) (*sqlchemy.SQuery, error) { + for _, f := range jointFilters { + jfc := filterclause.ParseJointFilterClause(f) + if jfc != nil { + jointModelManager := GetModelManager(jfc.GetJointModelName()) + schFields := searchFields(jointModelManager, userCred) + if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok { + sq := jointModelManager.Query(jfc.ReleatedKey) + cond := jfc.GetJointFilter(sq) + if cond != nil { + sq = sq.Filter(cond) + if filterAny { + q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("id"), sq))) + } else { + q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field("id"), sq))) + } + } + } + } + } + return q, nil +} + func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -272,11 +296,15 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem return nil, err } } + filterAny, _ := query.Bool("filter_any") filters := jsonutils.GetQueryStringArray(query, "filter") if len(filters) > 0 { - filterAny, _ := query.Bool("filter_any") q, err = applyListItemsGeneralFilters(manager, q, userCred, filters, filterAny) } + jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter") + if len(jointFilter) > 0 { + q, _ = applyListItemsGeneralJointFilters(manager, q, userCred, jointFilter, filterAny) + } return q, nil } diff --git a/vendor/yunion.io/x/log/hooks/file.go b/vendor/yunion.io/x/log/hooks/file.go new file mode 100644 index 0000000000..3a99b0397a --- /dev/null +++ b/vendor/yunion.io/x/log/hooks/file.go @@ -0,0 +1,108 @@ +package hooks + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +type LogFileHook struct { + FileDir string + FileName string + fullPath string + file *os.File + written int64 +} + +func (h *LogFileHook) Init() error { + if fi, err := os.Lstat(h.FileDir); err != nil { + if os.IsNotExist(err) { + os.MkdirAll(h.FileDir, 0755) + } else { + return fmt.Errorf("Lstat %s: %s", h.FileDir, err) + } + } else if !fi.Mode().IsDir() { + return fmt.Errorf("%s exists and it's not a directory", h.FileDir) + } + + h.fullPath = filepath.Join(h.FileDir, h.FileName) + file, err := os.OpenFile(h.fullPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("OpenFile %s: %s", h.fullPath, err) + } + h.file = file + h.written = 0 + return nil +} + +func (h *LogFileHook) DeInit() { + if h.file != nil { + h.file.Close() + } +} + +func (h *LogFileHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *LogFileHook) Fire(e *logrus.Entry) error { + if b, err := e.Logger.Formatter.Format(e); err != nil { + return err + } else { + n, err := h.file.Write(b) + h.written += int64(n) + return err + } + return nil +} + +func (h *LogFileHook) Written() int64 { + return h.written +} + +// rotate by size +type LogFileRotateHook struct { + LogFileHook + RotateNum int + RotateSize int64 + filePaths []string +} + +func (h *LogFileRotateHook) Init() error { + if err := h.LogFileHook.Init(); err != nil { + return err + } + h.filePaths = make([]string, h.RotateNum) + for i := 1; i < h.RotateNum; i++ { + fileName := fmt.Sprintf("%s.%d", h.FileName, i) + filePath := filepath.Join(h.FileDir, fileName) + h.filePaths[i] = filePath + } + h.filePaths[0] = filepath.Join(h.FileDir, h.FileName) + return nil +} + +func (h *LogFileRotateHook) rotate() { + for i := h.RotateNum - 1; i > 0; i-- { + filePath0 := h.filePaths[i-1] + if _, err := os.Lstat(filePath0); err != nil { + continue + } + filePath1 := h.filePaths[i] + os.Rename(filePath0, filePath1) + } + h.LogFileHook.DeInit() + h.LogFileHook.Init() +} + +func (h *LogFileRotateHook) Fire(e *logrus.Entry) error { + if err := h.LogFileHook.Fire(e); err != nil { + return err + } + if h.LogFileHook.Written() >= h.RotateSize { + h.rotate() + } + return nil +} diff --git a/vendor/yunion.io/x/log/hooks/stdio.go b/vendor/yunion.io/x/log/hooks/stdio.go index 6fa3dc4381..e5d32000ad 100644 --- a/vendor/yunion.io/x/log/hooks/stdio.go +++ b/vendor/yunion.io/x/log/hooks/stdio.go @@ -12,14 +12,9 @@ type StdioHook struct{} func (hook *StdioHook) Fire(entry *logrus.Entry) error { line, err := entry.String() if err != nil { - fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) return err } - if entry.Level >= logrus.ErrorLevel { - fmt.Fprintf(os.Stderr, line) - } else { - fmt.Fprintf(os.Stdout, line) - } + fmt.Fprintf(os.Stderr, line) return nil } diff --git a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go index 2a3a1ccd10..5518ea0f3b 100644 --- a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go +++ b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go @@ -16,6 +16,20 @@ type SFilterClause struct { params []string } +type SJointFilterClause struct { + SFilterClause + JointModel string + ReleatedKey string +} + +func (jfc *SJointFilterClause) GetJointFilter(q *sqlchemy.SQuery) sqlchemy.ICondition { + return jfc.QueryCondition(q) +} + +func (jfc *SJointFilterClause) GetJointModelName() string { + return jfc.JointModel[:len(jfc.JointModel)-1] +} + func (fc *SFilterClause) QueryCondition(q *sqlchemy.SQuery) sqlchemy.ICondition { field := q.Field(fc.field) if field == nil { @@ -63,11 +77,13 @@ func (fc *SFilterClause) String() string { } var ( - filterClausePattern *regexp.Regexp + filterClausePattern *regexp.Regexp + jointFilterClausePattern *regexp.Regexp ) func init() { filterClausePattern = regexp.MustCompile(`^(\w+)\.(\w+)\((.*)\)`) + jointFilterClausePattern = regexp.MustCompile(`^(\w+)\((\w+)\).(\w+)\.(\w+)\((.*)\)`) } func ParseFilterClause(filter string) *SFilterClause { @@ -79,3 +95,21 @@ func ParseFilterClause(filter string) *SFilterClause { fc := SFilterClause{field: matches[1], funcName: matches[2], params: params} return &fc } + +func ParseJointFilterClause(jointFilter string) *SJointFilterClause { + matches := jointFilterClausePattern.FindStringSubmatch(jointFilter) + if matches == nil { + return nil + } + params := utils.FindWords([]byte(matches[5]), 0) + jfc := SJointFilterClause{ + SFilterClause: SFilterClause{ + field: matches[3], + funcName: matches[4], + params: params, + }, + JointModel: matches[1], + ReleatedKey: matches[2], + } + return &jfc +} diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go index 476aa53bff..16baa403d2 100644 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go @@ -9,12 +9,18 @@ const ( STORAGE_NAS = "nas" STORAGE_VSAN = "vsan" + STORAGE_CLOUD = "cloud" STORAGE_CLOUD_SSD = "cloud_ssd" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + + STORAGE_STANDARD = "standard" //Azure hdd storage type + STORAGE_PREMIUM = "premium" //Azure ssd storage type ) var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY} + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD, STORAGE_PREMIUM} var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} From e6b77164c4e830737b266ded205cdefedc981105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 11:55:22 +0800 Subject: [PATCH 68/71] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=9B=9E=E6=94=B6=E7=AB=99=E6=97=B6=E6=9C=AA=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E8=AE=B0=E5=BD=95bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/hostdrivers/aliyun.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 0bcfa18005..57590014f8 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -81,9 +81,13 @@ func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err + } else if err := iDisk.Delete(); err != nil { + return err } else { - return iDisk.Delete() + data := jsonutils.NewDict() + task.ScheduleRun(data) } + return nil } func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { From 02e16544187f50939188b05bf84027414fa12806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 12:00:59 +0800 Subject: [PATCH 69/71] =?UTF-8?q?=E6=99=AE=E9=80=9A=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E4=BA=91=E7=9B=98=E6=97=B6=EF=BC=8C=E9=9C=80?= =?UTF-8?q?=E8=A6=81storage-list=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 28650d29f1..3797e8a33e 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -97,6 +97,10 @@ func (self *SStorage) IsLocal() bool { return self.StorageType == STORAGE_LOCAL || self.StorageType == STORAGE_BAREMETAL } +func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return true +} + func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id) From 852ba299e3e9ca74ae557450c36a1a2e2d3fc238 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 27 Aug 2018 12:15:21 +0800 Subject: [PATCH 70/71] region-dns: add vars to build rpm package --- build/region-dns/vars | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 build/region-dns/vars diff --git a/build/region-dns/vars b/build/region-dns/vars new file mode 100644 index 0000000000..778d49a5da --- /dev/null +++ b/build/region-dns/vars @@ -0,0 +1,2 @@ +DESCRIPTION="Yunion Cloud Region DNS Service" +# SERVICE="yes" From 23ebca6cfe41f016c2ea309a20a1246622ef7a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 12:18:18 +0800 Subject: [PATCH 71/71] =?UTF-8?q?=E6=B7=BB=E5=8A=A0interface=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudprovider/resources.go | 3 +- pkg/compute/guestdrivers/aliyun.go | 50 ++++++++++++++++++++----- pkg/compute/hostdrivers/aliyun.go | 16 +++++++- pkg/compute/hostdrivers/kvm.go | 2 +- pkg/compute/models/disks.go | 54 ++++++++++++++++++++++----- pkg/compute/models/guests.go | 35 ++++++++++++++++- pkg/compute/models/hostdrivers.go | 2 +- pkg/compute/tasks/disk_create_task.go | 4 +- pkg/util/aliyun/disk.go | 10 +++++ pkg/util/aliyun/host.go | 4 ++ pkg/util/aliyun/image.go | 4 ++ pkg/util/aliyun/instance.go | 15 ++++++++ pkg/util/aliyun/region.go | 4 ++ pkg/util/aliyun/securitygroup.go | 4 ++ pkg/util/aliyun/storage.go | 4 ++ pkg/util/aliyun/storagecache.go | 5 +++ pkg/util/aliyun/vpc.go | 4 ++ pkg/util/aliyun/vswitch.go | 4 ++ pkg/util/aliyun/wire.go | 5 +++ pkg/util/aliyun/zone.go | 5 +++ pkg/util/esxi/host.go | 4 ++ pkg/util/esxi/storage.go | 6 ++- pkg/util/esxi/virtualmachine.go | 8 +++- 23 files changed, 221 insertions(+), 31 deletions(-) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index ec124341c0..3c333f8946 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -18,6 +18,7 @@ type ICloudResource interface { Refresh() error IsEmulated() bool + GetMetadata() *jsonutils.JSONDict } type ICloudRegion interface { @@ -156,7 +157,7 @@ type ICloudVM interface { UpdateVM(name string) error RebuildRoot(imageId string) error DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error - ChangeConfig(instanceId string,ncpu int, vmem int) error + ChangeConfig(instanceId string, ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) AttachDisk(diskId string) error } diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index c8953351ad..45226734d3 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" @@ -116,8 +117,9 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo } type SDiskInfo struct { - Size int - Uuid string + Size int + Uuid string + Metadata map[string]string } func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { @@ -207,12 +209,19 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu dinfo := SDiskInfo{} dinfo.Uuid = idisks[i].GetGlobalId() dinfo.Size = idisks[i].GetDiskSizeMB() + if metaData := idisks[i].GetMetadata(); metaData != nil { + dinfo.Metadata = make(map[string]string, 0) + if err := metaData.Unmarshal(dinfo.Metadata); err != nil { + log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) + } + } diskInfo[i] = dinfo } data.Add(jsonutils.Marshal(&diskInfo), "disks") } data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + data.Add(iVM.GetMetadata(), "metadata") return data, nil }) @@ -236,7 +245,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu resetPassword := jsonutils.QueryBoolean(params, "reset_password", false) deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) password, _ := params.GetString("password") - if resetPassword && len(password) == 0{ + if resetPassword && len(password) == 0 { password = seclib2.RandomPassword2(12) } @@ -245,15 +254,15 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu publicKey = k } - taskman.LocalTaskRun(task, func () (jsonutils.JSONObject, error) { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { encpasswd, err := utils.EncryptAESBase64(guest.Id, password) if err != nil { log.Errorf("encrypt password failed %s", err) } data := jsonutils.NewDict() - data.Add(jsonutils.NewString("root"), "account") // 用户名 - data.Add(jsonutils.NewString(encpasswd), "key") // 密码 + data.Add(jsonutils.NewString("root"), "account") // 用户名 + data.Add(jsonutils.NewString(encpasswd), "key") // 密码 e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description) return data, e }) @@ -284,6 +293,13 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex disk.DiskSize = diskInfo[i].Size disk.ExternalId = diskInfo[i].Uuid disk.Status = models.DISK_READY + if len(diskInfo[i].Metadata) > 0 { + for key, value := range diskInfo[i].Metadata { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } return nil }) if err != nil { @@ -299,6 +315,20 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex if len(uuid) > 0 { guest.SetExternalId(uuid) } + + if metaData, _ := data.Get("metadata"); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get guest %s metadata error: %v", guest.Name, err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } @@ -320,9 +350,9 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue } type SAliyunVMChangeConfig struct { - InstanceId string - Cpu int - Memory int + InstanceId string + Cpu int + Memory int } func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { @@ -432,4 +462,4 @@ func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, gues task.ScheduleRun(nil) return nil -} \ No newline at end of file +} diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 0bcfa18005..b96ea60a30 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -48,7 +48,7 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host * return nil } -func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else { @@ -61,6 +61,20 @@ func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, } else { if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { disk.ExternalId = iDisk.GetGlobalId() + + if metaData := iDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return nil }); err != nil { log.Errorf("Update disk externalId err: %v", err) diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 0a5cbca5b2..ed08542a36 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -74,7 +74,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod return nil } -func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) header.Add("X-Region-Version", "v2") diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 3081709834..b05fa85b5e 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -297,7 +297,7 @@ func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.To return nil } -func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { +func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { log.Infof("Allocating disk on host %s ...", host.GetName()) templateId := self.GetTemplateId() @@ -326,7 +326,7 @@ func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, if rebuild { content.Add(jsonutils.JSONTrue, "rebuild") } - return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content) + return host.GetHostDriver().RequestAllocateDiskOnStorage(ctx, host, storage, self, task, content) } func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -433,7 +433,7 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro return disks, nil } -func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { +func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { @@ -444,13 +444,13 @@ func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vd return nil, err } storage := storageObj.(*SStorage) - return manager.newFromCloudDisk(userCred, vdisk, storage) + return manager.newFromCloudDisk(ctx, userCred, vdisk, storage) } else { return nil, err } } else { disk := diskObj.(*SDisk) - err = disk.syncWithCloudDisk(userCred, vdisk) + err = disk.syncWithCloudDisk(ctx, userCred, vdisk) if err != nil { return nil, err } @@ -490,7 +490,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudDisk(userCred, commonext[i]) + err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i]) if err != nil { syncResult.UpdateError(err) } else { @@ -501,7 +501,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudDisk(userCred, added[i], storage) + new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage) if err != nil { syncResult.AddError(err) } else { @@ -514,7 +514,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To return localDisks, remoteDisks, syncResult } -func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { +func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { extDisk.Refresh() self.Name = extDisk.GetName() @@ -535,11 +535,26 @@ func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk }) if err != nil { log.Errorf("syncWithCloudDisk error %s", err) + return err } - return err + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } + + return nil } -func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { +func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { disk := SDisk{} disk.SetModelManager(manager) @@ -562,6 +577,20 @@ func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, log.Errorf("newFromCloudZone fail %s", err) return nil, err } + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return &disk, nil } @@ -891,6 +920,11 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict { storage := self.GetStorage() desc.Add(jsonutils.NewString(storage.StorageType), "storage_type") desc.Add(jsonutils.NewString(storage.MediumType), "medium_type") + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + fs := self.GetFsFormat() if len(fs) > 0 { desc.Add(jsonutils.NewString(fs), "fs_format") diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 977a5dc28c..d4251224d2 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1249,6 +1249,18 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token db.OpsLog.LogEvent(self, db.ACT_UPDATE, diffStr, userCred) } } + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } return nil } @@ -1279,6 +1291,20 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. if err != nil { log.Errorf("Insert fail %s", err) } + + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + return &guest, nil } @@ -1668,7 +1694,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred if len(vdisks[i].GetGlobalId()) == 0 { continue } - disk, err := DiskManager.syncCloudDisk(userCred, vdisks[i]) + disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i]) if err != nil { result.Error(err) return result @@ -3140,6 +3166,11 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { if intBw > 0 { desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw))) } + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) spec := self.GetSpec(false) if self.GetHypervisor() == HYPERVISOR_BAREMETAL { @@ -3454,4 +3485,4 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use for i := 0; i < len(guests); i += 1 { guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true) } -} \ No newline at end of file +} diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 3f162d75da..6749e5ac68 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -12,7 +12,7 @@ import ( type IHostDriver interface { GetHostType() string CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error - RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error + RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go index 1c47d60d5a..be0e2635ed 100644 --- a/pkg/compute/tasks/disk_create_task.go +++ b/pkg/compute/tasks/disk_create_task.go @@ -40,10 +40,10 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis } storage := disk.GetStorage() host := storage.GetMasterHost() - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred()) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATING, disk.GetShortDesc(), self.GetUserCred()) disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") self.SetStage("on_disk_ready", nil) - if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { + if err := disk.StartAllocate(ctx, host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) } } diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index ca6f21d4f8..7349b8c7c0 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -54,6 +54,16 @@ type SDisk struct { ZoneId string } +func (self *SDisk) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::DiskCategory::DiskType + priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) { if limit > 50 || limit <= 0 { limit = 50 diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index cf842879ad..24c5a2a1b1 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -19,6 +19,10 @@ type SHost struct { zone *SZone } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { return self.zone.GetIWires() } diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index 9dfb6f0b3f..849e9eb570 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -55,6 +55,10 @@ type SImage struct { Usage string } +func (self *SImage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SImage) GetId() string { return self.ImageId } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 12874bc1bb..7f034089b6 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" "time" + "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/jsonutils" @@ -164,6 +165,20 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit return instances, int(total), nil } +func (self *SInstance) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::InstanceType::NetworkType::OSType::IoOptimized' + optimized := "optimized" + if !self.IoOptimized { + optimized = "none" + } + priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SInstance) GetCreateTime() time.Time { return self.CreationTime } diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index 78efb6a653..311f7bf195 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -36,6 +36,10 @@ func (self *SRegion) GetClient() *SAliyunClient { return self.client } +func (self *SRegion) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SRegion) getEcsClient() (*sdk.Client, error) { if self.ecsClient == nil { cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret) diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 04d787d838..3082451737 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -77,6 +77,10 @@ func (v PermissionSet) Less(i, j int) bool { return false } +func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SSecurityGroup) GetId() string { return self.SecurityGroupId } diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index 8f66d8b68b..d8dec31739 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -15,6 +15,10 @@ type SStorage struct { storageType string } +func (self *SStorage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStorage) GetId() string { return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) } diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 61394948c2..fab2072253 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -19,6 +20,10 @@ type SStoragecache struct { iimages []cloudprovider.ICloudImage } +func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStoragecache) GetId() string { return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) } diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index 6fb04fcc0d..ea645ff2be 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -45,6 +45,10 @@ type SVpc struct { VpcName string } +func (self *SVpc) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVpc) GetId() string { return self.VpcId } diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go index 9c9e7bbe84..bef2c3d74d 100644 --- a/pkg/util/aliyun/vswitch.go +++ b/pkg/util/aliyun/vswitch.go @@ -34,6 +34,10 @@ type SVSwitch struct { ZoneId string } +func (self *SVSwitch) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVSwitch) GetId() string { return self.VSwitchId } diff --git a/pkg/util/aliyun/wire.go b/pkg/util/aliyun/wire.go index 903dcc681a..12793359c1 100644 --- a/pkg/util/aliyun/wire.go +++ b/pkg/util/aliyun/wire.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -14,6 +15,10 @@ type SWire struct { inetworks []cloudprovider.ICloudNetwork } +func (self *SWire) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SWire) GetId() string { return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.zone.GetId()) } diff --git a/pkg/util/aliyun/zone.go b/pkg/util/aliyun/zone.go index 844b4d6e6f..2241e4d9ce 100644 --- a/pkg/util/aliyun/zone.go +++ b/pkg/util/aliyun/zone.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -111,6 +112,10 @@ type SZone struct { AvailableDedicatedHostTypes SDedicatedHostTypes } +func (self *SZone) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SZone) GetId() string { return self.ZoneId } diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index 4f524948d2..3a9a053c5d 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -68,6 +68,10 @@ func NewHost(manager *SESXiClient, host *mo.HostSystem, dc *SDatacenter) *SHost return &SHost{SManagedObject: newManagedObject(manager, host, dc)} } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) getHostSystem() *mo.HostSystem { return self.object.(*mo.HostSystem) } diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index dc154261d2..b64c57ce1f 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -18,6 +18,10 @@ func NewDatastore(manager *SESXiClient, ds *mo.Datastore, dc *SDatacenter) *SDat return &SDatastore{SManagedObject: newManagedObject(manager, ds, dc)} } +func (self *SDatastore) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SDatastore) getDatastore() *mo.Datastore { return self.object.(*mo.Datastore) } @@ -89,4 +93,4 @@ func (self *SDatastore) GetManagerId() string { func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented -} \ No newline at end of file +} diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 75ccccc448..34c1a9c682 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -27,6 +27,10 @@ func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacen return &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} } +func (self *SVirtualMachine) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVirtualMachine) getVirtualMachine() *mo.VirtualMachine { return self.object.(*mo.VirtualMachine) } @@ -226,6 +230,6 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { return ret, nil } -func (dc *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { +func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) error { return cloudprovider.ErrNotImplemented -} \ No newline at end of file +}