From 6429e7b1f8f4ebf5d77c354d7d7b708727cd3d5b Mon Sep 17 00:00:00 2001 From: Zhang Dongliang Date: Wed, 4 Sep 2019 10:01:57 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20devtool=20cron=20manag?= =?UTF-8?q?er=EF=BC=8C=E7=94=A8=E6=9D=A5=E7=AE=A1=E7=90=86Ansibleserver=20?= =?UTF-8?q?=E7=9A=84=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/devtool/vars | 1 + cmd/climc/shell/ansibleplaybooks.go | 3 - cmd/climc/shell/devtoolcronjob.go | 222 +++++++++++++++++ cmd/climc/shell/devtooltemplate.go | 154 ++++++++++++ cmd/devtool/main.go | 26 ++ pkg/ansibleserver/models/ansibleplaybooks.go | 25 +- .../models/ansibleplaybooks_v2.go | 17 +- .../models => apis/ansible}/status.go | 2 +- pkg/cloudcommon/db/db_dispatcher.go | 1 - pkg/devtool/models/actions.go | 235 ++++++++++++++++++ pkg/devtool/models/devtoolcronjob.go | 137 ++++++++++ pkg/devtool/models/devtooltemplate.go | 79 ++++++ pkg/devtool/models/doc.go | 1 + pkg/devtool/models/initdb.go | 37 +++ pkg/devtool/options/doc.go | 1 + pkg/devtool/options/option.go | 26 ++ pkg/devtool/service/doc.go | 1 + pkg/devtool/service/handler.go | 47 ++++ pkg/devtool/service/service.go | 58 +++++ pkg/devtool/tasks/doc.go | 15 ++ pkg/devtool/tasks/template_binding_servers.go | 45 ++++ .../tasks/template_unbinding_servers.go | 45 ++++ pkg/devtool/tasks/template_update.go | 45 ++++ pkg/mcclient/modules/managers.go | 6 + pkg/mcclient/modules/mod_devtools.go | 43 ++++ pkg/mcclient/options/devtooltemplate.go | 159 ++++++++++++ 26 files changed, 1406 insertions(+), 25 deletions(-) create mode 100644 build/devtool/vars create mode 100644 cmd/climc/shell/devtoolcronjob.go create mode 100644 cmd/climc/shell/devtooltemplate.go create mode 100644 cmd/devtool/main.go rename pkg/{ansibleserver/models => apis/ansible}/status.go (98%) create mode 100644 pkg/devtool/models/actions.go create mode 100644 pkg/devtool/models/devtoolcronjob.go create mode 100644 pkg/devtool/models/devtooltemplate.go create mode 100644 pkg/devtool/models/doc.go create mode 100644 pkg/devtool/models/initdb.go create mode 100644 pkg/devtool/options/doc.go create mode 100644 pkg/devtool/options/option.go create mode 100644 pkg/devtool/service/doc.go create mode 100644 pkg/devtool/service/handler.go create mode 100644 pkg/devtool/service/service.go create mode 100644 pkg/devtool/tasks/doc.go create mode 100644 pkg/devtool/tasks/template_binding_servers.go create mode 100644 pkg/devtool/tasks/template_unbinding_servers.go create mode 100644 pkg/devtool/tasks/template_update.go create mode 100644 pkg/mcclient/modules/mod_devtools.go create mode 100644 pkg/mcclient/options/devtooltemplate.go diff --git a/build/devtool/vars b/build/devtool/vars new file mode 100644 index 0000000000..54e6474a6b --- /dev/null +++ b/build/devtool/vars @@ -0,0 +1 @@ +DESCRIPTION="Yunion DevTools Server" diff --git a/cmd/climc/shell/ansibleplaybooks.go b/cmd/climc/shell/ansibleplaybooks.go index 392a16b123..1dedc124a4 100644 --- a/cmd/climc/shell/ansibleplaybooks.go +++ b/cmd/climc/shell/ansibleplaybooks.go @@ -15,9 +15,6 @@ package shell import ( - //"fmt" - //"strings" - "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" diff --git a/cmd/climc/shell/devtoolcronjob.go b/cmd/climc/shell/devtoolcronjob.go new file mode 100644 index 0000000000..ef89832633 --- /dev/null +++ b/cmd/climc/shell/devtoolcronjob.go @@ -0,0 +1,222 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func paramValidator(param jsonutils.JSONObject) (bool, error) { + // TODO 移到 server 端 + log.Infof("paramValidator: param: %+v", param) + interval, err := param.Int("interval") + if err != nil { + return true, nil + } + day, err := param.Int("day") + if err != nil { + return true, nil + } + if interval == 0 && day == 0 { + return false, fmt.Errorf("interval and day can not be 0 at the same time") + } + return true, nil +} + +func init() { + type CronjobListOptions struct { + options.BaseListOptions + Name string `help:"cloud region ID or Name" json:"-"` + } + + R(&CronjobListOptions{}, "devtoolcronjob-list", "List Devtool Cronjobs", func(s *mcclient.ClientSession, args *CronjobListOptions) error { + params, err := options.ListStructToParams(args) + if err != nil { + return err + } + var result *modulebase.ListResult + result, err = modules.DevToolCronjobs.List(s, params) + printList(result, modules.DevToolCronjobs.GetColumns(s)) + return nil + }) + + type CronjobCreateOptions struct { + NAME string `help:"Ansible Playbook ID or Name" json:"-"` + Day int `help:"Cronjob runs at given day" default:"0"` + Hour int `help:"Cronjob runs at given hour" default:"0"` + Min int `help:"Cronjob runs at given min" default:"0"` + Sec int `help:"Cronjob runs at given sec" default:"0"` + Interval int64 `help:"Cronjob runs at given interval" default:"0"` + Start bool `help:"start job when created" default:"false"` + Enabled bool `help:"Set job status enabled" default:"false"` + } + R( + &CronjobCreateOptions{}, + "devtoolcronjob-create", + "Create a cronjob repo component", + func(s *mcclient.ClientSession, args *CronjobCreateOptions) error { + result, err := modules.AnsiblePlaybooks.Get(s, args.NAME, nil) + if err != nil { + return err + } + ansiblePlaybookName, err := result.GetString("name") + if err != nil { + return err + } + + ansiblePlaybookID, err := result.GetString("id") + if err != nil { + return err + } + + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(ansiblePlaybookName), "name") + + params.Add(jsonutils.NewString(ansiblePlaybookID), "ansible_playbook_id") + + if args.Start { + params.Add(jsonutils.JSONTrue, "start") + + } + if args.Enabled { + params.Add(jsonutils.JSONTrue, "enabled") + } else if args.Interval > 0 { + params.Add(jsonutils.NewInt(int64(args.Interval)), "interval") + } else { + params.Add(jsonutils.NewInt(int64(0)), "interval") + params.Add(jsonutils.NewInt(int64(args.Day)), "day") + params.Add(jsonutils.NewInt(int64(args.Hour)), "hour") + params.Add(jsonutils.NewInt(int64(args.Min)), "min") + params.Add(jsonutils.NewInt(int64(args.Sec)), "sec") + } + ok, err := paramValidator(params) + if err != nil || !ok { + log.Infof("paramValidator error %s", err) + return err + } + cronjob, err := modules.DevToolCronjobs.Create(s, params) + if err != nil { + log.Errorf("modules.DevToolCronjobs.Create error %s", err) + return err + } + printObject(cronjob) + return nil + }, + ) + + type DevToolCronjobShowOptions struct { + ID string `help:"ID or Name of the DevToolCronjob to show"` + } + R(&DevToolCronjobShowOptions{}, "devtoolcronjob-show", "Show cronjob details", func(s *mcclient.ClientSession, args *DevToolCronjobShowOptions) error { + result, err := modules.DevToolCronjobs.Get(s, args.ID, nil) + if err != nil { + return err + } + printObject(result) + return nil + }) + + type DevToolCronjobUpdateOptions struct { + ID string `help:"ID or Name of DevToolCronjob to update"` + Day int `help:"Cronjob runs at given day" json:"-" default:"-1"` + Hour int `help:"Cronjob runs at given hour" json:"-" default:"-1"` + Min int `help:"Cronjob runs at given min" default:"-1"` + Sec int `help:"Cronjob runs at given sec" default:"-1"` + Interval int `help:"Cronjob runs at given interval" default:"-1"` + Start bool `help:"start job when created"` + Stop bool `help:"start job when created"` + Enable bool `help:"Set job status enabled"` + Disable bool `help:"Set job status enabled"` + } + R(&DevToolCronjobUpdateOptions{}, "devtoolcronjob-update", "Update DevToolCronjob", func(s *mcclient.ClientSession, args *DevToolCronjobUpdateOptions) error { + result, err := modules.DevToolCronjobs.Get(s, args.ID, nil) + if err != nil { + return err + } + + params := jsonutils.NewDict() + interval, _ := result.Int("interval") + day, _ := result.Int("day") + params.Add(jsonutils.NewString(args.ID), "id") + params.Add(jsonutils.NewInt(int64(interval)), "interval") + params.Add(jsonutils.NewInt(int64(day)), "day") + + log.Infof("DevToolCronjobUpdateOptions args: %+v", args) + if args.Interval >= 0 { + params.Add(jsonutils.NewInt(int64(args.Interval)), "interval") + if args.Interval > 0 { + params.Add(jsonutils.NewInt(0), "day") + } + } else if args.Day >= 0 { + params.Add(jsonutils.NewInt(int64(args.Day)), "day") + if args.Day > 0 { + params.Add(jsonutils.NewInt(0), "interval") + } + } + if args.Hour >= 0 { + params.Add(jsonutils.NewInt(int64(args.Hour)), "hour") + } + if args.Min >= 0 { + params.Add(jsonutils.NewInt(int64(args.Min)), "min") + } + if args.Sec >= 0 { + params.Add(jsonutils.NewInt(int64(args.Sec)), "sec") + } + + ok, err := paramValidator(params) + if err != nil || !ok { + return err + } + + if args.Start && args.Stop { + return fmt.Errorf("can not set job start and stop at the same time") + } else if args.Start { + params.Add(jsonutils.JSONTrue, "start") + } else if args.Stop { + params.Add(jsonutils.JSONFalse, "start") + } + if args.Enable && args.Disable { + return fmt.Errorf("can not set job enabled and disabled at the same time") + } else if args.Enable { + params.Add(jsonutils.JSONTrue, "enabled") + } else if args.Disable { + params.Add(jsonutils.JSONFalse, "enabled") + } + + result, err = modules.DevToolCronjobs.Update(s, args.ID, params) + if err != nil { + return err + } + printObject(result) + return nil + }) + + R(&DevToolCronjobShowOptions{}, "devtoolcronjob-delete", "Delete DevToolCronjob", func(s *mcclient.ClientSession, args *DevToolCronjobShowOptions) error { + result, err := modules.DevToolCronjobs.Delete(s, args.ID, nil) + if err != nil { + return err + } + printObject(result) + return nil + }) +} diff --git a/cmd/climc/shell/devtooltemplate.go b/cmd/climc/shell/devtooltemplate.go new file mode 100644 index 0000000000..615f3d6889 --- /dev/null +++ b/cmd/climc/shell/devtooltemplate.go @@ -0,0 +1,154 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + + printAnsiblePlaybookObject := func(obj jsonutils.JSONObject) { + dict := obj.(*jsonutils.JSONDict) + pbJson, err := dict.Get("playbook") + if err != nil { + printObject(obj) + return + } + pbStr := pbJson.YAMLString() + dict.Set("playbook", jsonutils.NewString(pbStr)) + printObject(obj) + } + + type TemplateListOptions struct { + options.BaseListOptions + Name string `help:"cloud region ID or Name" json:"-"` + } + + R(&TemplateListOptions{}, "devtooltemplate-list", "List Devtool Templates", func(s *mcclient.ClientSession, args *TemplateListOptions) error { + params, err := options.ListStructToParams(args) + if err != nil { + return err + } + var result *modulebase.ListResult + result, err = modules.DevToolTemplates.List(s, params) + printList(result, modules.DevToolTemplates.GetColumns(s)) + return nil + }) + + R( + &options.DevtoolTemplateCreateOptions{}, + "devtooltemplate-create", + "Create a template repo component", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateCreateOptions) error { + + params, err := opts.Params() + if err != nil { + return err + } + log.Infof("devtool playbook create opts: %+v", params) + apb, err := modules.DevToolTemplates.Create(s, params) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) + + R( + &options.DevtoolTemplateIdOptions{}, + "devtooltemplate-show", + "Show devtool template", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateIdOptions) error { + apb, err := modules.DevToolTemplates.Get(s, opts.ID, nil) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) + + R( + &options.DevtoolTemplateBindingOptions{}, + "devtooltemplate-bind", + "Binding devtool template to a host/vm", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateBindingOptions) error { + params := jsonutils.NewDict() + params.Set("server_id", jsonutils.NewString(opts.ServerID)) + apb, err := modules.DevToolTemplates.PerformAction(s, opts.ID, "bind", params) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) + + R( + &options.DevtoolTemplateBindingOptions{}, + "devtooltemplate-unbind", + "Binding devtool template to a host/vm", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateBindingOptions) error { + params := jsonutils.NewDict() + params.Set("server_id", jsonutils.NewString(opts.ServerID)) + apb, err := modules.DevToolTemplates.PerformAction(s, opts.ID, "unbind", params) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) + + R( + &options.DevtoolTemplateIdOptions{}, + "devtooltemplate-delete", + "Delete devtool template", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateIdOptions) error { + apb, err := modules.DevToolTemplates.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) + + R( + &options.DevtoolTemplateUpdateOptions{}, + "devtooltemplate-update", + "Update ansible playbook", + func(s *mcclient.ClientSession, opts *options.DevtoolTemplateUpdateOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + apb, err := modules.DevToolTemplates.Update(s, opts.ID, params) + if err != nil { + return err + } + printAnsiblePlaybookObject(apb) + return nil + }, + ) +} diff --git a/cmd/devtool/main.go b/cmd/devtool/main.go new file mode 100644 index 0000000000..a86b6892d2 --- /dev/null +++ b/cmd/devtool/main.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "yunion.io/x/onecloud/pkg/devtool/service" + "yunion.io/x/onecloud/pkg/util/atexit" +) + +func main() { + defer atexit.Handle() + + service.StartService() +} diff --git a/pkg/ansibleserver/models/ansibleplaybooks.go b/pkg/ansibleserver/models/ansibleplaybooks.go index 841cb34c98..b392b893f8 100644 --- a/pkg/ansibleserver/models/ansibleplaybooks.go +++ b/pkg/ansibleserver/models/ansibleplaybooks.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + apis "yunion.io/x/onecloud/pkg/apis/ansible" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -75,7 +76,7 @@ func (man *SAnsiblePlaybookManager) ValidateCreateData(ctx context.Context, user if err := pbV.Validate(data); err != nil { return nil, err } - data.Set("status", jsonutils.NewString(AnsiblePlaybookStatusInit)) + data.Set("status", jsonutils.NewString(apis.AnsiblePlaybookStatusInit)) return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) } @@ -90,14 +91,14 @@ func (apb *SAnsiblePlaybook) PostCreate(ctx context.Context, userCred mcclient.T func (man *SAnsiblePlaybookManager) InitializeData() error { pbs := []SAnsiblePlaybook{} q := AnsiblePlaybookManager.Query() - q = q.Filter(sqlchemy.Equals(q.Field("status"), AnsiblePlaybookStatusRunning)) + q = q.Filter(sqlchemy.Equals(q.Field("status"), apis.AnsiblePlaybookStatusRunning)) if err := db.FetchModelObjects(AnsiblePlaybookManager, q, &pbs); err != nil { return errors.WithMessage(err, "fetch running playbooks") } for i := 0; i < len(pbs); i++ { pb := &pbs[i] _, err := db.Update(pb, func() error { - pb.Status = AnsiblePlaybookStatusUnknown + pb.Status = apis.AnsiblePlaybookStatusUnknown return nil }) if err != nil { @@ -108,14 +109,14 @@ func (man *SAnsiblePlaybookManager) InitializeData() error { } func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context) error { - if apb.Status == AnsiblePlaybookStatusRunning { + if apb.Status == apis.AnsiblePlaybookStatusRunning { return httperrors.NewConflictError("playbook is in running state") } return nil } func (apb *SAnsiblePlaybook) ValidateUpdateCondition(ctx context.Context) error { - if apb.Status == AnsiblePlaybookStatusRunning { + if apb.Status == apis.AnsiblePlaybookStatusRunning { return httperrors.NewConflictError("playbook is in running state") } return nil @@ -127,7 +128,7 @@ func (apb *SAnsiblePlaybook) ValidateUpdateData(ctx context.Context, userCred mc return nil, err } apb.Playbook = pbV.Playbook // Update as a whole - data.Set("status", jsonutils.NewString(AnsiblePlaybookStatusInit)) + data.Set("status", jsonutils.NewString(apis.AnsiblePlaybookStatusInit)) return data, nil } @@ -184,7 +185,7 @@ func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient. apb.StartTime = time.Now() apb.EndTime = time.Time{} apb.Output = "" - apb.Status = AnsiblePlaybookStatusRunning + apb.Status = apis.AnsiblePlaybookStatusRunning return nil }) if err != nil { @@ -204,12 +205,12 @@ func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient. _, err := db.Update(apb, func() error { err := man.sessions.Err(apb.Id) if err != nil { - apb.Status = AnsiblePlaybookStatusCanceled + apb.Status = apis.AnsiblePlaybookStatusCanceled } else if runErr != nil { log.Warningf("playbook %s(%s) failed: %v", apb.Name, apb.Id, runErr) - apb.Status = AnsiblePlaybookStatusFailed + apb.Status = apis.AnsiblePlaybookStatusFailed } else { - apb.Status = AnsiblePlaybookStatusSucceeded + apb.Status = apis.AnsiblePlaybookStatusSucceeded } apb.EndTime = time.Now() return nil @@ -226,9 +227,9 @@ func (apb *SAnsiblePlaybook) stopPlaybook(ctx context.Context, userCred mcclient man.sessionsMux.Lock() defer man.sessionsMux.Unlock() if !man.sessions.Has(apb.Id) { - if apb.Status == AnsiblePlaybookStatusRunning { + if apb.Status == apis.AnsiblePlaybookStatusRunning { _, err := db.Update(apb, func() error { - apb.Status = AnsiblePlaybookStatusUnknown + apb.Status = apis.AnsiblePlaybookStatusUnknown return nil }) if err != nil { diff --git a/pkg/ansibleserver/models/ansibleplaybooks_v2.go b/pkg/ansibleserver/models/ansibleplaybooks_v2.go index fae6ab19b8..fd708efa97 100644 --- a/pkg/ansibleserver/models/ansibleplaybooks_v2.go +++ b/pkg/ansibleserver/models/ansibleplaybooks_v2.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + apis "yunion.io/x/onecloud/pkg/apis/ansible" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -71,7 +72,7 @@ func init() { } func (man *SAnsiblePlaybookV2Manager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - data.Set("status", jsonutils.NewString(AnsiblePlaybookStatusInit)) + data.Set("status", jsonutils.NewString(apis.AnsiblePlaybookStatusInit)) return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) } @@ -86,14 +87,14 @@ func (apb *SAnsiblePlaybookV2) PostCreate(ctx context.Context, userCred mcclient func (man *SAnsiblePlaybookV2Manager) InitializeData() error { pbs := []SAnsiblePlaybookV2{} q := AnsiblePlaybookV2Manager.Query() - q = q.Filter(sqlchemy.Equals(q.Field("status"), AnsiblePlaybookStatusRunning)) + q = q.Filter(sqlchemy.Equals(q.Field("status"), apis.AnsiblePlaybookStatusRunning)) if err := db.FetchModelObjects(AnsiblePlaybookV2Manager, q, &pbs); err != nil { return errors.WithMessage(err, "fetch running playbooks") } for i := 0; i < len(pbs); i++ { pb := &pbs[i] _, err := db.Update(pb, func() error { - pb.Status = AnsiblePlaybookStatusUnknown + pb.Status = apis.AnsiblePlaybookStatusUnknown return nil }) if err != nil { @@ -104,7 +105,7 @@ func (man *SAnsiblePlaybookV2Manager) InitializeData() error { } func (apb *SAnsiblePlaybookV2) ValidateDeleteCondition(ctx context.Context) error { - if apb.Status == AnsiblePlaybookStatusRunning { + if apb.Status == apis.AnsiblePlaybookStatusRunning { return httperrors.NewConflictError("playbook is in running state") } return nil @@ -173,7 +174,7 @@ func (apb *SAnsiblePlaybookV2) runPlaybook(ctx context.Context, userCred mcclien apb.StartTime = time.Now() apb.EndTime = time.Time{} apb.Output = "" - apb.Status = AnsiblePlaybookStatusRunning + apb.Status = apis.AnsiblePlaybookStatusRunning return nil }) if err != nil { @@ -200,12 +201,12 @@ func (apb *SAnsiblePlaybookV2) runPlaybook(ctx context.Context, userCred mcclien _, err := db.Update(apb, func() error { err := man.sessions.Err(apb.Id) if err != nil { - apb.Status = AnsiblePlaybookStatusCanceled + apb.Status = apis.AnsiblePlaybookStatusCanceled } else if runErr != nil { log.Warningf("playbook %s(%s) failed: %v", apb.Name, apb.Id, runErr) - apb.Status = AnsiblePlaybookStatusFailed + apb.Status = apis.AnsiblePlaybookStatusFailed } else { - apb.Status = AnsiblePlaybookStatusSucceeded + apb.Status = apis.AnsiblePlaybookStatusSucceeded } apb.EndTime = time.Now() return nil diff --git a/pkg/ansibleserver/models/status.go b/pkg/apis/ansible/status.go similarity index 98% rename from pkg/ansibleserver/models/status.go rename to pkg/apis/ansible/status.go index 5d65d67e4e..dc26bc9382 100644 --- a/pkg/ansibleserver/models/status.go +++ b/pkg/apis/ansible/status.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package models +package ansible const ( AnsiblePlaybookStatusInit = "init" diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 3183bc6228..1bd79a1b62 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -1021,7 +1021,6 @@ func _doCreateItem( if err != nil { return nil, httperrors.NewGeneralError(err) } - // run name validation after validate create data parentId := manager.FetchParentId(ctx, dataDict) name, _ := dataDict.GetString("name") diff --git a/pkg/devtool/models/actions.go b/pkg/devtool/models/actions.go new file mode 100644 index 0000000000..0b9c4ec7b0 --- /dev/null +++ b/pkg/devtool/models/actions.go @@ -0,0 +1,235 @@ +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + apis "yunion.io/x/onecloud/pkg/apis/ansible" + dev_options "yunion.io/x/onecloud/pkg/devtool/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func getServerAttrs(ID string, s *mcclient.ClientSession) (map[string]string, error) { + keys := []string{ + "hypervisor", + "id", + "ips", + "name", + "region_id", + "zone_id", + } + re, err := modules.Servers.List(s, nil) + if err != nil { + log.Errorf("Error List server: %s", err) + return nil, err + } + + params := make(map[string]string) + result, err := modules.Servers.Get(s, ID, nil) + if err != nil { + log.Errorf("Error show server: %s", err) + return nil, err + } + for _, key := range keys { + value, _ := result.GetString(key) + if key == "ips" { + key = "ip" + if strings.Contains(value, ",") { + value = strings.Split(value, ",")[0] + } + } + params["server_"+key] = value + } + return params, err +} + +func getInfluxdbURL() (string, error) { + + s := auth.GetAdminSession(nil, "", "") + url, err := s.GetServiceURL("influxdb", dev_options.Options.SessionEndpointType) + + if err != nil { + log.Errorf("get influxdb Endpoint error %s", err) + return "", err + } + return url, nil +} + +func renderExtraVars(vars map[string]string) { + + InfluxdbURL, err := getInfluxdbURL() + if err != nil { + log.Errorf("template binding: get influxdb url error: %s", err) + return + } + for key, value := range vars { + if key == "influxdb" && value == "INFLUXDB" { + vars[key] = InfluxdbURL + } + } +} + +func deleteAnsiblePlaybook(id string, s *mcclient.ClientSession) (jsonutils.JSONObject, error) { + apb, err := modules.AnsiblePlaybooks.Get(s, id, nil) + if err != nil { + log.Errorf("[deleteAnsiblePlaybook] get Ansible playbook error %s", err) + return apb, err + } + status, _ := apb.GetString("status") + if status == apis.AnsiblePlaybookStatusRunning { + apb, err = modules.AnsiblePlaybooks.PerformAction(s, id, "stop", nil) + if err != nil { + log.Errorf("[deleteAnsiblePlaybook] stop Ansible playbook error %s", err) + return apb, err + } + } + + apb, err = modules.AnsiblePlaybooks.Delete(s, id, nil) + if err != nil { + log.Errorf("[deleteAnsiblePlaybook] Delete Ansible playbook error %s", err) + return apb, err + } + log.Debugf("ansible playbook %s has been deleted successfully.", id) + return apb, nil +} + +func (obj *SDevtoolTemplate) Binding(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // * get server id + // * get playbook struct and create obj + // * get cronjob struct and create obj + // * create playbook + template := obj + s := auth.GetSession(ctx, userCred, "", "") + ServerID, err := data.GetString("server_id") + + attrs, err := getServerAttrs(ServerID, s) + if err != nil { + log.Errorf("TemplateBindingServers getServerAttrs failed %s", err) + return nil, err + } + newPlaybookName := template.Name + "-" + template.Id[0:8] + "-" + ServerID[0:8] + if len(newPlaybookName) > 32 { + newPlaybookName = newPlaybookName[0:32] + } + + playbook := template.Playbook + playbook.Inventory.Hosts[0].Name = ServerID + + for key, value := range attrs { + playbook.Inventory.Hosts[0].Vars[key] = value + } + + renderExtraVars(playbook.Inventory.Hosts[0].Vars) + params := jsonutils.Marshal(&playbook) + + host, _ := params.Get("inventory") + file, _ := params.Get("files") + mod, _ := params.Get("modules") + + newAnsiblPlaybookParams := jsonutils.NewDict() + newAnsiblPlaybookParams.Add(jsonutils.NewString(newPlaybookName), "name") + newAnsiblPlaybookParams.Add(host, "playbook", "inventory") + newAnsiblPlaybookParams.Add(file, "playbook", "files") + newAnsiblPlaybookParams.Add(mod, "playbook", "modules") + + apb, err := modules.AnsiblePlaybooks.Create(s, newAnsiblPlaybookParams) + if err != nil { + log.Errorf("TemplateBindingServers AnsiblePlaybooks.Create failed %s", err) + return nil, err + } + ansibleId, _ := apb.GetString("id") + + //get cronjob struct and create template + newCronjobName := template.Name + "-" + template.Id[0:8] + "-" + ansibleId[0:8] + if len(newCronjobName) > 32 { + newCronjobName = newCronjobName[0:32] + } + + newCronjobParams := jsonutils.NewDict() + newCronjobParams.Add(jsonutils.NewString(newCronjobName), "name") + newCronjobParams.Add(jsonutils.NewInt(int64(template.Day)), "day") + newCronjobParams.Add(jsonutils.NewInt(int64(template.Hour)), "hour") + newCronjobParams.Add(jsonutils.NewInt(int64(template.Min)), "min") + newCronjobParams.Add(jsonutils.NewInt(int64(template.Sec)), "sec") + newCronjobParams.Add(jsonutils.NewInt(int64(template.Interval)), "interval") + newCronjobParams.Add(jsonutils.NewBool(template.Start), "start") + newCronjobParams.Add(jsonutils.NewBool(template.Enabled), "enabled") + newCronjobParams.Add(jsonutils.NewString(ansibleId), "ansible_playbook_id") + newCronjobParams.Add(jsonutils.NewString(template.Id), "template_id") + newCronjobParams.Add(jsonutils.NewString(ServerID), "server_id") + + _, err = modules.DevToolCronjobs.Create(s, newCronjobParams) + if err != nil { + log.Errorf("TemplateBindingServers failed %s", err) + return nil, err + } + return nil, nil +} + +func (obj *SDevtoolTemplate) Unbinding(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // * get server id + // * get playbook struct and create obj + // * get cronjob struct and create obj + // * create playbook + template := obj + s := auth.GetSession(ctx, userCred, "", "") + ServerID, err := data.GetString("server_id") + + newPlaybookName := template.Name + "-" + template.Id[0:8] + "-" + ServerID[0:8] + if len(newPlaybookName) > 32 { + newPlaybookName = newPlaybookName[0:32] + } + + apb, err := deleteAnsiblePlaybook(newPlaybookName, s) + if err != nil { + log.Errorf("TemplateUnbindingServers failed %s", err) + return nil, err + } + ansibleId, _ := apb.GetString("id") + newCronjobName := template.Name + "-" + template.Id[0:8] + "-" + ansibleId[0:8] + + if len(newCronjobName) > 32 { + newCronjobName = newCronjobName[0:32] + } + + _, err = modules.DevToolCronjobs.Delete(s, newCronjobName, nil) + if err != nil { + log.Errorf("err: %+v", err) + log.Errorf("TemplateUnbindingServers failed %s", err) + return nil, err + } + return nil, nil +} + +func (obj *SDevtoolTemplate) TaskUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + + template := obj + template.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data) + + opt := options.DevtoolTemplateUpdateOptions{} + data.Unmarshal(&opt) + if !opt.Rebind { + return nil, nil + } + + items := make([]SCronjob, 0) + q := CronjobManager.Query().Equals("template_id", template.Id) + err := q.All(&items) + if err != nil { + log.Errorf("query error: %s", err) + return nil, err + } + for _, item := range items { + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(item.ServerID), "server_id") + template.Unbinding(ctx, userCred, nil, params) + template.Binding(ctx, userCred, nil, params) + } + return nil, nil +} diff --git a/pkg/devtool/models/devtoolcronjob.go b/pkg/devtool/models/devtoolcronjob.go new file mode 100644 index 0000000000..bed4b07592 --- /dev/null +++ b/pkg/devtool/models/devtoolcronjob.go @@ -0,0 +1,137 @@ +package models + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/cronman" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type SVSCronjob struct { + Day int `json:"day" nullable:"true" create:"optional" list:"user" update:"user" default:"0"` + Hour int `nullable:"true" create:"optional" list:"user" update:"user" default:"0"` + Min int `nullable:"true" create:"optional" list:"user" update:"user" default:"0"` + Sec int `nullable:"true" create:"optional" list:"user" update:"user" default:"0"` + Interval int64 `nullable:"true" create:"optional" list:"user" update:"user" default:"0"` + Start bool `nullable:"true" create:"optional" list:"user" update:"user" default:"false"` + Enabled bool `nullable:"true" create:"optional" list:"user" update:"user" default:"false"` +} + +// TODO: 为 AnsiblePlaybookID, TemplateID, ServerID 加索引 +type SCronjob struct { + SVSCronjob + AnsiblePlaybookID string `width:"36" nullable:"false" create:"required" index:"true" list:"user" update:"user"` + TemplateID string `width:"36" nullable:"true" create:"optional" index:"true" list:"user" update:"user"` + ServerID string `width:"36" nullable:"true" create:"optional" index:"true" list:"user" update:"user"` + db.SStandaloneResourceBase +} + +type SCronjobManager struct { + db.SStandaloneResourceBaseManager +} + +var ( + CronjobManager *SCronjobManager + DevToolCronManager *cronman.SCronJobManager + Session *mcclient.ClientSession +) + +func init() { + CronjobManager = &SCronjobManager{ + SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( + SCronjob{}, + "devtool_cronjobs_tbl", + "devtool_cronjob", + "devtool_cronjobs", + ), + } + CronjobManager.SetVirtualObject(CronjobManager) + db.RegisterModelManager(CronjobManager) + DevToolCronManager = cronman.InitCronJobManager(true, 8) + DevToolCronManager.Start() +} + +func RunAnsibleCronjob(id string, s *mcclient.ClientSession) cronman.TCronJobFunction { + return func(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + obj, err := CronjobManager.FetchById(id) + if err != nil { + log.Errorf("No cronjob with id: %s", id) + return + } + log.Debugf("[RunAnsibleCronjob] %+v: ", obj) + item := obj.(*SCronjob) + + log.Debugf("[RunAnsibleCronjob] perform ansible cronjob run: %s", item.AnsiblePlaybookID) + ret, err := modules.AnsiblePlaybooks.PerformAction(s, item.AnsiblePlaybookID, "run", nil) + if err != nil { + log.Errorf("AnsiblePlaybooks.PerformAction error: %s", err) + } + log.Debugf("AnsiblePlaybooks.PerformAction ret: %+v", ret) + } +} + +func AddOneCronjob(item *SCronjob, s *mcclient.ClientSession) error { + + if !item.Enabled { + log.Debugf("ansible cronjob %s (devtool item.Id: %s) is not enabled", item.Name, item.Id) + return nil + } + if item.Interval > 0 { + err := DevToolCronManager.AddJobAtIntervalsWithStartRun(item.Id, time.Duration(item.Interval)*time.Second, RunAnsibleCronjob(item.Id, s), item.Start) + if err != nil { + log.Errorf("ansible cronjob %s (devtool item.Id: %s) error! %s", item.Name, item.Id, err) + return err + } + log.Infof("ansible cronjob %s (devtool item.Id: %s) registered at item.Interval: %ds", item.Name, item.Id, item.Interval) + } else { + err := DevToolCronManager.AddJobEveryFewDays(item.Id, int(item.Day), int(item.Hour), int(item.Min), int(item.Sec), RunAnsibleCronjob(item.Id, s), item.Start) + if err != nil { + log.Errorf("ansible cronjob %s (devtool item.Id: %s) registered at item.Interval: item.Day(%d) item.Hour(%d) item.Min(%d) item.Sec(%d) error: %s", item.Name, item.Id, int(item.Day), int(item.Hour), int(item.Min), int(item.Sec), err) + return err + } + log.Infof("ansible cronjob %s (devtool item.Id: %s) registered at item.Interval: item.Day(%d) item.Hour(%d) item.Min(%d) item.Sec(%d)", item.Name, item.Id, int(item.Day), int(item.Hour), int(item.Min), int(item.Sec)) + } + return nil +} + +func (man *SCronjobManager) InitializeData() error { + if Session == nil { + Session = auth.GetAdminSession(nil, "", "") + } + + go func() { + items := make([]SCronjob, 0) + q := CronjobManager.Query().Equals("enabled", true) + err := q.All(&items) + if err != nil { + log.Errorf("query error: %s", err) + } + for _, item := range items { + AddOneCronjob(&item, Session) + } + }() + + return nil +} + +func (job *SCronjob) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerID mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + job.SStandaloneResourceBase.PostCreate(ctx, userCred, nil, query, data) + AddOneCronjob(job, Session) +} + +func (job *SCronjob) PostDelete(ctx context.Context, userCred mcclient.TokenCredential) { + DevToolCronManager.Remove(job.Id) +} + +func (job *SCronjob) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + job.SStandaloneResourceBase.PostUpdate(ctx, userCred, query, data) + DevToolCronManager.Remove(job.Id) + AddOneCronjob(job, Session) +} diff --git a/pkg/devtool/models/devtooltemplate.go b/pkg/devtool/models/devtooltemplate.go new file mode 100644 index 0000000000..67cbe5d9ef --- /dev/null +++ b/pkg/devtool/models/devtooltemplate.go @@ -0,0 +1,79 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/ansible" +) + +type SDevtoolTemplate struct { + SVSCronjob + Playbook *ansible.Playbook `length:"text" nullable:"false" create:"required" get:"user" update:"user"` + db.SVirtualResourceBase +} + +type SDevtoolTemplateManager struct { + db.SVirtualResourceBaseManager +} + +var ( + DevtoolTemplateManager *SDevtoolTemplateManager +) + +func init() { + DevtoolTemplateManager = &SDevtoolTemplateManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SDevtoolTemplate{}, + "devtool_templates_tbl", + "devtool_template", + "devtool_templates", + ), + } + DevtoolTemplateManager.SetVirtualObject(DevtoolTemplateManager) + db.RegisterModelManager(DevtoolTemplateManager) +} + +func (obj *SDevtoolTemplate) PerformBind(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // * get server id + // * get playbook struct and create obj + // * get cronjob struct and create obj + // * create playbook + // taskman.TaskManager.NewTask(ctx, "KVMGuestRebuildRootTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + + task, err := taskman.TaskManager.NewTask(ctx, "TemplateBindingServers", obj, userCred, data.(*jsonutils.JSONDict), "", "", nil) + if err != nil { + log.Errorf("register task TemplateBindingServers error %s", err) + return nil, err + } + task.ScheduleRun(nil) + return nil, nil +} + +func (obj *SDevtoolTemplate) PerformUnbind(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + // * get server id + // * stop and delete playbook + // * stop and delete cronjob + + task, err := taskman.TaskManager.NewTask(ctx, "TemplateUnbindingServers", obj, userCred, data.(*jsonutils.JSONDict), "", "", nil) + if err != nil { + log.Errorf("register task TemplateUnbindingServers error %s", err) + return nil, err + } + task.ScheduleRun(nil) + return nil, nil +} + +func (obj *SDevtoolTemplate) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + obj.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data) + task, err := taskman.TaskManager.NewTask(ctx, "TemplateUpdate", obj, userCred, data.(*jsonutils.JSONDict), "", "", nil) + if err != nil { + log.Errorf("register task TemplateUpdate error %s", err) + } + task.ScheduleRun(nil) +} diff --git a/pkg/devtool/models/doc.go b/pkg/devtool/models/doc.go new file mode 100644 index 0000000000..2b0595648b --- /dev/null +++ b/pkg/devtool/models/doc.go @@ -0,0 +1 @@ +package models // import "yunion.io/x/onecloud/pkg/devtool/models" diff --git a/pkg/devtool/models/initdb.go b/pkg/devtool/models/initdb.go new file mode 100644 index 0000000000..bf22f7dd90 --- /dev/null +++ b/pkg/devtool/models/initdb.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +func InitDB() error { + for _, manager := range []db.IModelManager{ + CronjobManager, + DevtoolTemplateManager, + } { + err := manager.InitializeData() + if err != nil { + log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err) + return err + } else { + log.Infof("Manager %s initializeData PASS!", manager.Keyword()) + } + } + return nil +} diff --git a/pkg/devtool/options/doc.go b/pkg/devtool/options/doc.go new file mode 100644 index 0000000000..51dd7b9643 --- /dev/null +++ b/pkg/devtool/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/devtool/options" diff --git a/pkg/devtool/options/option.go b/pkg/devtool/options/option.go new file mode 100644 index 0000000000..ead0094174 --- /dev/null +++ b/pkg/devtool/options/option.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + +type DevToolOptions struct { + common_options.CommonOptions + common_options.DBOptions +} + +var ( + Options DevToolOptions +) diff --git a/pkg/devtool/service/doc.go b/pkg/devtool/service/doc.go new file mode 100644 index 0000000000..6feab83a63 --- /dev/null +++ b/pkg/devtool/service/doc.go @@ -0,0 +1 @@ +package service // import "yunion.io/x/onecloud/pkg/devtool/service" diff --git a/pkg/devtool/service/handler.go b/pkg/devtool/service/handler.go new file mode 100644 index 0000000000..0493db6be9 --- /dev/null +++ b/pkg/devtool/service/handler.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/appsrv/dispatcher" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/devtool/models" +) + +func InitHandlers(app *appsrv.Application) { + db.InitAllManagers() + taskman.AddTaskHandler("", app) + + for _, manager := range []db.IModelManager{ + taskman.TaskManager, + taskman.SubTaskManager, + taskman.TaskObjectManager, + db.UserCacheManager, + db.TenantCacheManager, + } { + db.RegisterModelManager(manager) + } + + for _, manager := range []db.IModelManager{ + models.CronjobManager, + models.DevtoolTemplateManager, + } { + db.RegisterModelManager(manager) + handler := db.NewModelHandler(manager) + dispatcher.AddModelDispatcher("", app, handler) + } +} diff --git a/pkg/devtool/service/service.go b/pkg/devtool/service/service.go new file mode 100644 index 0000000000..1f5334ace2 --- /dev/null +++ b/pkg/devtool/service/service.go @@ -0,0 +1,58 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "os" + + _ "github.com/go-sql-driver/mysql" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" + "yunion.io/x/onecloud/pkg/devtool/models" + "yunion.io/x/onecloud/pkg/devtool/options" + _ "yunion.io/x/onecloud/pkg/devtool/tasks" +) + +// StartService the main service starts +func StartService() { + opts := &options.Options + commonOpts := &opts.CommonOptions + dbOpts := &options.Options.DBOptions + baseOpts := &opts.BaseOptions + common_options.ParseOptions(opts, os.Args, "devtool.conf", "devtool") + + app_common.InitAuth(commonOpts, func() { + log.Infof("Auth complete!!") + }) + + cloudcommon.InitDB(&opts.DBOptions) + defer cloudcommon.CloseDB() + + if !db.CheckSync(opts.AutoSyncTable) { + log.Fatalf("database schema not in sync!") + } + + app := app_common.InitApp(&opts.BaseOptions, false) + InitHandlers(app) + db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + app_common.ServeForeverWithCleanup(app, baseOpts, func() { + cloudcommon.CloseDB() + }) +} diff --git a/pkg/devtool/tasks/doc.go b/pkg/devtool/tasks/doc.go new file mode 100644 index 0000000000..60003b65f6 --- /dev/null +++ b/pkg/devtool/tasks/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks // import "yunion.io/x/onecloud/pkg/devtool/tasks" diff --git a/pkg/devtool/tasks/template_binding_servers.go b/pkg/devtool/tasks/template_binding_servers.go new file mode 100644 index 0000000000..a913ad41d9 --- /dev/null +++ b/pkg/devtool/tasks/template_binding_servers.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed unde3r the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/devtool/models" +) + +type TemplateBindingServers struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(TemplateBindingServers{}) +} + +func (self *TemplateBindingServers) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + + template := obj.(*models.SDevtoolTemplate) + _, err := template.Binding(ctx, self.UserCred, nil, self.Params) + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("TemplateUpdate failed %s", err)) + } else { + self.SetStageComplete(ctx, nil) + } +} diff --git a/pkg/devtool/tasks/template_unbinding_servers.go b/pkg/devtool/tasks/template_unbinding_servers.go new file mode 100644 index 0000000000..c34095cef7 --- /dev/null +++ b/pkg/devtool/tasks/template_unbinding_servers.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed unde3r the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/devtool/models" +) + +type TemplateUnbindingServers struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(TemplateUnbindingServers{}) +} + +func (self *TemplateUnbindingServers) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + + template := obj.(*models.SDevtoolTemplate) + _, err := template.Unbinding(ctx, self.UserCred, nil, self.Params) + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("TemplateUnBindingServers failed %s", err)) + } else { + self.SetStageComplete(ctx, nil) + } +} diff --git a/pkg/devtool/tasks/template_update.go b/pkg/devtool/tasks/template_update.go new file mode 100644 index 0000000000..c7b7c9833a --- /dev/null +++ b/pkg/devtool/tasks/template_update.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed unde3r the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/devtool/models" +) + +type TemplateUpdate struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(TemplateUpdate{}) +} + +func (self *TemplateUpdate) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + + template := obj.(*models.SDevtoolTemplate) + _, err := template.TaskUpdate(ctx, self.UserCred, nil, self.Params) + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("TemplateUpdate failed %s", err)) + } else { + self.SetStageComplete(ctx, nil) + } +} diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go index 18c15879be..708da7dd9e 100644 --- a/pkg/mcclient/modules/managers.go +++ b/pkg/mcclient/modules/managers.go @@ -163,3 +163,9 @@ func NewAnsibleManager(keyword, keywordPlural string, columns, adminColumns []st BaseManager: *modulebase.NewBaseManager("ansible", "", "", columns, adminColumns), Keyword: keyword, KeywordPlural: keywordPlural} } + +func NewDevtoolManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager { + return modulebase.ResourceManager{ + BaseManager: *modulebase.NewBaseManager("devtool", "", "", columns, adminColumns), + Keyword: keyword, KeywordPlural: keywordPlural} +} diff --git a/pkg/mcclient/modules/mod_devtools.go b/pkg/mcclient/modules/mod_devtools.go new file mode 100644 index 0000000000..f5d16d71b9 --- /dev/null +++ b/pkg/mcclient/modules/mod_devtools.go @@ -0,0 +1,43 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +var ( + DevToolCronjobs modulebase.ResourceManager + DevToolTemplates modulebase.ResourceManager +) + +func init() { + + DevToolCronjobs = NewDevtoolManager( + "devtool_cronjob", + "devtool_cronjobs", + []string{"id", "ansible_playbook_id", "template_id", "server_id", "name", "day", "hour", "min", "sec", "interval", "start", "enabled", "created_at"}, + []string{}, + ) + registerCompute(&DevToolCronjobs) + + DevToolTemplates = NewDevtoolManager( + "devtool_template", + "devtool_templates", + []string{"id", "name", "domain_id", "tenant_id", "day", "hour", "min", "sec", "interval", "start", "enabled", "description"}, + []string{"is_system"}, + ) + registerCompute(&DevToolTemplates) +} diff --git a/pkg/mcclient/options/devtooltemplate.go b/pkg/mcclient/options/devtooltemplate.go new file mode 100644 index 0000000000..06db1f5dd9 --- /dev/null +++ b/pkg/mcclient/options/devtooltemplate.go @@ -0,0 +1,159 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "fmt" + "io/ioutil" + "strings" + + "yunion.io/x/jsonutils" + + apis "yunion.io/x/onecloud/pkg/apis/ansible" + "yunion.io/x/onecloud/pkg/util/ansible" +) + +type DevtoolTemplateIdOptions struct { + ID string `help:"name/id of the playbook"` +} + +type DevtoolTemplateBindingOptions struct { + DevtoolTemplateIdOptions + ServerID string `help:"host/vm name/id to apply"` +} + +type DevtoolTemplateListOptions struct { + BaseListOptions +} + +type DevtoolTemplateCronjobOptions struct { + Day int `help:"Cronjob runs at given day" default:"0"` + Hour int `help:"Cronjob runs at given hour" default:"0"` + Min int `help:"Cronjob runs at given min" default:"0"` + Sec int `help:"Cronjob runs at given sec" default:"0"` + Interval int64 `help:"Cronjob runs at given interval" default:"0"` + Start bool `help:"start job when created" default:"false"` + Enabled bool `help:"Set job status enabled" default:"false"` +} + +type DevtoolTemplateCommonOptions struct { + DevtoolTemplateCronjobOptions + Host []string `help:"name or id of server or host in format '<[server:]id|host:id>|ipaddr var=val'"` + Mod []string `help:"ansible modules and their arguments in format 'name k1=v1 k2=v2'"` + File []string `help:"files for use by modules, e.g. name=content, name=@file"` +} + +func (opts *DevtoolTemplateCommonOptions) ToPlaybook() (*ansible.Playbook, error) { + if len(opts.Mod) == 0 { + return nil, fmt.Errorf("Requires at least one --mod argument") + } + if len(opts.Host) == 0 { + return nil, fmt.Errorf("Requires at least one server/host to operate on") + } + pb := ansible.NewPlaybook() + hosts := []ansible.Host{} + for _, s := range opts.Host { + host, err := ansible.ParseHostLine(s) + if err != nil { + return nil, err + } + hosts = append(hosts, host) + } + pb.Inventory = ansible.Inventory{Hosts: hosts} + for _, s := range opts.Mod { + module, err := ansible.ParseModuleLine(s) + if err != nil { + return nil, err + } + pb.Modules = append(pb.Modules, module) + } + files := map[string][]byte{} + for _, s := range opts.File { + i := strings.IndexByte(s, '=') + if i < 0 { + return nil, fmt.Errorf("missing '=' in argument for --file. Read command help") + } + name := strings.TrimSpace(s[:i]) + if name == "" { + return nil, fmt.Errorf("empty file name: %s", s) + } + v := s[i+1:] + if len(v) > 0 && v[0] == '@' { + path := v[1:] + d, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read file %s: %v", path, err) + } + files[name] = d + } else { + files[name] = []byte(v) + } + } + pb.Files = files + return pb, nil +} + +type DevtoolTemplateCreateOptions struct { + NAME string `help:"name of the playbook"` + DevtoolTemplateCommonOptions +} + +func (opts *DevtoolTemplateCreateOptions) Params() (*jsonutils.JSONDict, error) { + pb, err := opts.DevtoolTemplateCommonOptions.ToPlaybook() + if err != nil { + return nil, err + } + input := &apis.AnsiblePlaybookCreateInput{ + Name: opts.NAME, + Playbook: *pb, + } + params := input.JSON(input) + params.Add(jsonutils.NewInt(int64(opts.Day)), "day") + params.Add(jsonutils.NewInt(int64(opts.Hour)), "hour") + params.Add(jsonutils.NewInt(int64(opts.Min)), "min") + params.Add(jsonutils.NewInt(int64(opts.Sec)), "sec") + params.Add(jsonutils.NewInt(opts.Interval), "interval") + params.Add(jsonutils.NewBool(opts.Start), "start") + params.Add(jsonutils.NewBool(opts.Enabled), "enabled") + return params, nil +} + +type DevtoolTemplateUpdateOptions struct { + ID string `json:"-" help:"name/id of the playbook"` + Name string + DevtoolTemplateCommonOptions + Rebind bool `help:"Unbind and Bind all related servers" default:"false"` +} + +func (opts *DevtoolTemplateUpdateOptions) Params() (*jsonutils.JSONDict, error) { + pb, err := opts.DevtoolTemplateCommonOptions.ToPlaybook() + if err != nil { + return nil, err + } + input := &apis.AnsiblePlaybookUpdateInput{ + Name: opts.Name, + Playbook: *pb, + } + params := input.JSON(input) + params.Add(jsonutils.NewBool(opts.Rebind), "rebind") + params.Add(jsonutils.NewInt(int64(opts.Day)), "day") + params.Add(jsonutils.NewInt(int64(opts.Hour)), "hour") + params.Add(jsonutils.NewInt(int64(opts.Min)), "min") + params.Add(jsonutils.NewInt(int64(opts.Sec)), "sec") + params.Add(jsonutils.NewInt(opts.Interval), "interval") + params.Add(jsonutils.NewBool(opts.Start), "start") + params.Add(jsonutils.NewBool(opts.Enabled), "enabled") + return params, nil +} From b6fa205f184ad6ad990a4a823e4ed6dad75b537a Mon Sep 17 00:00:00 2001 From: Zhang Dongliang Date: Wed, 27 Nov 2019 15:15:02 +0800 Subject: [PATCH 2/3] rm unused re code --- pkg/devtool/models/actions.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/devtool/models/actions.go b/pkg/devtool/models/actions.go index 0b9c4ec7b0..37ab60171e 100644 --- a/pkg/devtool/models/actions.go +++ b/pkg/devtool/models/actions.go @@ -24,12 +24,6 @@ func getServerAttrs(ID string, s *mcclient.ClientSession) (map[string]string, er "region_id", "zone_id", } - re, err := modules.Servers.List(s, nil) - if err != nil { - log.Errorf("Error List server: %s", err) - return nil, err - } - params := make(map[string]string) result, err := modules.Servers.Get(s, ID, nil) if err != nil { From 0366347728bbafd7ad93c9ea88429832a689340b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 27 Nov 2019 15:06:33 +0800 Subject: [PATCH 3/3] fix caller input nil panic --- pkg/devtool/models/actions.go | 3 +-- pkg/devtool/models/devtoolcronjob.go | 11 +++++------ pkg/devtool/service/service.go | 3 +++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/devtool/models/actions.go b/pkg/devtool/models/actions.go index 37ab60171e..937cc0dfd5 100644 --- a/pkg/devtool/models/actions.go +++ b/pkg/devtool/models/actions.go @@ -8,7 +8,6 @@ import ( "yunion.io/x/log" apis "yunion.io/x/onecloud/pkg/apis/ansible" - dev_options "yunion.io/x/onecloud/pkg/devtool/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -46,7 +45,7 @@ func getServerAttrs(ID string, s *mcclient.ClientSession) (map[string]string, er func getInfluxdbURL() (string, error) { s := auth.GetAdminSession(nil, "", "") - url, err := s.GetServiceURL("influxdb", dev_options.Options.SessionEndpointType) + url, err := s.GetServiceURL("influxdb", "") if err != nil { log.Errorf("get influxdb Endpoint error %s", err) diff --git a/pkg/devtool/models/devtoolcronjob.go b/pkg/devtool/models/devtoolcronjob.go index bed4b07592..8303a5ca7f 100644 --- a/pkg/devtool/models/devtoolcronjob.go +++ b/pkg/devtool/models/devtoolcronjob.go @@ -24,7 +24,6 @@ type SVSCronjob struct { Enabled bool `nullable:"true" create:"optional" list:"user" update:"user" default:"false"` } -// TODO: 为 AnsiblePlaybookID, TemplateID, ServerID 加索引 type SCronjob struct { SVSCronjob AnsiblePlaybookID string `width:"36" nullable:"false" create:"required" index:"true" list:"user" update:"user"` @@ -40,7 +39,6 @@ type SCronjobManager struct { var ( CronjobManager *SCronjobManager DevToolCronManager *cronman.SCronJobManager - Session *mcclient.ClientSession ) func init() { @@ -101,10 +99,9 @@ func AddOneCronjob(item *SCronjob, s *mcclient.ClientSession) error { return nil } -func (man *SCronjobManager) InitializeData() error { - if Session == nil { - Session = auth.GetAdminSession(nil, "", "") - } +func InitializeCronjobs() error { + + Session := auth.GetAdminSession(nil, "", "") go func() { items := make([]SCronjob, 0) @@ -122,6 +119,7 @@ func (man *SCronjobManager) InitializeData() error { } func (job *SCronjob) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerID mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + Session := auth.GetAdminSession(nil, "", "") job.SStandaloneResourceBase.PostCreate(ctx, userCred, nil, query, data) AddOneCronjob(job, Session) } @@ -131,6 +129,7 @@ func (job *SCronjob) PostDelete(ctx context.Context, userCred mcclient.TokenCred } func (job *SCronjob) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + Session := auth.GetAdminSession(nil, "", "") job.SStandaloneResourceBase.PostUpdate(ctx, userCred, query, data) DevToolCronManager.Remove(job.Id) AddOneCronjob(job, Session) diff --git a/pkg/devtool/service/service.go b/pkg/devtool/service/service.go index 1f5334ace2..29b1fa6f70 100644 --- a/pkg/devtool/service/service.go +++ b/pkg/devtool/service/service.go @@ -52,6 +52,9 @@ func StartService() { app := app_common.InitApp(&opts.BaseOptions, false) InitHandlers(app) db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB) + + models.InitializeCronjobs() + app_common.ServeForeverWithCleanup(app, baseOpts, func() { cloudcommon.CloseDB() })