Merge pull request #910 from yousong/feature/yousong-ansible

region: ansible服务化

Gopkg.lock

@@ -1060,12 +1060,12 @@
   version = "v2.0.7"
 
 [[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
+  digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b"
   name = "github.com/pkg/errors"
   packages = ["."]
   pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
 
 [[projects]]
   branch = "master"

cmd/ansibleserver/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/ansibleserver/service"
+)
+
+func main() {
+	service.StartService()
+}

cmd/climc/shell/ansible.go

@@ -94,7 +94,7 @@ func doList(s *mcclient.ClientSession, args *AnsibleHostsOptions) error {
 	output.Add(children, "all", "children")
 	output.Add(hosts, "ungrouped", "hosts")
 
-	fmt.Printf("%s", output.PrettyString())
+	fmt.Printf("%s\n", output.PrettyString())
 
 	return nil
 }
@@ -123,7 +123,7 @@ func doHost(s *mcclient.ClientSession, host string, args *AnsibleHostsOptions) e
 		hostVar.Add(jsonutils.NewString(args.UserBecome), "ansible_become_user")
 	}
 
-	fmt.Printf("%s", hostVar.PrettyString())
+	fmt.Printf("%s\n", hostVar.PrettyString())
 
 	return nil
 }

cmd/climc/shell/ansibleplaybooks.go

@@ -0,0 +1,108 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	//"fmt"
+	//"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	printAnsiblePlaybookObject := func(obj jsonutils.JSONObject) {
+		dict := obj.(*jsonutils.JSONDict)
+		pbJson, err := dict.Get("playbook")
+		if err != nil {
+			printObject(obj)
+			return
+		}
+		pbStr := pbJson.YAMLString()
+		dict.Set("playbook", jsonutils.NewString(pbStr))
+		printObject(obj)
+	}
+	R(&options.AnsiblePlaybookCreateOptions{}, "ansibleplaybook-create", "Create ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		apb, err := modules.AnsiblePlaybooks.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+	R(&options.AnsiblePlaybookIdOptions{}, "ansibleplaybook-show", "Show ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookIdOptions) error {
+		apb, err := modules.AnsiblePlaybooks.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+	R(&options.AnsiblePlaybookListOptions{}, "ansibleplaybook-list", "List ansible playbooks", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookListOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		apbs, err := modules.AnsiblePlaybooks.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(apbs, modules.AnsiblePlaybooks.GetColumns(s))
+		return nil
+	})
+	R(&options.AnsiblePlaybookIdOptions{}, "ansibleplaybook-delete", "Delete ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookIdOptions) error {
+		apb, err := modules.AnsiblePlaybooks.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+	R(&options.AnsiblePlaybookUpdateOptions{}, "ansibleplaybook-update", "Update ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookUpdateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		apb, err := modules.AnsiblePlaybooks.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+	R(&options.AnsiblePlaybookIdOptions{}, "ansibleplaybook-run", "Run ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookIdOptions) error {
+		apb, err := modules.AnsiblePlaybooks.PerformAction(s, opts.ID, "run", nil)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+	R(&options.AnsiblePlaybookIdOptions{}, "ansibleplaybook-stop", "Stop ansible playbook", func(s *mcclient.ClientSession, opts *options.AnsiblePlaybookIdOptions) error {
+		apb, err := modules.AnsiblePlaybooks.PerformAction(s, opts.ID, "stop", nil)
+		if err != nil {
+			return err
+		}
+		printAnsiblePlaybookObject(apb)
+		return nil
+	})
+}

pkg/ansibleserver/models/ansibleplaybooks.go

@@ -0,0 +1,257 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/compute/sshkeys"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/ansible"
+)
+
+const (
+	AnsiblePlaybookStatusInit      = "init"
+	AnsiblePlaybookStatusRunning   = "running"
+	AnsiblePlaybookStatusSucceeded = "succeeded"
+	AnsiblePlaybookStatusFailed    = "failed"
+	AnsiblePlaybookStatusCanceled  = "canceled"
+	AnsiblePlaybookStatusUnknown   = "unknown"
+)
+
+// low priority
+//
+// - retry times and interval
+// - timeout,
+// - copied playbook has no added value
+
+type SAnsiblePlaybook struct {
+	db.SVirtualResourceBase
+
+	Playbook  *ansible.Playbook `nullable:"false" create:"required" get:"user" update:"user"`
+	Output    string            `get:"user"`
+	StartTime time.Time         `list:"user"`
+	EndTime   time.Time         `list:"user"`
+}
+
+type SAnsiblePlaybookManager struct {
+	db.SVirtualResourceBaseManager
+
+	sessions    ansible.SessionManager
+	sessionsMux *sync.Mutex
+}
+
+var AnsiblePlaybookManager *SAnsiblePlaybookManager
+
+func init() {
+	AnsiblePlaybookManager = &SAnsiblePlaybookManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SAnsiblePlaybook{},
+			"ansibleplaybooks_tbl",
+			"ansibleplaybook",
+			"ansibleplaybooks",
+		),
+		sessions:    ansible.SessionManager{},
+		sessionsMux: &sync.Mutex{},
+	}
+}
+
+func (man *SAnsiblePlaybookManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	v := NewAnsiblePlaybookValidator("playbook", userCred)
+	if err := v.Validate(data); err != nil {
+		return nil, err
+	}
+	data.Set("status", jsonutils.NewString(AnsiblePlaybookStatusInit))
+	return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data)
+}
+
+func (apb *SAnsiblePlaybook) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	apb.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
+	err := apb.runPlaybook(ctx, userCred)
+	if err != nil {
+		log.Errorf("postCreate: runPlaybook: %v", err)
+	}
+}
+
+func (man *SAnsiblePlaybookManager) InitializeData() error {
+	pbs := []SAnsiblePlaybook{}
+	q := AnsiblePlaybookManager.Query()
+	q = q.Filter(sqlchemy.Equals(q.Field("status"), AnsiblePlaybookStatusRunning))
+	if err := db.FetchModelObjects(AnsiblePlaybookManager, q, &pbs); err != nil {
+		return errors.WithMessage(err, "fetch running playbooks")
+	}
+	for i := 0; i < len(pbs); i++ {
+		pb := &pbs[i]
+		_, err := db.Update(pb, func() error {
+			pb.Status = AnsiblePlaybookStatusUnknown
+			return nil
+		})
+		if err != nil {
+			log.Errorf("set playbook %s(%s) to unknown state: %v", pb.Name, pb.Id, err)
+		}
+	}
+	return nil
+}
+
+func (apb *SAnsiblePlaybook) ValidateDeleteCondition(ctx context.Context) error {
+	if apb.Status == AnsiblePlaybookStatusRunning {
+		return httperrors.NewConflictError("playbook is in running state")
+	}
+	return nil
+}
+
+func (apb *SAnsiblePlaybook) ValidateUpdateCondition(ctx context.Context) error {
+	if apb.Status == AnsiblePlaybookStatusRunning {
+		return httperrors.NewConflictError("playbook is in running state")
+	}
+	return nil
+}
+
+func (apb *SAnsiblePlaybook) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	pbV := NewAnsiblePlaybookValidator("playbook", userCred)
+	if err := pbV.Validate(data); err != nil {
+		return nil, err
+	}
+	apb.Playbook = pbV.Playbook // Update as a whole
+	data.Set("status", jsonutils.NewString(AnsiblePlaybookStatusInit))
+	return data, nil
+}
+
+func (apb *SAnsiblePlaybook) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	apb.SVirtualResourceBase.PostUpdate(ctx, userCred, query, data)
+	err := apb.runPlaybook(ctx, userCred)
+	if err != nil {
+		log.Errorf("postUpdate: runPlaybook: %v", err)
+	}
+}
+
+func (apb *SAnsiblePlaybook) AllowPerformRun(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return apb.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, apb, "run")
+}
+
+func (apb *SAnsiblePlaybook) PerformRun(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	err := apb.runPlaybook(ctx, userCred)
+	if err != nil {
+		return nil, httperrors.NewConflictError("%s", err.Error())
+	}
+	return nil, nil
+}
+
+func (apb *SAnsiblePlaybook) AllowPerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return apb.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, apb, "stop")
+}
+
+func (apb *SAnsiblePlaybook) PerformStop(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	err := apb.stopPlaybook(ctx, userCred)
+	if err != nil {
+		return nil, httperrors.NewConflictError("%s", err.Error())
+	}
+	return nil, nil
+}
+
+func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient.TokenCredential) error {
+	man := AnsiblePlaybookManager
+	man.sessionsMux.Lock()
+	if man.sessions.Has(apb.Id) {
+		man.sessionsMux.Unlock()
+		return fmt.Errorf("playbook is already running")
+	}
+
+	// init private key
+	pb := apb.Playbook.Copy()
+	if userCred.HasSystemAdminPrivilege() {
+		k, _, err := sshkeys.GetSshAdminKeypair(ctx)
+		if err != nil {
+			return errors.WithMessage(err, "get admin ssh key")
+		}
+		pb.PrivateKey = []byte(k)
+	} else {
+		k, _, err := sshkeys.GetSshProjectKeypair(ctx, userCred.GetTenantId())
+		if err != nil {
+			return errors.WithMessage(err, "get project ssh key")
+		}
+		pb.PrivateKey = []byte(k)
+	}
+
+	man.sessions.Add(apb.Id, pb)
+	man.sessionsMux.Unlock()
+
+	_, err := db.Update(apb, func() error {
+		apb.StartTime = time.Now()
+		apb.EndTime = time.Time{}
+		apb.Output = ""
+		apb.Status = AnsiblePlaybookStatusRunning
+		return nil
+	})
+	if err != nil {
+		log.Errorf("run playbook: update db failed before run: %v", err)
+	}
+	// NOTE host state check? run only on online hosts and running guests, skip others
+	go func() {
+		defer func() {
+			man.sessionsMux.Lock()
+			defer man.sessionsMux.Unlock()
+			man.sessions.Remove(apb.Id)
+		}()
+		runErr := man.sessions.Run(apb.Id)
+
+		_, err := db.Update(apb, func() error {
+			err := man.sessions.Err(apb.Id)
+			if err != nil {
+				apb.Status = AnsiblePlaybookStatusCanceled
+			} else if runErr != nil {
+				apb.Status = AnsiblePlaybookStatusFailed
+			} else {
+				apb.Status = AnsiblePlaybookStatusSucceeded
+			}
+			apb.EndTime = time.Now()
+			// truncate to preserve the tail
+			output := pb.Output()
+			textMax := 64*1024 - 1
+			if len(output) > textMax {
+				output = output[len(output)-textMax:]
+			}
+			apb.Output = string(output)
+			return nil
+		})
+		if err != nil {
+			log.Errorf("updating ansible playbook failed: %v", err)
+		}
+	}()
+	return nil
+}
+
+func (apb *SAnsiblePlaybook) stopPlaybook(ctx context.Context, userCred mcclient.TokenCredential) error {
+	man := AnsiblePlaybookManager
+	man.sessionsMux.Lock()
+	defer man.sessionsMux.Unlock()
+	if !man.sessions.Has(apb.Id) {
+		return fmt.Errorf("playbook is not running")
+	}
+	// the playbook will be removed from session map in runPlaybook() on return from run
+	man.sessions.Stop(apb.Id)
+	return nil
+}

pkg/ansibleserver/models/ansibleplaybooks_validator.go

@@ -0,0 +1,129 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/regutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	mcclient "yunion.io/x/onecloud/pkg/mcclient"
+	mcclient_auth "yunion.io/x/onecloud/pkg/mcclient/auth"
+	mcclient_models "yunion.io/x/onecloud/pkg/mcclient/models"
+	mcclient_modules "yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/ansible"
+)
+
+type ValidatorAnsiblePlaybook struct {
+	validators.Validator
+	Playbook *ansible.Playbook
+
+	userCred mcclient.TokenCredential
+}
+
+func NewAnsiblePlaybookValidator(key string, userCred mcclient.TokenCredential) *ValidatorAnsiblePlaybook {
+	v := &ValidatorAnsiblePlaybook{
+		Validator: validators.Validator{Key: key},
+		userCred:  userCred,
+	}
+	v.SetParent(v)
+	return v
+}
+
+func (v *ValidatorAnsiblePlaybook) Validate(data *jsonutils.JSONDict) error {
+	pb := ansible.NewPlaybook()
+	err := data.Unmarshal(pb, "playbook")
+	if err != nil {
+		return httperrors.NewBadRequestError("unmarshaling json: %v", err)
+	}
+	hosts := pb.Inventory.Hosts
+	for i := range hosts {
+		name := strings.TrimSpace(hosts[i].Name)
+		if len(name) == 0 {
+			return httperrors.NewBadRequestError("empty host name")
+		}
+		switch {
+		case regutils.MatchIP4Addr(name):
+			continue
+		case strings.HasPrefix(name, "host:"):
+			var err error
+			name = strings.TrimSpace(name[len("host:"):])
+			name, err = v.getHostAccessIp(name)
+			if err != nil {
+				return err
+			}
+		case strings.HasPrefix(name, "server:"):
+			name = name[len("server:"):]
+			fallthrough
+		default:
+			var err error
+			name = strings.TrimSpace(name)
+			name, err = v.getServerIp(name)
+			if err != nil {
+				return err
+			}
+		}
+		hosts[i].Name = name
+		if username, _ := hosts[i].GetVar("ansible_user"); username == "" {
+			hosts[i].SetVar("ansible_user", ansible.PUBLIC_CLOUD_ANSIBLE_USER)
+		}
+	}
+	v.Playbook = pb
+	pbJson := jsonutils.Marshal(pb)
+	data.Set("playbook", pbJson)
+	return nil
+}
+
+func (v *ValidatorAnsiblePlaybook) getHostAccessIp(name string) (string, error) {
+	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "", "")
+	hostJson, err := mcclient_modules.Hosts.Get(sess, name, nil)
+	if err != nil {
+		return "", httperrors.NewBadRequestError("cannot find host %s", name)
+	}
+	host := &mcclient_models.Host{}
+	if err := hostJson.Unmarshal(host); err != nil {
+		return "", httperrors.NewBadRequestError("unmarshal host %s: %v", name, err)
+	}
+	if host.AccessIp == "" {
+		return "", httperrors.NewBadRequestError("host %s has no access ip", name)
+	}
+	return host.AccessIp, nil
+}
+
+func (v *ValidatorAnsiblePlaybook) getServerIp(name string) (string, error) {
+	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "", "")
+	serverJson, err := mcclient_modules.Servers.Get(sess, name, nil)
+	if err != nil {
+		return "", httperrors.NewBadRequestError("find server %s: %v", name, err)
+	}
+	server := &mcclient_models.Server{}
+	if err := serverJson.Unmarshal(server); err != nil {
+		return "", httperrors.NewBadRequestError("unmarshal server %s: %v", name, err)
+	}
+	serverNetworks, err := mcclient_models.ParseServerNetworkDetailedString(server.Networks)
+	if err != nil {
+		return "", httperrors.NewConflictError("parse networks of %s: %v", name, err)
+	}
+	ips := serverNetworks.GetPrivateIPs()
+	if len(ips) == 0 {
+		return "", httperrors.NewBadRequestError("server %s has no private ips", name)
+	}
+	name = ips[0].String()
+	return name, nil
+}

pkg/ansibleserver/models/initdb.go

@@ -0,0 +1,38 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitDB() error {
+	for _, manager := range []db.IModelManager{
+		/*
+		 * Important!!!
+		 * initialization order matters, do not change the order
+		 */
+		AnsiblePlaybookManager,
+	} {
+		err := manager.InitializeData()
+		if err != nil {
+			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
+			// return err skip error table
+		}
+	}
+	return nil
+}

pkg/ansibleserver/options/options.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+
+type AnsibleServerOptions struct {
+	common_options.CommonOptions
+	common_options.DBOptions
+}
+
+var (
+	Options AnsibleServerOptions
+)

pkg/ansibleserver/service/handlers.go

@@ -0,0 +1,34 @@
+package service
+
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import (
+	"yunion.io/x/onecloud/pkg/ansibleserver/models"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitHandlers(app *appsrv.Application) {
+	db.InitAllManagers()
+
+	for _, manager := range []db.IModelManager{
+		models.AnsiblePlaybookManager,
+	} {
+		db.RegisterModelManager(manager)
+		handler := db.NewModelHandler(manager)
+		dispatcher.AddModelDispatcher("", app, handler)
+	}
+}

pkg/ansibleserver/service/service.go

@@ -0,0 +1,59 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"os"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/ansibleserver/models"
+	"yunion.io/x/onecloud/pkg/ansibleserver/options"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	common_app "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
+
+func StartService() {
+	opts := &options.Options
+	common_options.ParseOptions(opts, os.Args, "ansibleserver.conf", "ansibleserver")
+
+	commonOpts := &opts.CommonOptions
+	common_app.InitAuth(commonOpts, func() {
+		log.Infof("Auth complete")
+	})
+
+	dbOpts := &opts.DBOptions
+	cloudcommon.InitDB(dbOpts)
+	defer cloudcommon.CloseDB()
+
+	baseOpts := &opts.BaseOptions
+	app := common_app.InitApp(baseOpts, false)
+	InitHandlers(app)
+
+	if !db.CheckSync(opts.AutoSyncTable) {
+		log.Fatalf("database schema not in sync!")
+	}
+
+	err := models.InitDB()
+	if err != nil {
+		log.Errorf("InitDB fail: %s", err)
+	}
+
+	common_app.ServeForever(app, baseOpts)
+}

pkg/cloudcommon/validators/validators.go

@@ -60,6 +60,11 @@ type Validator struct {
 	value      jsonutils.JSONObject
 }
 
+func (v *Validator) SetParent(parent IValidator) IValidator {
+	v.parent = parent
+	return parent
+}
+
 func (v *Validator) Optional(optional bool) IValidator {
 	v.optional = optional
 	return v.parent
@@ -132,7 +137,7 @@ func NewIPv4PrefixValidator(key string) *ValidatorIPv4Prefix {
 	v := &ValidatorIPv4Prefix{
 		Validator: Validator{Key: key},
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -164,7 +169,7 @@ func NewStringChoicesValidator(key string, choices choices.Choices) *ValidatorSt
 		Validator: Validator{Key: key},
 		Choices:   choices,
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -210,7 +215,7 @@ func NewStringMultiChoicesValidator(key string, choices choices.Choices) *Valida
 		Validator: Validator{Key: key},
 		Choices:   choices,
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -311,7 +316,7 @@ func NewBoolValidator(key string) *ValidatorBool {
 	v := &ValidatorBool{
 		Validator: Validator{Key: key},
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -354,7 +359,7 @@ func NewRangeValidator(key string, lower int64, upper int64) *ValidatorRange {
 		Lower:     lower,
 		Upper:     upper,
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -434,7 +439,7 @@ func NewModelIdOrNameValidator(key string, modelKeyword string, ownerId mcclient
 		ModelKeyword: modelKeyword,
 		modelIdKey:   key + "_id",
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -535,7 +540,7 @@ func NewRegexpValidator(key string, regexp *regexp.Regexp) *ValidatorRegexp {
 		Validator: Validator{Key: key},
 		Regexp:    regexp,
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -547,7 +552,7 @@ func NewDomainNameValidator(key string) *ValidatorDomainName {
 	v := &ValidatorDomainName{
 		ValidatorRegexp: *NewRegexpValidator(key, regutils.DOMAINNAME_REG),
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -562,7 +567,7 @@ func NewURLPathValidator(key string) *ValidatorURLPath {
 	v := &ValidatorURLPath{
 		ValidatorRegexp: *NewRegexpValidator(key, regexpURLPath),
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -598,7 +603,7 @@ func NewStructValidator(key string, value interface{}) *ValidatorStruct {
 		Validator: Validator{Key: key},
 		Value:     value,
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -645,6 +650,6 @@ func NewIPv4AddrValidator(key string) *ValidatorIPv4Addr {
 	v := &ValidatorIPv4Addr{
 		Validator: Validator{Key: key},
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }

pkg/cloudcommon/validators/validators_cert.go

@@ -37,7 +37,7 @@ func NewPEMValidator(key string) *ValidatorPEM {
 	v := &ValidatorPEM{
 		Validator: Validator{Key: key},
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -101,7 +101,7 @@ func NewCertificateValidator(key string) *ValidatorCertificate {
 	v := &ValidatorCertificate{
 		ValidatorPEM: *NewPEMValidator(key),
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 
@@ -232,7 +232,7 @@ func NewPrivateKeyValidator(key string) *ValidatorPrivateKey {
 	v := &ValidatorPrivateKey{
 		ValidatorPEM: *NewPEMValidator(key),
 	}
-	v.parent = v
+	v.SetParent(v)
 	return v
 }
 

pkg/compute/models/guests.go

@@ -1692,6 +1692,21 @@ func (self *SGuest) getVirtualIPs() []string {
 	return ips
 }
 
+func (self *SGuest) GetPrivateIPs() []string {
+	ips := self.GetRealIPs()
+	for i := len(ips) - 1; i >= 0; i-- {
+		ipAddr, err := netutils.NewIPV4Addr(ips[i])
+		if err != nil {
+			log.Errorf("guest %s(%s) has bad ipv4 address (%s): %v", self.Name, self.Id, ips[i], err)
+			continue
+		}
+		if !netutils.IsPrivate(ipAddr) {
+			ips = append(ips[:i], ips[i+1:]...)
+		}
+	}
+	return ips
+}
+
 func (self *SGuest) getIPs() []string {
 	ips := self.GetRealIPs()
 	vips := self.getVirtualIPs()

pkg/mcclient/models/base.go

@@ -69,6 +69,12 @@ type StatusStandaloneResource struct {
 	Status string
 }
 
+type EnabledStatusStandaloneResourceBase struct {
+	StatusStandaloneResource
+
+	Enabled bool
+}
+
 type VirtualResource struct {
 	StatusStandaloneResource
 

pkg/mcclient/models/hosts.go

@@ -0,0 +1,60 @@
+package models
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+)
+
+type Host struct {
+	EnabledStatusStandaloneResourceBase
+
+	Rack  string
+	Slots string
+
+	AccessMac  string
+	AccessIp   string
+	ManagerUri string
+
+	SysInfo jsonutils.JSONObject
+	SN      string
+
+	CpuCount    int
+	NodeCount   int8
+	CpuDesc     string
+	CpuMhz      int
+	CpuCache    int
+	CpuReserved int
+	CpuCmtbound float32
+
+	MemSize     int
+	MemReserved int
+	MemCmtbound float32
+
+	StorageSize   int
+	StorageType   string
+	StorageDriver string
+	StorageInfo   jsonutils.JSONObject
+
+	IpmiInfo jsonutils.JSONObject
+
+	HostStatus string
+
+	ZoneId string
+
+	HostType string
+
+	Version string
+
+	IsBaremetal bool
+
+	IsMaintenance bool
+
+	LastPingAt time.Time
+
+	ResourceType string
+
+	RealExternalId string
+
+	IsImport bool
+}

pkg/mcclient/models/servers.go

@@ -0,0 +1,158 @@
+package models
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/netutils"
+)
+
+type Server struct {
+	VirtualResource
+
+	VcpuCount int
+	VmemSize  int
+
+	BootOrder string
+
+	DisableDelete    tristate.TriState
+	ShutdownBehavior string
+
+	KeypairId string
+
+	HostId       string
+	BackupHostId string
+
+	Vga     string
+	Vdi     string
+	Machine string
+	Bios    string
+	OsType  string
+
+	FlavorId string
+
+	SecgrpId      string
+	AdminSecgrpId string
+
+	Hypervisor string
+
+	InstanceType string
+
+	// Derived attributes
+	Networks string
+}
+
+type ServerNetworks []ServerNetwork
+type ServerNetwork struct {
+	Index   int
+	Ip      net.IP
+	IpMask  int
+	MacAddr net.HardwareAddr
+	VlanId  int
+
+	Name   string
+	Driver string
+	Bw     int
+}
+
+func (sns ServerNetworks) GetPrivateIPs() []net.IP {
+	ips := []net.IP{}
+	for _, sn := range sns {
+		ipStr := sn.Ip.String()
+		ipAddr, err := netutils.NewIPV4Addr(ipStr)
+		if err != nil {
+			continue
+		}
+		if netutils.IsPrivate(ipAddr) {
+			ips = append(ips, sn.Ip)
+		}
+	}
+	return ips
+}
+
+func ParseServerNetworkDetailedString(s string) (ServerNetworks, error) {
+	sns := ServerNetworks{}
+	lines := strings.Split(s, "\n")
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if !strings.HasPrefix(line, "eth") {
+			return nil, errors.New("should be prefixed with 'eth'")
+		}
+		parts := strings.Split(line, "/")
+		if len(parts) < 7 {
+			return nil, errors.New("less than 7 parts separated by '/'")
+		}
+		sn := ServerNetwork{
+			Name:   parts[4],
+			Driver: parts[5],
+		}
+		{ // index, ipaddr
+			idip := parts[0][3:]
+			i := strings.IndexByte(idip, ':')
+			if i <= 0 {
+				return nil, errors.New("no sep ':'")
+			}
+			idStr, ipStr := idip[:i], idip[i+1:]
+			{
+				id, err := strconv.ParseUint(idStr, 10, 8)
+				if err != nil {
+					return nil, errors.WithMessagef(err, "bad index %s", idStr)
+				}
+				sn.Index = int(id)
+			}
+			{
+				ip := net.ParseIP(ipStr)
+				if ip == nil {
+					return nil, fmt.Errorf("bad ip %s", ipStr)
+				}
+				ip = ip.To4()
+				if ip == nil {
+					return nil, fmt.Errorf("not ipv4 addr: %s", ipStr)
+				}
+				sn.Ip = ip
+			}
+		}
+		{ // ip mask
+			masklenStr := parts[1]
+			masklen, err := strconv.ParseUint(masklenStr, 10, 8)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "bad network mask len: %s", masklenStr)
+			}
+			sn.IpMask = int(masklen)
+		}
+		{ // macaddr
+			macaddrStr := parts[2]
+			macaddr, err := net.ParseMAC(macaddrStr)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "bad macaddr: %s", macaddrStr)
+			}
+			sn.MacAddr = macaddr
+		}
+		{ // vlan
+			vlanStr := parts[3]
+			vlan, err := strconv.ParseUint(vlanStr, 10, 16)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "bad vlan id: %s", vlanStr)
+			}
+			sn.VlanId = int(vlan)
+		}
+		{ // bw
+			bwStr := parts[6]
+			bw, err := strconv.ParseUint(bwStr, 10, 32)
+			if err != nil {
+				return nil, errors.WithMessagef(err, "bad bw: %s", bwStr)
+			}
+			sn.Bw = int(bw)
+		}
+		sns = append(sns, sn)
+	}
+	return sns, nil
+}

pkg/mcclient/modules/managers.go

@@ -192,3 +192,11 @@ func NewCloudmetaManager(keyword, keywordPlural string, columns, adminColumns []
 			serviceType:  "cloudmeta"},
 		Keyword: keyword, KeywordPlural: keywordPlural}
 }
+
+func NewAnsibleManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
+	return ResourceManager{
+		BaseManager: BaseManager{columns: columns,
+			adminColumns: adminColumns,
+			serviceType:  "ansible"},
+		Keyword: keyword, KeywordPlural: keywordPlural}
+}

pkg/mcclient/modules/mod_ansibleplaybooks.go

@@ -0,0 +1,41 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+type AnsiblePlaybookManager struct {
+	ResourceManager
+}
+
+var (
+	AnsiblePlaybooks AnsiblePlaybookManager
+)
+
+func init() {
+	AnsiblePlaybooks = AnsiblePlaybookManager{
+		NewAnsibleManager(
+			"ansibleplaybook",
+			"ansibleplaybooks",
+			[]string{
+				"id",
+				"name",
+				"status",
+				"start_time",
+				"end_time",
+			},
+			[]string{},
+		),
+	}
+	register(&AnsiblePlaybooks)
+}

pkg/mcclient/options/ansibleplaybooks.go

@@ -0,0 +1,95 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/util/ansible"
+)
+
+type AnsiblePlaybookIdOptions struct {
+	ID string `help:"name/id of the playbook"`
+}
+
+type AnsiblePlaybookListOptions struct {
+	BaseListOptions
+}
+
+type AnsiblePlaybookCommonOptions struct {
+	Host []string `help:"name or id of server or host in format '<[server:]id|host:id>|ipaddr var=val'"`
+	Mod  []string `help:"ansible modules and their arguments in format 'name k1=v1 k2=v2'"`
+}
+
+func (opts *AnsiblePlaybookCommonOptions) params() (jsonutils.JSONObject, error) {
+	if len(opts.Mod) == 0 {
+		return nil, fmt.Errorf("Requires at least one --mod argument")
+	}
+	if len(opts.Host) == 0 {
+		return nil, fmt.Errorf("Requires at least one server/host to operate on")
+	}
+	pb := ansible.NewPlaybook()
+	hosts := []ansible.Host{}
+	for _, s := range opts.Host {
+		host, err := ansible.ParseHostLine(s)
+		if err != nil {
+			return nil, err
+		}
+		hosts = append(hosts, host)
+	}
+	pb.Inventory = ansible.Inventory{Hosts: hosts}
+	for _, s := range opts.Mod {
+		module, err := ansible.ParseModuleLine(s)
+		if err != nil {
+			return nil, err
+		}
+		pb.Modules = append(pb.Modules, module)
+	}
+	pbJson := jsonutils.Marshal(pb)
+	return pbJson, nil
+}
+
+type AnsiblePlaybookCreateOptions struct {
+	NAME string `help:"name of the playbook"`
+	AnsiblePlaybookCommonOptions
+}
+
+func (opts *AnsiblePlaybookCreateOptions) Params() (*jsonutils.JSONDict, error) {
+	pbJson, err := opts.AnsiblePlaybookCommonOptions.params()
+	if err != nil {
+		return nil, err
+	}
+	params := jsonutils.NewDict()
+	params.Set("playbook", pbJson)
+	params.Set("name", jsonutils.NewString(opts.NAME))
+	return params, nil
+}
+
+type AnsiblePlaybookUpdateOptions struct {
+	ID string `json:"-" help:"name/id of the playbook"`
+	AnsiblePlaybookCommonOptions
+}
+
+func (opts *AnsiblePlaybookUpdateOptions) Params() (*jsonutils.JSONDict, error) {
+	pbJson, err := opts.AnsiblePlaybookCommonOptions.params()
+	if err != nil {
+		return nil, err
+	}
+	params := jsonutils.NewDict()
+	params.Set("playbook", pbJson)
+	return params, nil
+}

pkg/util/ansible/module.go

@@ -0,0 +1,68 @@
+package ansible
+
+import (
+	"bytes"
+)
+
+// Module represents name and args of ansible module to execute
+type Module struct {
+	// Name is ansible module name
+	Name string
+	// Args is a list of module arguments in form of key=value
+	Args []string
+}
+
+// Host represents an ansible host
+type Host struct {
+	// Name denotes the host to operate on
+	Name string
+	// Vars is a map recording host vars
+	Vars map[string]string
+}
+
+// GetVar returns variable value.  Second return value will be false if the
+// variable does not exist
+func (h *Host) GetVar(k string) (v string, exist bool) {
+	if len(h.Vars) == 0 {
+		return
+	}
+	v, exist = h.Vars[k]
+	return
+}
+
+// SetVar sets variable k to value v
+func (h *Host) SetVar(k, v string) {
+	if h.Vars == nil {
+		h.Vars = map[string]string{
+			k: v,
+		}
+		return
+	}
+	h.Vars[k] = v
+}
+
+// Inventory contains a list of ansible hosts
+type Inventory struct {
+	Hosts []Host
+}
+
+// IsEmpty returns true if the inventory is empty
+func (i *Inventory) IsEmpty() bool {
+	return len(i.Hosts) == 0
+}
+
+// Data returns serialized form of the inventory as represented on disk
+func (i *Inventory) Data() []byte {
+	b := &bytes.Buffer{}
+	for _, h := range i.Hosts {
+		b.WriteString(h.Name)
+		for k, v := range h.Vars {
+			b.WriteRune(' ')
+			b.WriteString(k)
+			b.WriteRune('=')
+			b.WriteString(v)
+		}
+		b.WriteRune('\n')
+	}
+	return b.Bytes()
+}

pkg/util/ansible/parser.go

@@ -0,0 +1,69 @@
+package ansible
+
+import (
+	"fmt"
+	"strings"
+)
+
+// TODO
+//
+// quote parser
+
+// ParseHostLine parse string representation of an ansible host
+//
+// Input should be in format "name key=value"
+func ParseHostLine(s string) (host Host, err error) {
+	s = strings.TrimSpace(s)
+	parts := strings.Split(s, " ")
+	if len(parts) == 0 {
+		err = fmt.Errorf("Host name must not be empty")
+		return
+	}
+	host.Name = parts[0]
+	vars := map[string]string{}
+	for _, kv := range parts[1:] {
+		s := strings.SplitN(kv, "=", 2)
+		if len(s) != 2 {
+			err = fmt.Errorf("host var not in the form name=val: %s", kv)
+			return
+		}
+		vars[s[0]] = s[1]
+	}
+	host.Vars = vars
+	return
+}
+
+// ParseModuleLine parse string representation of an ansible module task
+//
+// The input should be in format "name key=value arg0 arg1".  The argN form is
+// For module "command" and "shell"
+func ParseModuleLine(s string) (mod Module, err error) {
+	s = strings.TrimSpace(s)
+	parts := strings.Split(s, " ")
+	if len(parts) == 0 {
+		err = fmt.Errorf("Module name must not be empty")
+		return
+	}
+	mod.Name = parts[0]
+
+	args := []string{}
+	command := ""
+	freeForm := false // command and shell module take free form arguments
+	for _, part := range parts[1:] {
+		if freeForm {
+			command = command + " " + part
+			continue
+		}
+		if strings.Contains(part, "=") {
+			args = append(args, part)
+		} else {
+			freeForm = true
+			command = command + " " + part
+		}
+	}
+	if command != "" {
+		args = append(args, command)
+	}
+	mod.Args = args
+	return
+}

pkg/util/ansible/playbook.go

@@ -0,0 +1,211 @@
+package ansible
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"yunion.io/x/pkg/gotypes"
+	yerrors "yunion.io/x/pkg/util/errors"
+)
+
+type pbState int
+
+const (
+	pbStateInit pbState = iota
+	pbStateRunning
+	pbStateStopped
+)
+
+func (pbs pbState) String() string {
+	switch pbs {
+	case pbStateInit:
+		return "init"
+	case pbStateRunning:
+		return "running"
+	case pbStateStopped:
+		return "stopped"
+	}
+	return "unknown"
+}
+
+type Playbook struct {
+	Inventory  Inventory
+	Modules    []Module
+	PrivateKey []byte
+
+	tmpdir        string
+	noCleanOnExit bool
+	stdio         *bytes.Buffer
+	state         pbState
+	stateMux      *sync.Mutex
+}
+
+func NewPlaybook() *Playbook {
+	pb := &Playbook{
+		state:    pbStateInit,
+		stateMux: &sync.Mutex{},
+		stdio:    &bytes.Buffer{},
+	}
+	return pb
+}
+
+func (pb *Playbook) Copy() *Playbook {
+	pb1 := NewPlaybook()
+	pb1.Inventory = gotypes.DeepCopy(pb.Inventory).(Inventory)
+	pb1.Modules = gotypes.DeepCopy(pb.Modules).([]Module)
+	pb1.PrivateKey = gotypes.DeepCopy(pb.PrivateKey).([]byte)
+	return pb1
+}
+
+// CleanOnExit decide whether temporary workdir will be cleaned up after Run
+func (pb *Playbook) CleanOnExit(b bool) {
+	pb.noCleanOnExit = !b
+}
+
+// State returns current state of the playbook
+func (pb *Playbook) State() pbState {
+	pb.stateMux.Lock()
+	defer pb.stateMux.Unlock()
+	return pb.state
+}
+
+// Runnable returns whether the playbook is in a state feasible to be run
+func (pb *Playbook) Runnable() bool {
+	return pb.state == pbStateInit
+}
+
+// Running returns whether the playbook is currently running
+func (pb *Playbook) Running() bool {
+	return pb.state == pbStateRunning
+}
+
+// Run runs the playbook
+func (pb *Playbook) Run(ctx context.Context) (err error) {
+	var (
+		tmpdir string
+	)
+
+	pb.stateMux.Lock()
+	if pb.state != pbStateInit {
+		return errors.Errorf("playbook state %s, want %s",
+			pb.state, pbStateInit)
+	}
+	pb.state = pbStateRunning
+	pb.stateMux.Unlock()
+	defer func() {
+		pb.stateMux.Lock()
+		pb.state = pbStateStopped
+		pb.stateMux.Unlock()
+	}()
+
+	if pb.Inventory.IsEmpty() {
+		return errors.New("empty inventory")
+	}
+
+	// make tmpdir
+	tmpdir, err = ioutil.TempDir("", "onecloud-ansible")
+	if err != nil {
+		err = errors.WithMessage(err, "making tmp dir")
+		return
+	}
+	pb.tmpdir = tmpdir
+	defer func() {
+		if pb.noCleanOnExit {
+			return
+		}
+		if err1 := os.RemoveAll(tmpdir); err1 != nil {
+			err = errors.WithMessagef(err1, "removing %q", tmpdir)
+		}
+	}()
+
+	// write out inventory
+	inventory := filepath.Join(tmpdir, "inventory")
+	err = ioutil.WriteFile(inventory, pb.Inventory.Data(), os.FileMode(0600))
+	if err != nil {
+		err = errors.WithMessagef(err, "writing inventory %s", inventory)
+		return
+	}
+
+	// write out private key
+	var privateKey string
+	if len(pb.PrivateKey) > 0 {
+		privateKey = filepath.Join(tmpdir, "private_key")
+		err = ioutil.WriteFile(privateKey, pb.PrivateKey, os.FileMode(0600))
+		if err != nil {
+			err = errors.WithMessagef(err, "writing private key %s", privateKey)
+			return
+		}
+	}
+
+	// run modules one by one
+	var errs []error
+	defer func() {
+		if len(errs) > 0 {
+			err = yerrors.NewAggregate(errs)
+		}
+	}()
+	for _, m := range pb.Modules {
+		select {
+		case <-ctx.Done():
+			err = ctx.Err()
+			return
+		default:
+		}
+		modArgs := strings.Join(m.Args, " ")
+		args := []string{
+			"--inventory", inventory,
+			"--module-name", m.Name,
+			"--args", modArgs,
+			"all",
+		}
+		if privateKey != "" {
+			args = append(args, "--private-key", privateKey)
+		}
+		cmd := exec.CommandContext(ctx, "ansible", args...)
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, "ANSIBLE_HOST_KEY_CHECKING=False")
+		stdout, _ := cmd.StdoutPipe()
+		stderr, _ := cmd.StderrPipe()
+		if err1 := cmd.Start(); err1 != nil {
+			errs = append(errs, errors.WithMessagef(err1, "run module %q, args %q", m.Name, modArgs))
+			return
+		}
+		f := func(r io.Reader) {
+			b := make([]byte, 4096)
+			for {
+				n, err := r.Read(b)
+				if n > 0 {
+					// Mix stdout, stderr
+					pb.stdio.Write(b[:n])
+				}
+				if err != nil {
+					return
+				}
+			}
+		}
+		go f(stdout)
+		go f(stderr)
+		if err1 := cmd.Wait(); err1 != nil {
+			errs = append(errs, errors.WithMessagef(err1, "wait module %q, args %q", m.Name, modArgs))
+			// continue to next
+		}
+	}
+	return nil
+}
+
+// Output returns the stdio output of the playbook
+func (pb *Playbook) Output() []byte {
+	if pb.stdio != nil {
+		return pb.stdio.Bytes()
+	}
+	return nil
+}

pkg/util/ansible/playbook_test.go

@@ -0,0 +1,40 @@
+package ansible
+
+import (
+	"context"
+	"os/exec"
+	"testing"
+)
+
+func skipIfNoAnsible(t *testing.T) {
+	_, err := exec.LookPath("ansible")
+	if err != nil {
+		t.Skipf("looking for ansible: %v", err)
+	}
+}
+
+func TestPlaybook(t *testing.T) {
+	skipIfNoAnsible(t)
+
+	pb := NewPlaybook()
+	pb.Inventory = Inventory{
+		Hosts: []Host{
+			{
+				Name: "127.0.0.1",
+				Vars: map[string]string{
+					"ansible_connection": "local",
+				},
+			},
+		},
+	}
+	pb.Modules = []Module{
+		{
+			Name: "ping",
+		},
+	}
+	err := pb.Run(context.TODO())
+	if err != nil {
+		t.Fatalf("not expecting err: %v", err)
+	}
+	t.Logf("%s", pb.Output())
+}

pkg/util/ansible/serializable.go

@@ -0,0 +1,29 @@
+package ansible
+
+import (
+	"reflect"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/gotypes"
+)
+
+// String implements gotypes.ISerializable
+func (pb *Playbook) String() string {
+	return jsonutils.Marshal(pb).String()
+}
+
+// IsZero implements gotypes.ISerializable
+func (pb *Playbook) IsZero() bool {
+	if len(pb.Inventory.Hosts) == 0 {
+		return true
+	}
+	if len(pb.Modules) == 0 {
+		return true
+	}
+	return false
+}
+
+func init() {
+	gotypes.RegisterSerializable(reflect.TypeOf(&Playbook{}), func() gotypes.ISerializable {
+		return NewPlaybook()
+	})
+}

pkg/util/ansible/session.go

@@ -0,0 +1,71 @@
+package ansible
+
+import (
+	"context"
+)
+
+// Session is a container for execution of playbook
+type Session struct {
+	// Ctx is the context under which the playbook will run
+	Ctx context.Context
+	// Playbook is the ansible playbook to be run
+	Playbook *Playbook
+
+	// cancelFunc can be called to cancel the running playbook
+	cancelFunc context.CancelFunc
+}
+
+// SessionManager manages a collection of keyed sessions
+type SessionManager map[string]*Session
+
+// Has returns true if a session with the specified id exists in the manager
+func (sm SessionManager) Has(id string) bool {
+	_, ok := sm[id]
+	return ok
+}
+
+// Add adds a Playbook to the manager keyed with the specified id
+func (sm SessionManager) Add(id string, pb *Playbook) *Session {
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	session := &Session{
+		Playbook:   pb,
+		Ctx:        ctx,
+		cancelFunc: cancelFunc,
+	}
+	sm[id] = session
+	return session
+}
+
+// Remove stops (if appliable) and removes the playbook keyed with the
+// specified id from the manager
+func (sm SessionManager) Remove(id string) {
+	sm.Stop(id)
+	delete(sm, id)
+}
+
+// Run runs the playbook keyed with specified id
+func (sm SessionManager) Run(id string) error {
+	s, ok := sm[id]
+	if !ok {
+		return nil
+	}
+	return s.Playbook.Run(s.Ctx)
+}
+
+// Stop stops the playbook keyed with specified id
+func (sm SessionManager) Stop(id string) {
+	s, ok := sm[id]
+	if !ok {
+		return
+	}
+	s.cancelFunc()
+}
+
+// Err returns possible error from sm.Ctx.Err()
+func (sm SessionManager) Err(id string) error {
+	s, ok := sm[id]
+	if !ok {
+		return nil
+	}
+	return s.Ctx.Err()
+}

vendor/github.com/pkg/errors/.travis.yml

@@ -1,10 +1,14 @@
 language: go
 go_import_path: github.com/pkg/errors
 go:
-  - 1.4.3
-  - 1.5.4
-  - 1.6.2
-  - 1.7.1
+  - 1.4.x
+  - 1.5.x
+  - 1.6.x
+  - 1.7.x
+  - 1.8.x
+  - 1.9.x
+  - 1.10.x
+  - 1.11.x
   - tip
 
 script:

vendor/github.com/pkg/errors/README.md

@@ -1,4 +1,4 @@
-# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors)
+# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) [![Sourcegraph](https://sourcegraph.com/github.com/pkg/errors/-/badge.svg)](https://sourcegraph.com/github.com/pkg/errors?badge)
 
 Package errors provides simple error handling primitives.
 
@@ -47,6 +47,6 @@ We welcome pull requests, bug fixes and issue reports. With that said, the bar f
 
 Before proposing a change, please discuss your change by raising an issue.
 
-## Licence
+## License
 
 BSD-2-Clause

vendor/github.com/pkg/errors/errors.go

@@ -6,7 +6,7 @@
 //             return err
 //     }
 //
-// which applied recursively up the call stack results in error reports
+// which when applied recursively up the call stack results in error reports
 // without context or debugging information. The errors package allows
 // programmers to add context to the failure path in their code in a way
 // that does not destroy the original value of the error.
@@ -15,16 +15,17 @@
 //
 // The errors.Wrap function returns a new error that adds context to the
 // original error by recording a stack trace at the point Wrap is called,
-// and the supplied message. For example
+// together with the supplied message. For example
 //
 //     _, err := ioutil.ReadAll(r)
 //     if err != nil {
 //             return errors.Wrap(err, "read failed")
 //     }
 //
-// If additional control is required the errors.WithStack and errors.WithMessage
-// functions destructure errors.Wrap into its component operations of annotating
-// an error with a stack trace and an a message, respectively.
+// If additional control is required, the errors.WithStack and
+// errors.WithMessage functions destructure errors.Wrap into its component
+// operations: annotating an error with a stack trace and with a message,
+// respectively.
 //
 // Retrieving the cause of an error
 //
@@ -38,7 +39,7 @@
 //     }
 //
 // can be inspected by errors.Cause. errors.Cause will recursively retrieve
-// the topmost error which does not implement causer, which is assumed to be
+// the topmost error that does not implement causer, which is assumed to be
 // the original cause. For example:
 //
 //     switch err := errors.Cause(err).(type) {
@@ -48,16 +49,16 @@
 //             // unknown error
 //     }
 //
-// causer interface is not exported by this package, but is considered a part
-// of stable public API.
+// Although the causer interface is not exported by this package, it is
+// considered a part of its stable public interface.
 //
 // Formatted printing of errors
 //
 // All error values returned from this package implement fmt.Formatter and can
-// be formatted by the fmt package. The following verbs are supported
+// be formatted by the fmt package. The following verbs are supported:
 //
 //     %s    print the error. If the error has a Cause it will be
-//           printed recursively
+//           printed recursively.
 //     %v    see %s
 //     %+v   extended format. Each Frame of the error's StackTrace will
 //           be printed in detail.
@@ -65,13 +66,13 @@
 // Retrieving the stack trace of an error or wrapper
 //
 // New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
-// invoked. This information can be retrieved with the following interface.
+// invoked. This information can be retrieved with the following interface:
 //
 //     type stackTracer interface {
 //             StackTrace() errors.StackTrace
 //     }
 //
-// Where errors.StackTrace is defined as
+// The returned errors.StackTrace type is defined as
 //
 //     type StackTrace []Frame
 //
@@ -85,8 +86,8 @@
 //             }
 //     }
 //
-// stackTracer interface is not exported by this package, but is considered a part
-// of stable public API.
+// Although the stackTracer interface is not exported by this package, it is
+// considered a part of its stable public interface.
 //
 // See the documentation for Frame.Format for more details.
 package errors
@@ -192,7 +193,7 @@ func Wrap(err error, message string) error {
 }
 
 // Wrapf returns an error annotating err with a stack trace
-// at the point Wrapf is call, and the format specifier.
+// at the point Wrapf is called, and the format specifier.
 // If err is nil, Wrapf returns nil.
 func Wrapf(err error, format string, args ...interface{}) error {
 	if err == nil {
@@ -220,6 +221,18 @@ func WithMessage(err error, message string) error {
 	}
 }
 
+// WithMessagef annotates err with the format specifier.
+// If err is nil, WithMessagef returns nil.
+func WithMessagef(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	return &withMessage{
+		cause: err,
+		msg:   fmt.Sprintf(format, args...),
+	}
+}
+
 type withMessage struct {
 	cause error
 	msg   string

vendor/github.com/pkg/errors/stack.go

@@ -46,7 +46,8 @@ func (f Frame) line() int {
 //
 // Format accepts flags that alter the printing of some verbs, as follows:
 //
-//    %+s   path of source file relative to the compile time GOPATH
+//    %+s   function name and path of source file relative to the compile time
+//          GOPATH separated by \n\t (<funcname>\n\t<path>)
 //    %+v   equivalent to %+s:%d
 func (f Frame) Format(s fmt.State, verb rune) {
 	switch verb {
@@ -79,6 +80,14 @@ func (f Frame) Format(s fmt.State, verb rune) {
 // StackTrace is stack of Frames from innermost (newest) to outermost (oldest).
 type StackTrace []Frame
 
+// Format formats the stack of Frames according to the fmt.Formatter interface.
+//
+//    %s	lists source files for each Frame in the stack
+//    %v	lists the source file and line number for each Frame in the stack
+//
+// Format accepts flags that alter the printing of some verbs, as follows:
+//
+//    %+v   Prints filename, function, and line number for each Frame in the stack.
 func (st StackTrace) Format(s fmt.State, verb rune) {
 	switch verb {
 	case 'v':
@@ -136,43 +145,3 @@ func funcname(name string) string {
 	i = strings.Index(name, ".")
 	return name[i+1:]
 }
-
-func trimGOPATH(name, file string) string {
-	// Here we want to get the source file path relative to the compile time
-	// GOPATH. As of Go 1.6.x there is no direct way to know the compiled
-	// GOPATH at runtime, but we can infer the number of path segments in the
-	// GOPATH. We note that fn.Name() returns the function name qualified by
-	// the import path, which does not include the GOPATH. Thus we can trim
-	// segments from the beginning of the file path until the number of path
-	// separators remaining is one more than the number of path separators in
-	// the function name. For example, given:
-	//
-	//    GOPATH     /home/user
-	//    file       /home/user/src/pkg/sub/file.go
-	//    fn.Name()  pkg/sub.Type.Method
-	//
-	// We want to produce:
-	//
-	//    pkg/sub/file.go
-	//
-	// From this we can easily see that fn.Name() has one less path separator
-	// than our desired output. We count separators from the end of the file
-	// path until it finds two more than in the function name and then move
-	// one character forward to preserve the initial path segment without a
-	// leading separator.
-	const sep = "/"
-	goal := strings.Count(name, sep) + 2
-	i := len(file)
-	for n := 0; n < goal; n++ {
-		i = strings.LastIndex(file[:i], sep)
-		if i == -1 {
-			// not enough separators found, set i so that the slice expression
-			// below leaves file unmodified
-			i = -len(sep)
-			break
-		}
-	}
-	// get back to 0 or trim the leading separator
-	file = file[i+len(sep):]
-	return file
-}