Merge pull request #13339 from zexi/tls-migration

Tls migration

pkg/apis/compute/api.go

@@ -571,6 +571,7 @@ type GuestBatchMigrateParams struct {
 	Id           string
 	LiveMigrate  bool
 	SkipCpuCheck bool
+	EnableTLS    bool
 	RescueMode   bool
 	OldStatus    string
 }

pkg/apis/compute/guests.go

@@ -382,6 +382,8 @@ type GuestLiveMigrateInput struct {
 	PreferHost string `json:"prefer_host"`
 	// 是否跳过CPU检查，默认要做CPU检查
 	SkipCpuCheck *bool `json:"skip_cpu_check"`
+	// 是否启用 tls
+	EnableTLS *bool `json:"enable_tls"`
 }
 
 type GuestSetSecgroupInput struct {

pkg/compute/models/guest_actions.go

@@ -491,10 +491,10 @@ func (self *SGuest) PerformLiveMigrate(ctx context.Context, userCred mcclient.To
 	if err := self.validateMigrate(ctx, userCred, nil, input); err != nil {
 		return nil, err
 	}
-	return nil, self.StartGuestLiveMigrateTask(ctx, userCred, self.Status, input.PreferHost, input.SkipCpuCheck, "")
+	return nil, self.StartGuestLiveMigrateTask(ctx, userCred, self.Status, input.PreferHost, input.SkipCpuCheck, input.EnableTLS, "")
 }
 
-func (self *SGuest) StartGuestLiveMigrateTask(ctx context.Context, userCred mcclient.TokenCredential, guestStatus, preferHostId string, skipCpuCheck *bool, parentTaskId string) error {
+func (self *SGuest) StartGuestLiveMigrateTask(ctx context.Context, userCred mcclient.TokenCredential, guestStatus, preferHostId string, skipCpuCheck *bool, enableTLS *bool, parentTaskId string) error {
 	self.SetStatus(userCred, api.VM_START_MIGRATE, "")
 	data := jsonutils.NewDict()
 	if len(preferHostId) > 0 {
@@ -503,6 +503,9 @@ func (self *SGuest) StartGuestLiveMigrateTask(ctx context.Context, userCred mccl
 	if skipCpuCheck != nil {
 		data.Set("skip_cpu_check", jsonutils.NewBool(*skipCpuCheck))
 	}
+	if enableTLS != nil {
+		data.Set("enable_tls", jsonutils.NewBool(*enableTLS))
+	}
 	data.Set("guest_status", jsonutils.NewString(guestStatus))
 	dedicateMigrateTask := "GuestLiveMigrateTask"
 	if self.GetHypervisor() != api.HYPERVISOR_KVM {

pkg/compute/tasks/guest_live_migrate_task.go

@@ -184,6 +184,7 @@ func (self *GuestMigrateTask) OnCachedCdromComplete(ctx context.Context, guest *
 	guestStatus, _ := self.Params.GetString("guest_status")
 	if !jsonutils.QueryBoolean(self.Params, "is_rescue_mode", false) && (guestStatus == api.VM_RUNNING || guestStatus == api.VM_SUSPEND) {
 		body.Set("live_migrate", jsonutils.JSONTrue)
+		body.Set("enable_tls", jsonutils.NewBool(jsonutils.QueryBoolean(self.GetParams(), "enable_tls", false)))
 	}
 
 	if !jsonutils.QueryBoolean(self.Params, "is_rescue_mode", false) {
@@ -223,6 +224,15 @@ func (self *GuestMigrateTask) OnSrcPrepareComplete(ctx context.Context, guest *m
 	} else {
 		body, err = self.sharedStorageMigrateConf(ctx, guest, targetHost)
 	}
+	if jsonutils.QueryBoolean(self.GetParams(), "enable_tls", false) {
+		body.Set("enable_tls", jsonutils.JSONTrue)
+		certsObj, err := data.Get("migrate_certs")
+		if err != nil {
+			self.TaskFailed(ctx, guest, jsonutils.NewString(errors.Wrap(err, "get migrate_certs from data").Error()))
+			return
+		}
+		body.Set("migrate_certs", certsObj)
+	}
 	if err != nil {
 		self.TaskFailed(ctx, guest, jsonutils.NewString(err.Error()))
 		return
@@ -384,6 +394,7 @@ func (self *GuestLiveMigrateTask) OnStartDestComplete(ctx context.Context, guest
 	body.Set("is_local_storage", isLocalStorage)
 	body.Set("live_migrate_dest_port", liveMigrateDestPort)
 	body.Set("dest_ip", jsonutils.NewString(targetHost.AccessIp))
+	body.Set("enable_tls", jsonutils.NewBool(jsonutils.QueryBoolean(self.GetParams(), "enable_tls", false)))
 
 	headers := self.GetTaskRequestHeader()
 
@@ -443,6 +454,7 @@ func (self *GuestLiveMigrateTask) OnLiveMigrateComplete(ctx context.Context, gue
 	headers := self.GetTaskRequestHeader()
 	body := jsonutils.NewDict()
 	body.Set("live_migrate", jsonutils.JSONTrue)
+	body.Set("clean_tls", jsonutils.NewBool(jsonutils.QueryBoolean(self.GetParams(), "enable_tls", false)))
 	targetHostId, _ := self.Params.GetString("target_host_id")
 
 	self.SetStage("OnResumeDestGuestComplete", nil)

pkg/compute/tasks/host_guests_migrate_task.go

@@ -49,7 +49,7 @@ func (self *HostGuestsMigrateTask) OnInit(ctx context.Context, objs []db.IStanda
 		guest := objs[i].(*models.SGuest)
 		if guests[i].LiveMigrate {
 			err := guest.StartGuestLiveMigrateTask(
-				ctx, self.UserCred, guests[i].OldStatus, preferHostId, &guests[i].SkipCpuCheck, self.Id)
+				ctx, self.UserCred, guests[i].OldStatus, preferHostId, &guests[i].SkipCpuCheck, &guests[i].EnableTLS, self.Id)
 			if err != nil {
 				log.Errorln(err)
 			}

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -274,10 +274,12 @@ func guestSrcPrepareMigrate(ctx context.Context, sid string, body jsonutils.JSON
 		return nil, httperrors.NewNotFoundError("Guest %s not found", sid)
 	}
 	liveMigrate := jsonutils.QueryBoolean(body, "live_migrate", false)
+	liveMigrateEnableTls := jsonutils.QueryBoolean(body, "enable_tls", false)
 	hostutils.DelayTask(ctx, guestman.GetGuestManager().SrcPrepareMigrate,
 		&guestman.SSrcPrepareMigrate{
-			Sid:         sid,
-			LiveMigrate: liveMigrate,
+			Sid:               sid,
+			LiveMigrate:       liveMigrate,
+			LiveMigrateUseTLS: liveMigrateEnableTls,
 		})
 	return nil, nil
 }
@@ -309,6 +311,16 @@ func guestDestPrepareMigrate(ctx context.Context, sid string, body jsonutils.JSO
 	params.QemuVersion = qemuVersion
 	params.LiveMigrate = liveMigrate
 	params.SourceQemuCmdline = qemuCmdline
+	certsObj, err := body.Get("migrate_certs")
+	if err != nil {
+		return nil, httperrors.NewMissingParameterError("migrate_certs")
+	}
+	certs := map[string]string{}
+	if err := certsObj.Unmarshal(&certs); err != nil {
+		return nil, httperrors.NewInputParameterError("unmarshal migrate_certs to map: %s", err)
+	}
+	params.MigrateCerts = certs
+	params.EnableTLS = jsonutils.QueryBoolean(body, "enable_tls", false)
 	if isLocal {
 		serverUrl, err := body.GetString("server_url")
 		if err != nil {
@@ -378,8 +390,13 @@ func guestLiveMigrate(ctx context.Context, sid string, body jsonutils.JSONObject
 	if err != nil {
 		return nil, httperrors.NewMissingParameterError("is_local_storage")
 	}
+	enableTLS := jsonutils.QueryBoolean(body, "enable_tls", false)
 	hostutils.DelayTaskWithoutReqctx(ctx, guestman.GetGuestManager().LiveMigrate, &guestman.SLiveMigrate{
-		Sid: sid, DestPort: int(destPort), DestIp: destIp, IsLocal: isLocal,
+		Sid:       sid,
+		DestPort:  int(destPort),
+		DestIp:    destIp,
+		IsLocal:   isLocal,
+		EnableTLS: enableTLS,
 	})
 	return nil, nil
 }
@@ -389,7 +406,8 @@ func guestResume(ctx context.Context, sid string, body jsonutils.JSONObject) (in
 		return nil, httperrors.NewNotFoundError("Guest %s not found", sid)
 	}
 	isLiveMigrate := jsonutils.QueryBoolean(body, "live_migrate", false)
-	guestman.GetGuestManager().Resume(ctx, sid, isLiveMigrate)
+	cleanTLS := jsonutils.QueryBoolean(body, "clean_tls", false)
+	guestman.GetGuestManager().Resume(ctx, sid, isLiveMigrate, cleanTLS)
 	return nil, nil
 }
 

pkg/hostman/guestman/guesthelper.go

@@ -33,8 +33,9 @@ type SGuestDeploy struct {
 }
 
 type SSrcPrepareMigrate struct {
-	Sid         string
-	LiveMigrate bool
+	Sid               string
+	LiveMigrate       bool
+	LiveMigrateUseTLS bool
 }
 
 type SDestPrepareMigrate struct {
@@ -42,6 +43,8 @@ type SDestPrepareMigrate struct {
 	ServerUrl         string
 	QemuVersion       string
 	SourceQemuCmdline string
+	MigrateCerts      map[string]string
+	EnableTLS         bool
 	SnapshotsUri      string
 	DisksUri          string
 	// TargetStorageId string
@@ -55,10 +58,11 @@ type SDestPrepareMigrate struct {
 }
 
 type SLiveMigrate struct {
-	Sid      string
-	DestPort int
-	DestIp   string
-	IsLocal  bool
+	Sid       string
+	DestPort  int
+	DestIp    string
+	IsLocal   bool
+	EnableTLS bool
 }
 
 type SDriverMirror struct {

pkg/hostman/guestman/guestman.go

@@ -774,16 +774,23 @@ func (m *SGuestManager) SrcPrepareMigrate(ctx context.Context, params interface{
 		return nil, hostutils.ParamsError
 	}
 	guest, _ := m.GetServer(migParams.Sid)
-	disksPrepare, err := guest.PrepareMigrate(migParams.LiveMigrate)
+	disksPrepare, err := guest.PrepareDisksMigrate(migParams.LiveMigrate)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "PrepareDisksMigrate")
 	}
+	ret := jsonutils.NewDict()
 	if disksPrepare.Length() > 0 {
-		ret := jsonutils.NewDict()
 		ret.Set("disks_back", disksPrepare)
-		return ret, nil
 	}
-	return nil, nil
+
+	if migParams.LiveMigrate && migParams.LiveMigrateUseTLS {
+		certs, err := guest.PrepareMigrateCerts()
+		if err != nil {
+			return nil, errors.Wrap(err, "PrepareMigrateCerts")
+		}
+		ret.Set("migrate_certs", jsonutils.Marshal(certs))
+	}
+	return ret, nil
 }
 
 func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
@@ -825,6 +832,12 @@ func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface
 		startParams.Set("qemu_version", jsonutils.NewString(migParams.QemuVersion))
 		startParams.Set("need_migrate", jsonutils.JSONTrue)
 		startParams.Set("source_qemu_cmdline", jsonutils.NewString(migParams.SourceQemuCmdline))
+		startParams.Set("live_migrate_use_tls", jsonutils.NewBool(migParams.EnableTLS))
+		if len(migParams.MigrateCerts) > 0 {
+			if err := guest.WriteMigrateCerts(migParams.MigrateCerts); err != nil {
+				return nil, errors.Wrap(err, "write migrate certs")
+			}
+		}
 		hostutils.DelayTaskWithoutReqctx(ctx, guest.asyncScriptStart, startParams)
 	} else {
 		hostutils.UpdateServerProgress(context.Background(), migParams.Sid, 100.0, 0)
@@ -934,13 +947,13 @@ func (m *SGuestManager) DeleteSnapshot(ctx context.Context, params interface{})
 	}
 }
 
-func (m *SGuestManager) Resume(ctx context.Context, sid string, isLiveMigrate bool) (jsonutils.JSONObject, error) {
+func (m *SGuestManager) Resume(ctx context.Context, sid string, isLiveMigrate bool, cleanTLS bool) (jsonutils.JSONObject, error) {
 	guest, _ := m.GetServer(sid)
 	if guest.IsStopping() || guest.IsStopped() {
 		return nil, httperrors.NewInvalidStatusError("resume stopped server???")
 	}
 	var cb = func() {
-		resumeTask := NewGuestResumeTask(ctx, guest, !isLiveMigrate)
+		resumeTask := NewGuestResumeTask(ctx, guest, !isLiveMigrate, cleanTLS)
 		if isLiveMigrate {
 			guest.StartPresendArp()
 		}

pkg/hostman/guestman/guesttasks.go

@@ -659,7 +659,34 @@ func (s *SGuestLiveMigrateTask) startMigrate(res string) {
 		hostutils.TaskFailed(s.ctx, fmt.Sprintf("Migrate set capability auto-converge error: %s", res))
 		return
 	}
+	if s.params.EnableTLS {
+		// https://wiki.qemu.org/Features/MigrationTLS
+		s.Monitor.ObjectAdd("tls-creds-x509", map[string]string{
+			"dir":         s.getPKIDirPath(),
+			"endpoint":    "client",
+			"id":          "tls0",
+			"verify-peer": "no",
+		}, func(res string) {
+			if strings.Contains(strings.ToLower(res), "error") {
+				s.migrateTask = nil
+				hostutils.TaskFailed(s.ctx, fmt.Sprintf("Migrate add tls-creds-x509 object client tls0 error: %s", res))
+				return
+			}
+			s.Monitor.MigrateSetParameter("tls-creds", "tls0", func(res string) {
+				if strings.Contains(strings.ToLower(res), "error") {
+					s.migrateTask = nil
+					hostutils.TaskFailed(s.ctx, fmt.Sprintf("Migrate set tls-creds tls0 error: %s", res))
+					return
+				}
+				s.doMigrate()
+			})
+		})
+	} else {
+		s.doMigrate()
+	}
+}
 
+func (s *SGuestLiveMigrateTask) doMigrate() {
 	var copyIncremental = false
 	if s.params.IsLocal {
 		// copy disk data
@@ -747,19 +774,32 @@ type SGuestResumeTask struct {
 	startTime time.Time
 
 	isTimeout bool
+	cleanTLS  bool
 }
 
-func NewGuestResumeTask(ctx context.Context, s *SKVMGuestInstance, isTimeout bool) *SGuestResumeTask {
+func NewGuestResumeTask(ctx context.Context, s *SKVMGuestInstance, isTimeout bool, cleanTLS bool) *SGuestResumeTask {
 	return &SGuestResumeTask{
 		SKVMGuestInstance: s,
 		ctx:               ctx,
 		isTimeout:         isTimeout,
+		cleanTLS:          cleanTLS,
 	}
 }
 
 func (s *SGuestResumeTask) Start() {
 	log.Debugf("[%s] GuestResumeTask start", s.GetId())
 	s.startTime = time.Now()
+	if s.cleanTLS {
+		s.Monitor.ObjectDel("tls0", func(res string) {
+			log.Infof("Clean %s tls0 object: %s", s.GetName(), res)
+			pkiPath := s.getPKIDirPath()
+			if err := os.RemoveAll(pkiPath); err != nil {
+				log.Warningf("Remove tls pki dir %s error: %v", pkiPath, err)
+			}
+			s.confirmRunning()
+		})
+		return
+	}
 	s.confirmRunning()
 }
 

pkg/hostman/guestman/qemu-kvm.go

@@ -590,6 +590,35 @@ func (s *SKVMGuestInstance) onMonitorConnected(ctx context.Context) {
 	})
 }
 
+func (s *SKVMGuestInstance) setDestMigrateTLS(ctx context.Context, data *jsonutils.JSONDict) {
+	port, _ := data.Int("live_migrate_dest_port")
+	s.Monitor.ObjectAdd("tls-creds-x509", map[string]string{
+		"dir":         s.getPKIDirPath(),
+		"endpoint":    "server",
+		"id":          "tls0",
+		"verify-peer": "no",
+	}, func(res string) {
+		if strings.Contains(strings.ToLower(res), "error") {
+			hostutils.TaskFailed(ctx, fmt.Sprintf("Migrate add tls-creds-x509 object server tls0 error: %s", res))
+			return
+		}
+		s.Monitor.MigrateSetParameter("tls-creds", "tls0", func(res string) {
+			if strings.Contains(strings.ToLower(res), "error") {
+				hostutils.TaskFailed(ctx, fmt.Sprintf("Migrate set tls-creds tls0 error: %s", res))
+				return
+			}
+			address := fmt.Sprintf("tcp:0:%d", port)
+			s.Monitor.MigrateIncoming(address, func(res string) {
+				if strings.Contains(strings.ToLower(res), "error") {
+					hostutils.TaskFailed(ctx, fmt.Sprintf("Migrate set incoming %q error: %s", address, res))
+					return
+				}
+				hostutils.TaskComplete(ctx, data)
+			})
+		})
+	})
+}
+
 func (s *SKVMGuestInstance) onGetQemuVersion(ctx context.Context, version string) {
 	s.QemuVersion = version
 	log.Infof("Guest(%s) qemu version %s", s.Id, s.QemuVersion)
@@ -597,7 +626,11 @@ func (s *SKVMGuestInstance) onGetQemuVersion(ctx context.Context, version string
 		migratePort, _ := s.Desc.Get("live_migrate_dest_port")
 		body := jsonutils.NewDict()
 		body.Set("live_migrate_dest_port", migratePort)
-		hostutils.TaskComplete(ctx, body)
+		if jsonutils.QueryBoolean(s.Desc, "live_migrate_use_tls", false) {
+			s.setDestMigrateTLS(ctx, body)
+		} else {
+			hostutils.TaskComplete(ctx, body)
+		}
 	} else if s.IsSlave() {
 		s.startQemuBuiltInNbdServer(ctx)
 	} else if s.IsMaster() {
@@ -813,7 +846,7 @@ func (s *SKVMGuestInstance) saveVncPort(port int) error {
 }
 
 func (s *SKVMGuestInstance) DoResumeTask(ctx context.Context, isTimeout bool) {
-	s.startupTask = NewGuestResumeTask(ctx, s, isTimeout)
+	s.startupTask = NewGuestResumeTask(ctx, s, isTimeout, false)
 	s.startupTask.Start()
 }
 
@@ -1515,12 +1548,14 @@ func (s *SKVMGuestInstance) optimizeOom() error {
 	return fmt.Errorf("Guest %s not running?", s.GetId())
 }
 
-func (s *SKVMGuestInstance) SyncMetadata(meta *jsonutils.JSONDict) {
+func (s *SKVMGuestInstance) SyncMetadata(meta *jsonutils.JSONDict) error {
 	_, err := modules.Servers.SetMetadata(hostutils.GetComputeSession(context.Background()),
 		s.Id, meta)
 	if err != nil {
-		log.Errorln(err)
+		log.Errorln("sync metadata error: %v", err)
+		return errors.Wrap(err, "set metadata")
 	}
+	return nil
 }
 
 func (s *SKVMGuestInstance) SetVncPassword() {
@@ -1719,7 +1754,7 @@ func (s *SKVMGuestInstance) deleteStaticSnapshotFile(
 	return res, nil
 }
 
-func (s *SKVMGuestInstance) PrepareMigrate(liveMigrage bool) (*jsonutils.JSONDict, error) {
+func (s *SKVMGuestInstance) PrepareDisksMigrate(liveMigrage bool) (*jsonutils.JSONDict, error) {
 	disksBackFile := jsonutils.NewDict()
 	disks, _ := s.Desc.GetArray("disks")
 	for _, disk := range disks {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -32,7 +32,9 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/qemu"
+	qemucerts "yunion.io/x/onecloud/pkg/hostman/guestman/qemu/certs"
 	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemutils"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
@@ -494,6 +496,10 @@ function nic_mtu() {
 		migratePort := s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE)
 		s.Desc.Set("live_migrate_dest_port", jsonutils.NewInt(int64(migratePort)))
 		input.LiveMigratePort = uint(migratePort)
+		if jsonutils.QueryBoolean(data, "live_migrate_use_tls", false) {
+			input.LiveMigrateUseTLS = true
+			s.Desc.Set("live_migrate_use_tls", jsonutils.JSONTrue)
+		}
 	} else if jsonutils.QueryBoolean(s.Desc, "is_slave", false) {
 		input.IsSlave = true
 		input.LiveMigratePort = uint(s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE))
@@ -533,7 +539,7 @@ func (s *SKVMGuestInstance) parseCmdline(input string) (*qemutils.Cmdline, []qem
 	cl.FilterOption(func(o qemutils.Option) bool {
 		switch o.Key {
 		case "incoming":
-			if strings.HasPrefix(o.Value, "tcp:") {
+			if strings.HasPrefix(o.Value, "tcp:") || strings.HasPrefix(o.Value, "defer") {
 				filterOpts = append(filterOpts, o)
 				return true
 			}
@@ -685,3 +691,41 @@ func (s *SKVMGuestInstance) StartPresendArp() {
 		}
 	}()
 }
+
+func (s *SKVMGuestInstance) getPKIDirPath() string {
+	return path.Join(s.HomeDir(), "pki")
+}
+
+func (s *SKVMGuestInstance) makePKIDir() error {
+	output, err := procutils.NewCommand("mkdir", "-p", s.getPKIDirPath()).Output()
+	if err != nil {
+		return errors.Wrapf(err, "mkdir %s failed: %s", s.getPKIDirPath(), output)
+	}
+	return nil
+}
+
+func (s *SKVMGuestInstance) PrepareMigrateCerts() (map[string]string, error) {
+	pkiDir := s.getPKIDirPath()
+	if err := s.makePKIDir(); err != nil {
+		return nil, errors.Wrap(err, "make pki dir")
+	}
+	tree, err := qemucerts.GetDefaultCertList().AsMap().CertTree()
+	if err != nil {
+		return nil, errors.Wrap(err, "construct cert tree")
+	}
+	if err := tree.CreateTree(pkiDir); err != nil {
+		return nil, errors.Wrap(err, "create certs")
+	}
+	return qemucerts.FetchDefaultCerts(pkiDir)
+}
+
+func (s *SKVMGuestInstance) WriteMigrateCerts(certs map[string]string) error {
+	pkiDir := s.getPKIDirPath()
+	if err := s.makePKIDir(); err != nil {
+		return errors.Wrap(err, "make pki dir")
+	}
+	if err := qemucerts.CreateByMap(pkiDir, certs); err != nil {
+		return errors.Wrapf(err, "create by map %#v", certs)
+	}
+	return nil
+}

pkg/hostman/guestman/qemu/certs/certlist.go

@@ -0,0 +1,328 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package certs
+
+import (
+	"crypto"
+	"crypto/x509"
+	"fmt"
+	"path/filepath"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	certutil "yunion.io/x/onecloud/pkg/util/tls/cert"
+	pkiutil "yunion.io/x/onecloud/pkg/util/tls/pki"
+)
+
+type configMutatorsFunc func(*certutil.Config) error
+
+// QemuCert represents a cretificate that qemu required.
+type QemuCert struct {
+	Name           string
+	LongName       string
+	BaseName       string
+	CAName         string
+	configMutators []configMutatorsFunc
+	config         certutil.Config
+}
+
+// GetConfig returns the definition for the given cert.
+func (k *QemuCert) GetConfig() (*certutil.Config, error) {
+	for _, f := range k.configMutators {
+		if err := f(&k.config); err != nil {
+			return nil, err
+		}
+	}
+
+	return &k.config, nil
+}
+
+// CreateFromCA makes and writes a certificate using the given CA cert and key.
+func (k *QemuCert) CreateFromCA(dir string, caCert *x509.Certificate, caKey crypto.Signer) error {
+	cfg, err := k.GetConfig()
+	if err != nil {
+		return errors.Wrapf(err, "couldn't create %q certificate", k.Name)
+	}
+	cert, key, err := pkiutil.NewCertAndKey(
+		caCert, caKey,
+		&pkiutil.CertConfig{
+			Config: *cfg,
+		})
+	if err != nil {
+		return err
+	}
+
+	if err := writeCertificateFilesIfNotExist(
+		dir,
+		k.BaseName,
+		caCert,
+		cert,
+		key,
+		cfg,
+	); err != nil {
+		return errors.Wrapf(err, "failed to write or validate certificate %q", k.Name)
+	}
+
+	return nil
+}
+
+// CreateAsCA creates a certificate authority, writing the files to disk and also returning the created CA so it can be used to sign child certs.
+func (k *QemuCert) CreateAsCA(dir string) (*x509.Certificate, crypto.Signer, error) {
+	cfg, err := k.GetConfig()
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "couldn't get configuration for %q CA certificate", k.Name)
+	}
+	caCert, caKey, err := pkiutil.NewCertificateAuthority(&pkiutil.CertConfig{Config: *cfg})
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "couldn't generate %q CA certificate", k.Name)
+	}
+
+	if err := writeCertificateAuthorithyFilesIfNotExist(
+		dir,
+		k.BaseName,
+		caCert,
+		caKey,
+	); err != nil {
+		return nil, nil, errors.Wrapf(err, "couldn't write out %q CA certificate", k.Name)
+	}
+
+	return caCert, caKey, nil
+}
+
+// CertificateTree is represents a one-level-deep tree, mapping a CA to the certs that depend on it.
+type CertificateTree map[*QemuCert]Certificates
+
+// CreateTree creates the CAs, certs signed by the CAs, and writes them all to disk.
+func (t CertificateTree) CreateTree(dir string) error {
+	for ca, leaves := range t {
+		cfg, err := ca.GetConfig()
+		if err != nil {
+			return err
+		}
+
+		var caKey crypto.Signer
+
+		caCert, err := pkiutil.TryLoadCertFromDisk(dir, ca.BaseName)
+		if err == nil {
+			// Cert exists already, make sure it's valid
+			if !caCert.IsCA {
+				return errors.Errorf("certificate %q is not a CA", ca.Name)
+			}
+			// Try and load a CA Key
+			caKey, err = pkiutil.TryLoadKeyFromDisk(dir, ca.BaseName)
+			if err != nil {
+				// If there's no CA key, make sure every certificate exists.
+				for _, leaf := range leaves {
+					cl := certKeyLocation{
+						pkiDir:   dir,
+						baseName: leaf.BaseName,
+						uxName:   leaf.Name,
+					}
+					if err := validateSignedCertWithCA(cl, caCert); err != nil {
+						return errors.Wrapf(err, "could not load expected certificate %q or validate the existence of key %q for it", leaf.Name, ca.Name)
+					}
+				}
+				continue
+			}
+			// CA key exists; just use that to create new certificates.
+		} else {
+			// CACert doesn't already exist, create a new cert and key.
+			caCert, caKey, err = pkiutil.NewCertificateAuthority(&pkiutil.CertConfig{Config: *cfg})
+			if err != nil {
+				return err
+			}
+
+			err = writeCertificateAuthorithyFilesIfNotExist(
+				dir,
+				ca.BaseName,
+				caCert,
+				caKey,
+			)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, leaf := range leaves {
+			if err := leaf.CreateFromCA(dir, caCert, caKey); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// CertificateMap is a flat map of certificates, keyed by Name.
+type CertificateMap map[string]*QemuCert
+
+// CertTree returns a one-level-deep tree, mapping a CA cert to an array of certificates that should be signed by it.
+func (m CertificateMap) CertTree() (CertificateTree, error) {
+	caMap := make(CertificateTree)
+
+	for _, cert := range m {
+		if cert.CAName == "" {
+			if _, ok := caMap[cert]; !ok {
+				caMap[cert] = []*QemuCert{}
+			}
+		} else {
+			ca, ok := m[cert.CAName]
+			if !ok {
+				return nil, errors.Errorf("certificate %q references unknown CA %q", cert.Name, cert.CAName)
+			}
+			caMap[ca] = append(caMap[ca], cert)
+		}
+	}
+
+	return caMap, nil
+}
+
+// Certificates is a list of Certificates that should be created
+type Certificates []*QemuCert
+
+func (c Certificates) AsMap() CertificateMap {
+	certMap := make(map[string]*QemuCert)
+	for _, cert := range c {
+		certMap[cert.Name] = cert
+	}
+
+	return certMap
+}
+
+const (
+	CACertAndKeyBaseName     = "ca"
+	ServerCertBaseName       = "server"
+	QemuServerCertCommonName = "qemu-server"
+	ClientCertBaseName       = "client"
+	QemuClientCertCommonName = "qemu-client"
+)
+
+var (
+	QemuCertRootCA = QemuCert{
+		Name:     "ca",
+		LongName: "self-signed CA to provision identities for other qemu actions",
+		BaseName: CACertAndKeyBaseName,
+		config: certutil.Config{
+			CommonName: "qemu",
+		},
+	}
+
+	QemuCertServer = QemuCert{
+		Name:     "server",
+		LongName: "certificate for server",
+		BaseName: ServerCertBaseName,
+		CAName:   "ca",
+		config: certutil.Config{
+			CommonName: QemuServerCertCommonName,
+			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+			AltNames:   certutil.AltNames{
+				/*
+				 * IPs: []net.IP{
+				 * 	net.ParseIP("192.168.121.21"),
+				 * 	net.ParseIP("192.168.121.61"),
+				 * },
+				 */
+			},
+		},
+	}
+
+	QemuCertClient = QemuCert{
+		Name:     "client",
+		LongName: "certificate for the server to connect to client",
+		BaseName: ClientCertBaseName,
+		CAName:   "ca",
+		config: certutil.Config{
+			CommonName:   QemuClientCertCommonName,
+			Organization: []string{"system:host"},
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+			AltNames:     certutil.AltNames{
+				/*
+				 * IPs: []net.IP{
+				 * 	net.ParseIP("192.168.121.21"),
+				 * 	net.ParseIP("192.168.121.61"),
+				 * },
+				 */
+			},
+		},
+	}
+)
+
+func setCommonNameToNodeName(commonName string) configMutatorsFunc {
+	return func(cc *certutil.Config) error {
+		cc.CommonName = commonName
+		return nil
+	}
+}
+
+func init() {
+	pkiutil.SetPathForCert(func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s-cert.pem", name))
+	})
+
+	pkiutil.SetPathForKey(func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s-key.pem", name))
+	})
+}
+
+// GetDefaultCertList returns all of the certificates qemu requires.
+func GetDefaultCertList() Certificates {
+	return Certificates{
+		&QemuCertRootCA,
+		&QemuCertServer,
+		&QemuCertClient,
+	}
+}
+
+const (
+	CA_CERT_NAME     = "ca-cert.pem"
+	CA_KEY_NAME      = "ca-key.pem"
+	SERVER_CERT_NAME = "server-cert.pem"
+	SERVER_KEY_NAME  = "server-key.pem"
+	CLIENT_CERT_NAME = "client-cert.pem"
+	CLIENT_KEY_NAME  = "client-key.pem"
+)
+
+func FetchDefaultCerts(dir string) (map[string]string, error) {
+	ret := make(map[string]string)
+
+	for _, key := range []string{
+		CA_CERT_NAME,
+		CA_KEY_NAME,
+		SERVER_CERT_NAME,
+		SERVER_KEY_NAME,
+		CLIENT_CERT_NAME,
+		CLIENT_KEY_NAME,
+	} {
+		fp := filepath.Join(dir, key)
+		content, err := fileutils2.FileGetContents(fp)
+		if err != nil {
+			return nil, errors.Wrapf(err, "get %q content", fp)
+		}
+		ret[key] = content
+	}
+
+	return ret, nil
+}
+
+func CreateByMap(dir string, input map[string]string) error {
+	for key := range input {
+		fp := filepath.Join(dir, key)
+		content := input[key]
+		if err := fileutils2.FilePutContents(fp, content, false); err != nil {
+			return errors.Wrapf(err, "put %q to %q", content, fp)
+		}
+	}
+	return nil
+}

pkg/hostman/guestman/qemu/certs/certlist_test.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package certs
+
+import (
+	"crypto"
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	certutil "yunion.io/x/onecloud/pkg/util/tls/cert"
+)
+
+func TestCAPointersValid(t *testing.T) {
+	tests := []struct {
+		certs Certificates
+		name  string
+	}{
+		{
+			name:  "Default Certificate List",
+			certs: GetDefaultCertList(),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			certMap := test.certs.AsMap()
+
+			for _, cert := range test.certs {
+				if cert.CAName != "" && certMap[cert.CAName] == nil {
+					t.Errorf("Certificate %q references non existent CA %q", cert.Name, cert.CAName)
+				}
+			}
+		})
+	}
+}
+
+func TestMakeCertTree(t *testing.T) {
+	rootCert := &QemuCert{
+		Name: "root",
+	}
+	leaf0 := &QemuCert{
+		Name:   "leaf0",
+		CAName: "root",
+	}
+	leaf1 := &QemuCert{
+		Name:   "leaf1",
+		CAName: "root",
+	}
+	selfSigned := &QemuCert{
+		Name: "self-signed",
+	}
+
+	certMap := CertificateMap{
+		"root":        rootCert,
+		"leaf0":       leaf0,
+		"leaf1":       leaf1,
+		"self-signed": selfSigned,
+	}
+
+	orphanCertMap := CertificateMap{
+		"leaf0": leaf0,
+	}
+
+	if _, err := orphanCertMap.CertTree(); err == nil {
+		t.Error("expected orphan cert map to error, but got nil")
+	}
+
+	certTree, err := certMap.CertTree()
+	t.Logf("cert tree: %v", certTree)
+	if err != nil {
+		t.Errorf("expected no error, but got %v", err)
+	}
+
+	if len(certTree) != 2 {
+		t.Errorf("Expected tree to have 2 roots, got %d", len(certTree))
+	}
+
+	if len(certTree[rootCert]) != 2 {
+		t.Errorf("Expected root to have 2 leaves, got %d", len(certTree[rootCert]))
+	}
+
+	if _, ok := certTree[selfSigned]; !ok {
+		t.Error("Expected selfSigned to be present in tree, but missing")
+	}
+}
+
+func TestCreateCertificateChain(t *testing.T) {
+	dir, err := ioutil.TempDir("", t.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	caCfg := Certificates{
+		{
+			config:   certutil.Config{},
+			Name:     "test-ca",
+			BaseName: "test-ca",
+		},
+		{
+			config: certutil.Config{
+				AltNames: certutil.AltNames{
+					DNSNames: []string{"test-domain.space"},
+				},
+				Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+			},
+			configMutators: []configMutatorsFunc{
+				setCommonNameToNodeName("test-node"),
+			},
+			CAName:   "test-ca",
+			Name:     "test-daughter",
+			BaseName: "test-daughter",
+		},
+	}
+
+	certTree, err := caCfg.AsMap().CertTree()
+	if err != nil {
+		t.Fatalf("unexpected error getting tree: %v", err)
+	}
+
+	if certTree.CreateTree(dir); err != nil {
+		t.Fatal(err)
+	}
+
+	caCert, _ := parseCertAndKey(path.Join(dir, "test-ca"), t)
+	daughterCert, _ := parseCertAndKey(path.Join(dir, "test-daughter"), t)
+
+	pool := x509.NewCertPool()
+	pool.AddCert(caCert)
+
+	_, err = daughterCert.Verify(x509.VerifyOptions{
+		DNSName:   "test-domain.space",
+		Roots:     pool,
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	})
+	if err != nil {
+		t.Errorf("couldn't verify daughter cert: %v", err)
+	}
+
+}
+
+func parseCertAndKey(basePath string, t *testing.T) (*x509.Certificate, crypto.PrivateKey) {
+	certPair, err := tls.LoadX509KeyPair(basePath+"-cert.pem", basePath+"-key.pem")
+	if err != nil {
+		t.Fatalf("couldn't parse certificate and key: %v", err)
+	}
+
+	parsedCert, err := x509.ParseCertificate(certPair.Certificate[0])
+	if err != nil {
+		t.Fatalf("couldn't parse certificate: %v", err)
+	}
+
+	return parsedCert, certPair.PrivateKey
+}

pkg/hostman/guestman/qemu/certs/certs.go

@@ -0,0 +1,161 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package certs
+
+import (
+	"crypto"
+	"crypto/x509"
+	"fmt"
+
+	"yunion.io/x/pkg/errors"
+
+	certutil "yunion.io/x/onecloud/pkg/util/tls/cert"
+	pkiutil "yunion.io/x/onecloud/pkg/util/tls/pki"
+)
+
+type certKeyLocation struct {
+	pkiDir     string
+	caBaseName string
+	baseName   string
+	uxName     string
+}
+
+// validateSignedCert tries to load a x509 certificate and private key from pkiDir and validates
+// that the cert is signed by a given CA
+func validateSignedCert(l certKeyLocation) error {
+	// Try to load CA
+	caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName)
+	if err != nil {
+		return errors.Wrapf(err, "failure loading certificate authority for %s", l.uxName)
+	}
+
+	return validateSignedCertWithCA(l, caCert)
+}
+
+// validateSignedCertWithCA tries to load a certificate and validate it with the given caCert
+func validateSignedCertWithCA(l certKeyLocation, caCert *x509.Certificate) error {
+	// Try to load key and signed certificate
+	signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName)
+	if err != nil {
+		return errors.Wrapf(err, "failure loading certificate for %s", l.uxName)
+	}
+
+	// Check if the cert is signed by the CA
+	if err := signedCert.CheckSignatureFrom(caCert); err != nil {
+		return errors.Wrapf(err, "certificate %s is not signed by corresponding CA", l.uxName)
+	}
+	return nil
+}
+
+// validatePrivatePublicKey tries to load a private key from pkiDir
+func validatePrivatePublicKey(l certKeyLocation) error {
+	// Try to load key
+	_, _, err := pkiutil.TryLoadPrivatePublicKeyFromDisk(l.pkiDir, l.baseName)
+	if err != nil {
+		return errors.Wrapf(err, "failure loading key for %s", l.uxName)
+	}
+	return nil
+}
+
+// writeCertificateAuthorithyFilesIfNotExist write a new certificate Authority to the given path.
+// If there already is a certificate file at the given path; tries to load it and check if the values in the
+// existing and the expected certificate equals. If they do; will just skip writing the file as it's up-to-date,
+// otherwise this function returns an error.
+func writeCertificateAuthorithyFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey crypto.Signer) error {
+
+	// If cert or key exists, we should try to load them
+	if pkiutil.CertOrKeyExist(pkiDir, baseName) {
+
+		// Try to load .crt and .key from the PKI directory
+		caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName)
+		if err != nil {
+			return errors.Wrapf(err, "failure loading %s certificate", baseName)
+		}
+
+		// Check if the existing cert is a CA
+		if !caCert.IsCA {
+			return errors.Errorf("certificate %s is not a CA", baseName)
+		}
+
+		// kubeadm doesn't validate the existing certificate Authority more than this;
+		// Basically, if we find a certificate file with the same path; and it is a CA
+		// kubeadm thinks those files are equal and doesn't bother writing a new file
+		fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName)
+	} else {
+		// Write .crt and .key files to disk
+		fmt.Printf("[certs] Generating %q certificate and key\n", baseName)
+
+		if err := pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil {
+			return errors.Wrapf(err, "failure while saving %s certificate and key", baseName)
+		}
+	}
+	return nil
+}
+
+// writeCertificateFilesIfNotExist write a new certificate to the given path.
+// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the
+// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
+// otherwise this function returns an error.
+func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key crypto.Signer, cfg *certutil.Config) error {
+
+	// Checks if the signed certificate exists in the PKI directory
+	if pkiutil.CertOrKeyExist(pkiDir, baseName) {
+		// Try to load signed certificate .crt and .key from the PKI directory
+		signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName)
+		if err != nil {
+			return errors.Wrapf(err, "failure loading %s certificate", baseName)
+		}
+
+		// Check if the existing cert is signed by the given CA
+		if err := signedCert.CheckSignatureFrom(signingCert); err != nil {
+			return errors.Errorf("certificate %s is not signed by corresponding CA", baseName)
+		}
+
+		// Check if the certificate has the correct attributes
+		if err := validateCertificateWithConfig(signedCert, baseName, cfg); err != nil {
+			return err
+		}
+
+		fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName)
+	} else {
+		// Write .crt and .key files to disk
+		fmt.Printf("[certs] Generating %q certificate and key\n", baseName)
+
+		if err := pkiutil.WriteCertAndKey(pkiDir, baseName, cert, key); err != nil {
+			return errors.Wrapf(err, "failure while saving %s certificate and key", baseName)
+		}
+		if pkiutil.HasServerAuth(cert) {
+			fmt.Printf("[certs] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses)
+		}
+	}
+
+	return nil
+}
+
+// validateCertificateWithConfig makes sure that a given certificate is valid at
+// least for the SANs defined in the configuration.
+func validateCertificateWithConfig(cert *x509.Certificate, baseName string, cfg *certutil.Config) error {
+	for _, dnsName := range cfg.AltNames.DNSNames {
+		if err := cert.VerifyHostname(dnsName); err != nil {
+			return errors.Wrapf(err, "certificate %s is invalid", baseName)
+		}
+	}
+	for _, ipAddress := range cfg.AltNames.IPs {
+		if err := cert.VerifyHostname(ipAddress.String()); err != nil {
+			return errors.Wrapf(err, "certificate %s is invalid", baseName)
+		}
+	}
+	return nil
+}

pkg/hostman/guestman/qemu/certs/doc.go

@@ -0,0 +1 @@
+package certs // import "yunion.io/x/onecloud/pkg/hostman/guestman/qemu/certs"

pkg/hostman/guestman/qemu/generate.go

@@ -74,6 +74,7 @@ type GenerateStartOptionsInput struct {
 	EnableSerialDevice    bool
 	NeedMigrate           bool
 	LiveMigratePort       uint
+	LiveMigrateUseTLS     bool
 	IsSlave               bool
 	IsMaster              bool
 	EnablePvpanic         bool
@@ -491,7 +492,11 @@ func getRNGRandomOptions(drvOpt QemuOptions) []string {
 func getMigrateOptions(drvOpt QemuOptions, input *GenerateStartOptionsInput) []string {
 	opts := []string{}
 	if input.NeedMigrate {
-		opts = append(opts, fmt.Sprintf("-incoming tcp:0:%d", input.LiveMigratePort))
+		if input.LiveMigrateUseTLS {
+			opts = append(opts, fmt.Sprintf("-incoming defer"))
+		} else {
+			opts = append(opts, fmt.Sprintf("-incoming tcp:0:%d", input.LiveMigratePort))
+		}
 	} else if input.IsSlave {
 		opts = append(opts, fmt.Sprintf("-incoming tcp:0:%d", input.LiveMigratePort))
 	} else if input.IsMaster {

pkg/hostman/monitor/hmp.go

@@ -308,6 +308,16 @@ func (m *HmpMonitor) MigrateSetCapability(capability, state string, callback Str
 	m.Query(fmt.Sprintf("migrate_set_capability %s %s", capability, state), callback)
 }
 
+func (m *HmpMonitor) MigrateSetParameter(key string, val string, callback StringCallback) {
+	cmd := fmt.Sprintf("migrate_set_parameter %s %s", key, val)
+	m.Query(cmd, callback)
+}
+
+func (m *HmpMonitor) MigrateIncoming(address string, callback StringCallback) {
+	cmd := fmt.Sprintf("migrate_incoming %s", address)
+	m.Query(cmd, callback)
+}
+
 func (m *HmpMonitor) Migrate(
 	destStr string, copyIncremental, copyFull bool, callback StringCallback,
 ) {

pkg/hostman/monitor/monitor.go

@@ -175,6 +175,8 @@ type Monitor interface {
 	DriveMirror(callback StringCallback, drive, target, syncMode string, unmap, blockReplication bool)
 
 	MigrateSetCapability(capability, state string, callback StringCallback)
+	MigrateSetParameter(key, val string, callback StringCallback)
+	MigrateIncoming(address string, callback StringCallback)
 	Migrate(destStr string, copyIncremental, copyFull bool, callback StringCallback)
 	GetMigrateStatus(callback StringCallback)
 	MigrateStartPostcopy(callback StringCallback)

pkg/hostman/monitor/qmp.go

@@ -608,6 +608,42 @@ func (m *QmpMonitor) MigrateSetCapability(capability, state string, callback Str
 	m.Query(cmd, cb)
 }
 
+func (m *QmpMonitor) MigrateSetParameter(key string, val string, callback StringCallback) {
+	var (
+		cb = func(res *Response) {
+			callback(m.actionResult(res))
+		}
+		cmd = &Command{
+			Execute: "migrate-set-parameters",
+			Args: map[string]interface{}{
+				key: val,
+			},
+		}
+	)
+
+	m.Query(cmd, cb)
+}
+
+func (m *QmpMonitor) MigrateIncoming(address string, callback StringCallback) {
+	/*
+			 * var (
+			 * 	cb = func(res *Response) {
+			 * 		callback(m.actionResult(res))
+			 * 	}
+			 * 	cmd = &Command{
+			 * 		Execute: "migrate-incoming",
+			 * 		Args: map[string]interface{}{
+			 * 			"uri": address,
+			 * 		},
+			 * 	}
+			 * )
+		     *
+			 * m.Query(cmd, cb)
+	*/
+	cmd := fmt.Sprintf("migrate_incoming %s", address)
+	m.HumanMonitorCommand(cmd, callback)
+}
+
 func (m *QmpMonitor) Migrate(
 	destStr string, copyIncremental, copyFull bool, callback StringCallback,
 ) {

pkg/mcclient/options/compute/servers.go

@@ -917,6 +917,7 @@ type ServerLiveMigrateOptions struct {
 	PreferHost      string `help:"Server migration prefer host id or name" json:"prefer_host"`
 	SkipCpuCheck    *bool  `help:"Skip check CPU mode of the target host" json:"skip_cpu_check"`
 	SkipKernelCheck *bool  `help:"Skip target kernel version check" json:"skip_kernel_check"`
+	EnableTLS       *bool  `help:"Enable tls migration" json:"enable_tls"`
 }
 
 func (o *ServerLiveMigrateOptions) GetId() string {

pkg/util/tls/cert/cert.go

@@ -0,0 +1,206 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"bytes"
+	"crypto"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"path/filepath"
+	"strings"
+	"time"
+
+	keyutil "yunion.io/x/onecloud/pkg/util/tls/key"
+)
+
+const duration365d = time.Hour * 24 * 365
+
+// Config contains the basic fields required for creating a certificate
+type Config struct {
+	CommonName   string
+	Organization []string
+	AltNames     AltNames
+	Usages       []x509.ExtKeyUsage
+}
+
+// AltNames contains the domain names and IP addresses that will be added
+// to the API Server's x509 certificate SubAltNames field. The values will
+// be passed directly to the x509.Certificate object.
+type AltNames struct {
+	DNSNames []string
+	IPs      []net.IP
+}
+
+// NewSelfSignedCACert creates a CA certificate
+func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
+	now := time.Now()
+	tmpl := x509.Certificate{
+		SerialNumber: new(big.Int).SetInt64(0),
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		NotBefore:             now.UTC(),
+		NotAfter:              now.Add(duration365d * 100).UTC(),
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+
+	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
+	if err != nil {
+		return nil, err
+	}
+	return x509.ParseCertificate(certDERBytes)
+}
+
+// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
+// Host may be an IP or a DNS name
+// You may also specify additional subject alt names (either ip or dns names) for the certificate.
+func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
+	return GenerateSelfSignedCertKeyWithFixtures(host, alternateIPs, alternateDNS, "")
+}
+
+// GenerateSelfSignedCertKeyWithFixtures creates a self-signed certificate and key for the given host.
+// Host may be an IP or a DNS name. You may also specify additional subject alt names (either ip or dns names)
+// for the certificate.
+//
+// If fixtureDirectory is non-empty, it is a directory path which can contain pre-generated certs. The format is:
+// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
+// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
+// Certs/keys not existing in that directory are created.
+func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, alternateDNS []string, fixtureDirectory string) ([]byte, []byte, error) {
+	validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew
+	maxAge := time.Hour * 24 * 365          // one year self-signed certs
+
+	baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-"))
+	certFixturePath := filepath.Join(fixtureDirectory, baseName+".crt")
+	keyFixturePath := filepath.Join(fixtureDirectory, baseName+".key")
+	if len(fixtureDirectory) > 0 {
+		cert, err := ioutil.ReadFile(certFixturePath)
+		if err == nil {
+			key, err := ioutil.ReadFile(keyFixturePath)
+			if err == nil {
+				return cert, key, nil
+			}
+			return nil, nil, fmt.Errorf("cert %s can be read, but key %s cannot: %v", certFixturePath, keyFixturePath, err)
+		}
+		maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures
+	}
+
+	caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caTemplate := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()),
+		},
+		NotBefore: validFrom,
+		NotAfter:  validFrom.Add(maxAge),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+
+	caDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caCertificate, err := x509.ParseCertificate(caDERBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(2),
+		Subject: pkix.Name{
+			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
+		},
+		NotBefore: validFrom,
+		NotAfter:  validFrom.Add(maxAge),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	if ip := net.ParseIP(host); ip != nil {
+		template.IPAddresses = append(template.IPAddresses, ip)
+	} else {
+		template.DNSNames = append(template.DNSNames, host)
+	}
+
+	template.IPAddresses = append(template.IPAddresses, alternateIPs...)
+	template.DNSNames = append(template.DNSNames, alternateDNS...)
+
+	derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, caCertificate, &priv.PublicKey, caKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Generate cert, followed by ca
+	certBuffer := bytes.Buffer{}
+	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil {
+		return nil, nil, err
+	}
+	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: caDERBytes}); err != nil {
+		return nil, nil, err
+	}
+
+	// Generate key
+	keyBuffer := bytes.Buffer{}
+	if err := pem.Encode(&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
+		return nil, nil, err
+	}
+
+	if len(fixtureDirectory) > 0 {
+		if err := ioutil.WriteFile(certFixturePath, certBuffer.Bytes(), 0644); err != nil {
+			return nil, nil, fmt.Errorf("failed to write cert fixture to %s: %v", certFixturePath, err)
+		}
+		if err := ioutil.WriteFile(keyFixturePath, keyBuffer.Bytes(), 0644); err != nil {
+			return nil, nil, fmt.Errorf("failed to write key fixture to %s: %v", certFixturePath, err)
+		}
+	}
+
+	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
+}
+
+func ipsToStrings(ips []net.IP) []string {
+	ss := make([]string, 0, len(ips))
+	for _, ip := range ips {
+		ss = append(ss, ip.String())
+	}
+	return ss
+}

pkg/util/tls/cert/csr.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"net"
+)
+
+// MakeCSR generates a PEM-encoded CSR using the supplied private key, subject, and SANs.
+// All key types that are implemented via crypto.Signer are supported (This includes *rsa.PrivateKey and *ecdsa.PrivateKey.)
+func MakeCSR(privateKey interface{}, subject *pkix.Name, dnsSANs []string, ipSANs []net.IP) (csr []byte, err error) {
+	template := &x509.CertificateRequest{
+		Subject:     *subject,
+		DNSNames:    dnsSANs,
+		IPAddresses: ipSANs,
+	}
+
+	return MakeCSRFromTemplate(privateKey, template)
+}
+
+// MakeCSRFromTemplate generates a PEM-encoded CSR using the supplied private
+// key and certificate request as a template. All key types that are
+// implemented via crypto.Signer are supported (This includes *rsa.PrivateKey
+// and *ecdsa.PrivateKey.)
+func MakeCSRFromTemplate(privateKey interface{}, template *x509.CertificateRequest) ([]byte, error) {
+	t := *template
+	t.SignatureAlgorithm = sigType(privateKey)
+
+	csrDER, err := x509.CreateCertificateRequest(cryptorand.Reader, &t, privateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	csrPemBlock := &pem.Block{
+		Type:  CertificateRequestBlockType,
+		Bytes: csrDER,
+	}
+
+	return pem.EncodeToMemory(csrPemBlock), nil
+}
+
+func sigType(privateKey interface{}) x509.SignatureAlgorithm {
+	// Customize the signature for RSA keys, depending on the key size
+	if privateKey, ok := privateKey.(*rsa.PrivateKey); ok {
+		keySize := privateKey.N.BitLen()
+		switch {
+		case keySize >= 4096:
+			return x509.SHA512WithRSA
+		case keySize >= 3072:
+			return x509.SHA384WithRSA
+		default:
+			return x509.SHA256WithRSA
+		}
+	}
+	return x509.UnknownSignatureAlgorithm
+}

pkg/util/tls/cert/csr_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"io/ioutil"
+	"net"
+	"testing"
+
+	keyutil "yunion.io/x/onecloud/pkg/util/tls/key"
+)
+
+func TestMakeCSR(t *testing.T) {
+	keyFile := "testdata/dontUseThisKey.pem"
+	subject := &pkix.Name{
+		CommonName: "kube-worker",
+	}
+	dnsSANs := []string{"localhost"}
+	ipSANs := []net.IP{net.ParseIP("127.0.0.1")}
+
+	keyData, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	key, err := keyutil.ParsePrivateKeyPEM(keyData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	csrPEM, err := MakeCSR(key, subject, dnsSANs, ipSANs)
+	if err != nil {
+		t.Error(err)
+	}
+	csrBlock, rest := pem.Decode(csrPEM)
+	if csrBlock == nil {
+		t.Fatal("Unable to decode MakeCSR result.")
+	}
+	if len(rest) != 0 {
+		t.Error("Found more than one PEM encoded block in the result.")
+	}
+	if csrBlock.Type != CertificateRequestBlockType {
+		t.Errorf("Found block type %q, wanted 'CERTIFICATE REQUEST'", csrBlock.Type)
+	}
+	csr, err := x509.ParseCertificateRequest(csrBlock.Bytes)
+	if err != nil {
+		t.Errorf("Found %v parsing MakeCSR result as a CertificateRequest.", err)
+	}
+	if csr.Subject.CommonName != subject.CommonName {
+		t.Errorf("Wanted %v, got %v", subject, csr.Subject)
+	}
+	if len(csr.DNSNames) != 1 {
+		t.Errorf("Wanted 1 DNS name in the result, got %d", len(csr.DNSNames))
+	} else if csr.DNSNames[0] != dnsSANs[0] {
+		t.Errorf("Wanted %v, got %v", dnsSANs[0], csr.DNSNames[0])
+	}
+	if len(csr.IPAddresses) != 1 {
+		t.Errorf("Wanted 1 IP address in the result, got %d", len(csr.IPAddresses))
+	} else if csr.IPAddresses[0].String() != ipSANs[0].String() {
+		t.Errorf("Wanted %v, got %v", ipSANs[0], csr.IPAddresses[0])
+	}
+}

pkg/util/tls/cert/doc.go

@@ -0,0 +1 @@
+package cert // import "yunion.io/x/onecloud/pkg/util/tls/cert"

pkg/util/tls/cert/io.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// CanReadCertAndKey returns true if the certificate and key files already exists,
+// otherwise returns false. If lost one of cert and key, returns error.
+func CanReadCertAndKey(certPath, keyPath string) (bool, error) {
+	certReadable := canReadFile(certPath)
+	keyReadable := canReadFile(keyPath)
+
+	if certReadable == false && keyReadable == false {
+		return false, nil
+	}
+
+	if certReadable == false {
+		return false, fmt.Errorf("error reading %s, certificate and key must be supplied as a pair", certPath)
+	}
+
+	if keyReadable == false {
+		return false, fmt.Errorf("error reading %s, certificate and key must be supplied as a pair", keyPath)
+	}
+
+	return true, nil
+}
+
+// If the file represented by path exists and
+// readable, returns true otherwise returns false.
+func canReadFile(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+
+	defer f.Close()
+
+	return true
+}
+
+// WriteCert writes the pem-encoded certificate data to certPath.
+// The certificate file will be created with file mode 0644.
+// If the certificate file already exists, it will be overwritten.
+// The parent directory of the certPath will be created as needed with file mode 0755.
+func WriteCert(certPath string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(certPath), os.FileMode(0755)); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(certPath, data, os.FileMode(0644))
+}
+
+// NewPool returns an x509.CertPool containing the certificates in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func NewPool(filename string) (*x509.CertPool, error) {
+	pemBlock, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	pool, err := NewPoolFromBytes(pemBlock)
+	if err != nil {
+		return nil, fmt.Errorf("error creating pool from %s: %s", filename, err)
+	}
+	return pool, nil
+}
+
+// NewPoolFromBytes returns an x509.CertPool containing the certificates in the given PEM-encoded bytes.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func NewPoolFromBytes(pemBlock []byte) (*x509.CertPool, error) {
+	certs, err := ParseCertsPEM(pemBlock)
+	if err != nil {
+		return nil, err
+	}
+	pool := x509.NewCertPool()
+	for _, cert := range certs {
+		pool.AddCert(cert)
+	}
+	return pool, nil
+}
+
+// CertsFromFile returns the x509.Certificates contained in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func CertsFromFile(file string) ([]*x509.Certificate, error) {
+	pemBlock, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := ParseCertsPEM(pemBlock)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %s", file, err)
+	}
+	return certs, nil
+}

pkg/util/tls/cert/pem.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+)
+
+const (
+	// CertificateBlockType is a possible value for pem.Block.Type.
+	CertificateBlockType = "CERTIFICATE"
+	// CertificateRequestBlockType is a possible value for pem.Block.Type.
+	CertificateRequestBlockType = "CERTIFICATE REQUEST"
+)
+
+// ParseCertsPEM returns the x509.Certificates contained in the given PEM-encoded byte array
+// Returns an error if a certificate could not be parsed, or if the data does not contain any certificates
+func ParseCertsPEM(pemCerts []byte) ([]*x509.Certificate, error) {
+	ok := false
+	certs := []*x509.Certificate{}
+	for len(pemCerts) > 0 {
+		var block *pem.Block
+		block, pemCerts = pem.Decode(pemCerts)
+		if block == nil {
+			break
+		}
+		// Only use PEM "CERTIFICATE" blocks without extra headers
+		if block.Type != CertificateBlockType || len(block.Headers) != 0 {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return certs, err
+		}
+
+		certs = append(certs, cert)
+		ok = true
+	}
+
+	if !ok {
+		return certs, errors.New("data does not contain any valid RSA or ECDSA certificates")
+	}
+	return certs, nil
+}
+
+// EncodeCertificates returns the PEM-encoded byte array that represents by the specified certs.
+func EncodeCertificates(certs ...*x509.Certificate) ([]byte, error) {
+	b := bytes.Buffer{}
+	for _, cert := range certs {
+		if err := pem.Encode(&b, &pem.Block{Type: CertificateBlockType, Bytes: cert.Raw}); err != nil {
+			return []byte{}, err
+		}
+	}
+	return b.Bytes(), nil
+}

pkg/util/tls/cert/server_inspection.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// GetClientCANames gets the CA names for client certs that a server accepts.  This is useful when inspecting the
+// state of particular servers.  apiHost is "host:port"
+func GetClientCANames(apiHost string) ([]string, error) {
+	// when we run this the second time, we know which one we are expecting
+	acceptableCAs := []string{}
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true, // this is insecure to always get to the GetClientCertificate
+		GetClientCertificate: func(hello *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			acceptableCAs = []string{}
+			for _, curr := range hello.AcceptableCAs {
+				acceptableCAs = append(acceptableCAs, string(curr))
+			}
+			return &tls.Certificate{}, nil
+		},
+	}
+
+	conn, err := tls.Dial("tcp", apiHost, tlsConfig)
+	if err != nil {
+		return nil, err
+	}
+	if err := conn.Close(); err != nil {
+		return nil, err
+	}
+
+	return acceptableCAs, nil
+}
+
+// GetClientCANamesForURL is GetClientCANames against a URL string like we use in kubeconfigs
+func GetClientCANamesForURL(kubeConfigURL string) ([]string, error) {
+	apiserverURL, err := url.Parse(kubeConfigURL)
+	if err != nil {
+		return nil, err
+	}
+	return GetClientCANames(apiserverURL.Host)
+}
+
+// GetServingCertificates returns the x509 certs used by a server as certificates and pem encoded bytes.
+// The serverName is optional for specifying a different name to get SNI certificates.  apiHost is "host:port"
+func GetServingCertificates(apiHost, serverName string) ([]*x509.Certificate, [][]byte, error) {
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true, // this is insecure so that we always get connected
+	}
+	// if a name is specified for SNI, set it.
+	if len(serverName) > 0 {
+		tlsConfig.ServerName = serverName
+	}
+
+	conn, err := tls.Dial("tcp", apiHost, tlsConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err = conn.Close(); err != nil {
+		return nil, nil, fmt.Errorf("failed to close connection : %v", err)
+	}
+
+	peerCerts := conn.ConnectionState().PeerCertificates
+	peerCertBytes := [][]byte{}
+	for _, a := range peerCerts {
+		actualCert, err := EncodeCertificates(a)
+		if err != nil {
+			return nil, nil, err
+		}
+		peerCertBytes = append(peerCertBytes, []byte(strings.TrimSpace(string(actualCert))))
+	}
+
+	return peerCerts, peerCertBytes, err
+}
+
+// GetServingCertificatesForURL is GetServingCertificates against a URL string like we use in kubeconfigs
+func GetServingCertificatesForURL(kubeConfigURL, serverName string) ([]*x509.Certificate, [][]byte, error) {
+	apiserverURL, err := url.Parse(kubeConfigURL)
+	if err != nil {
+		return nil, nil, err
+	}
+	return GetServingCertificates(apiserverURL.Host, serverName)
+}

pkg/util/tls/cert/testdata/dontUseThisKey.pem

@@ -0,0 +1,7 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDAPEbSXwyDfWf0+61Oofd7aHkmdX69mrzD2Xb1CHF5syfsoRIhnG0dJ
+ozBulPZCDDWgBwYFK4EEACKhZANiAATjlMJAtKhEPqU/i7MsrgKcK/RmXHC6He7W
+0p69+9qFXg2raJ9zvvbKxkiu2ELOYRDAz0utcFTBOIgoUJEzBVmsjZQ7dvFa1BKP
+Ym7MFAKG3O2espBqXn+audgdHGh5B0I=
+-----END EC PRIVATE KEY-----
+

pkg/util/tls/key/doc.go

@@ -0,0 +1 @@
+package key // import "yunion.io/x/onecloud/pkg/util/tls/key"

pkg/util/tls/key/key.go

@@ -0,0 +1,323 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package key contains utilities for managing public/private key pairs.
+package key
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const (
+	// ECPrivateKeyBlockType is a possible value for pem.Block.Type.
+	ECPrivateKeyBlockType = "EC PRIVATE KEY"
+	// RSAPrivateKeyBlockType is a possible value for pem.Block.Type.
+	RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
+	// PrivateKeyBlockType is a possible value for pem.Block.Type.
+	PrivateKeyBlockType = "PRIVATE KEY"
+	// PublicKeyBlockType is a possible value for pem.Block.Type.
+	PublicKeyBlockType = "PUBLIC KEY"
+)
+
+// MakeEllipticPrivateKeyPEM creates an ECDSA private key
+func MakeEllipticPrivateKeyPEM() ([]byte, error) {
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	if err != nil {
+		return nil, err
+	}
+
+	derBytes, err := x509.MarshalECPrivateKey(privateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	privateKeyPemBlock := &pem.Block{
+		Type:  ECPrivateKeyBlockType,
+		Bytes: derBytes,
+	}
+	return pem.EncodeToMemory(privateKeyPemBlock), nil
+}
+
+// WriteKey writes the pem-encoded key data to keyPath.
+// The key file will be created with file mode 0600.
+// If the key file already exists, it will be overwritten.
+// The parent directory of the keyPath will be created as needed with file mode 0755.
+func WriteKey(keyPath string, data []byte) error {
+	if err := os.MkdirAll(filepath.Dir(keyPath), os.FileMode(0755)); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(keyPath, data, os.FileMode(0600))
+}
+
+// LoadOrGenerateKeyFile looks for a key in the file at the given path. If it
+// can't find one, it will generate a new key and store it there.
+func LoadOrGenerateKeyFile(keyPath string) (data []byte, wasGenerated bool, err error) {
+	loadedData, err := ioutil.ReadFile(keyPath)
+	// Call verifyKeyData to ensure the file wasn't empty/corrupt.
+	if err == nil && verifyKeyData(loadedData) {
+		return loadedData, false, err
+	}
+	if !os.IsNotExist(err) {
+		return nil, false, fmt.Errorf("error loading key from %s: %v", keyPath, err)
+	}
+
+	generatedData, err := MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, false, fmt.Errorf("error generating key: %v", err)
+	}
+	if err := WriteKey(keyPath, generatedData); err != nil {
+		return nil, false, fmt.Errorf("error writing key to %s: %v", keyPath, err)
+	}
+	return generatedData, true, nil
+}
+
+// MarshalPrivateKeyToPEM converts a known private key type of RSA or ECDSA to
+// a PEM encoded block or returns an error.
+func MarshalPrivateKeyToPEM(privateKey crypto.PrivateKey) ([]byte, error) {
+	switch t := privateKey.(type) {
+	case *ecdsa.PrivateKey:
+		derBytes, err := x509.MarshalECPrivateKey(t)
+		if err != nil {
+			return nil, err
+		}
+		block := &pem.Block{
+			Type:  ECPrivateKeyBlockType,
+			Bytes: derBytes,
+		}
+		return pem.EncodeToMemory(block), nil
+	case *rsa.PrivateKey:
+		block := &pem.Block{
+			Type:  RSAPrivateKeyBlockType,
+			Bytes: x509.MarshalPKCS1PrivateKey(t),
+		}
+		return pem.EncodeToMemory(block), nil
+	default:
+		return nil, fmt.Errorf("private key is not a recognized type: %T", privateKey)
+	}
+}
+
+// PrivateKeyFromFile returns the private key in rsa.PrivateKey or ecdsa.PrivateKey format from a given PEM-encoded file.
+// Returns an error if the file could not be read or if the private key could not be parsed.
+func PrivateKeyFromFile(file string) (interface{}, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	key, err := ParsePrivateKeyPEM(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading private key file %s: %v", file, err)
+	}
+	return key, nil
+}
+
+// PublicKeysFromFile returns the public keys in rsa.PublicKey or ecdsa.PublicKey format from a given PEM-encoded file.
+// Reads public keys from both public and private key files.
+func PublicKeysFromFile(file string) ([]interface{}, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	keys, err := ParsePublicKeysPEM(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading public key file %s: %v", file, err)
+	}
+	return keys, nil
+}
+
+// verifyKeyData returns true if the provided data appears to be a valid private key.
+func verifyKeyData(data []byte) bool {
+	if len(data) == 0 {
+		return false
+	}
+	_, err := ParsePrivateKeyPEM(data)
+	return err == nil
+}
+
+// ParsePrivateKeyPEM returns a private key parsed from a PEM block in the supplied data.
+// Recognizes PEM blocks for "EC PRIVATE KEY", "RSA PRIVATE KEY", or "PRIVATE KEY"
+func ParsePrivateKeyPEM(keyData []byte) (interface{}, error) {
+	var privateKeyPemBlock *pem.Block
+	for {
+		privateKeyPemBlock, keyData = pem.Decode(keyData)
+		if privateKeyPemBlock == nil {
+			break
+		}
+
+		switch privateKeyPemBlock.Type {
+		case ECPrivateKeyBlockType:
+			// ECDSA Private Key in ASN.1 format
+			if key, err := x509.ParseECPrivateKey(privateKeyPemBlock.Bytes); err == nil {
+				return key, nil
+			}
+		case RSAPrivateKeyBlockType:
+			// RSA Private Key in PKCS#1 format
+			if key, err := x509.ParsePKCS1PrivateKey(privateKeyPemBlock.Bytes); err == nil {
+				return key, nil
+			}
+		case PrivateKeyBlockType:
+			// RSA or ECDSA Private Key in unencrypted PKCS#8 format
+			if key, err := x509.ParsePKCS8PrivateKey(privateKeyPemBlock.Bytes); err == nil {
+				return key, nil
+			}
+		}
+
+		// tolerate non-key PEM blocks for compatibility with things like "EC PARAMETERS" blocks
+		// originally, only the first PEM block was parsed and expected to be a key block
+	}
+
+	// we read all the PEM blocks and didn't recognize one
+	return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA private key")
+}
+
+// ParsePublicKeysPEM is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded byte array.
+// Reads public keys from both public and private key files.
+func ParsePublicKeysPEM(keyData []byte) ([]interface{}, error) {
+	var block *pem.Block
+	keys := []interface{}{}
+	for {
+		// read the next block
+		block, keyData = pem.Decode(keyData)
+		if block == nil {
+			break
+		}
+
+		// test block against parsing functions
+		if privateKey, err := parseRSAPrivateKey(block.Bytes); err == nil {
+			keys = append(keys, &privateKey.PublicKey)
+			continue
+		}
+		if publicKey, err := parseRSAPublicKey(block.Bytes); err == nil {
+			keys = append(keys, publicKey)
+			continue
+		}
+		if privateKey, err := parseECPrivateKey(block.Bytes); err == nil {
+			keys = append(keys, &privateKey.PublicKey)
+			continue
+		}
+		if publicKey, err := parseECPublicKey(block.Bytes); err == nil {
+			keys = append(keys, publicKey)
+			continue
+		}
+
+		// tolerate non-key PEM blocks for backwards compatibility
+		// originally, only the first PEM block was parsed and expected to be a key block
+	}
+
+	if len(keys) == 0 {
+		return nil, fmt.Errorf("data does not contain any valid RSA or ECDSA public keys")
+	}
+	return keys, nil
+}
+
+// parseRSAPublicKey parses a single RSA public key from the provided data
+func parseRSAPublicKey(data []byte) (*rsa.PublicKey, error) {
+	var err error
+
+	// Parse the key
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKIXPublicKey(data); err != nil {
+		if cert, err := x509.ParseCertificate(data); err == nil {
+			parsedKey = cert.PublicKey
+		} else {
+			return nil, err
+		}
+	}
+
+	// Test if parsed key is an RSA Public Key
+	var pubKey *rsa.PublicKey
+	var ok bool
+	if pubKey, ok = parsedKey.(*rsa.PublicKey); !ok {
+		return nil, fmt.Errorf("data doesn't contain valid RSA Public Key")
+	}
+
+	return pubKey, nil
+}
+
+// parseRSAPrivateKey parses a single RSA private key from the provided data
+func parseRSAPrivateKey(data []byte) (*rsa.PrivateKey, error) {
+	var err error
+
+	// Parse the key
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKCS1PrivateKey(data); err != nil {
+		if parsedKey, err = x509.ParsePKCS8PrivateKey(data); err != nil {
+			return nil, err
+		}
+	}
+
+	// Test if parsed key is an RSA Private Key
+	var privKey *rsa.PrivateKey
+	var ok bool
+	if privKey, ok = parsedKey.(*rsa.PrivateKey); !ok {
+		return nil, fmt.Errorf("data doesn't contain valid RSA Private Key")
+	}
+
+	return privKey, nil
+}
+
+// parseECPublicKey parses a single ECDSA public key from the provided data
+func parseECPublicKey(data []byte) (*ecdsa.PublicKey, error) {
+	var err error
+
+	// Parse the key
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKIXPublicKey(data); err != nil {
+		if cert, err := x509.ParseCertificate(data); err == nil {
+			parsedKey = cert.PublicKey
+		} else {
+			return nil, err
+		}
+	}
+
+	// Test if parsed key is an ECDSA Public Key
+	var pubKey *ecdsa.PublicKey
+	var ok bool
+	if pubKey, ok = parsedKey.(*ecdsa.PublicKey); !ok {
+		return nil, fmt.Errorf("data doesn't contain valid ECDSA Public Key")
+	}
+
+	return pubKey, nil
+}
+
+// parseECPrivateKey parses a single ECDSA private key from the provided data
+func parseECPrivateKey(data []byte) (*ecdsa.PrivateKey, error) {
+	var err error
+
+	// Parse the key
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParseECPrivateKey(data); err != nil {
+		return nil, err
+	}
+
+	// Test if parsed key is an ECDSA Private Key
+	var privKey *ecdsa.PrivateKey
+	var ok bool
+	if privKey, ok = parsedKey.(*ecdsa.PrivateKey); !ok {
+		return nil, fmt.Errorf("data doesn't contain valid ECDSA Private Key")
+	}
+
+	return privKey, nil
+}

pkg/util/tls/key/key_test.go

@@ -0,0 +1,197 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package key
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+const (
+	// rsaPrivateKey is a RSA Private Key in PKCS#1 format
+	// openssl genrsa -out rsa2048.pem 2048
+	rsaPrivateKey = `-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA92mVjhBKOFsdxFzb/Pjq+7b5TJlODAdY5hK+WxLZTIrfhDPq
+FWrGKdjSNiHbXrdEtwJh9V+RqPZVSN3aWy1224RgkyNdMJsXhJKuCC24ZKY8SXtW
+xuTYmMRaMnCsv6QBGRTIbZ2EFbAObVM7lDyv1VqY3amZIWFQMlZ9CNpxDSPa5yi4
+3gopbXkne0oGNmey9X0qtpk7NMZIgAL6Zz4rZ30bcfC2ag6RLOFI2E/c4n8c38R8
+9MfXfLkj8/Cxo4JfI9NvRCpPOpFO8d/ZtWVUuIrBQN+Y7tkN2T60Qq/TkKXUrhDe
+fwlTlktZVJ/GztLYU41b2GcWsh/XO+PH831rmwIDAQABAoIBAQCC9c6GDjVbM0/E
+WurPMusfJjE7zII1d8YkspM0HfwLug6qKdikUYpnKC/NG4rEzfl/bbFwco/lgc6O
+7W/hh2U8uQttlvCDA/Uk5YddKOZL0Hpk4vaB/SxxYK3luSKXpjY2knutGg2KdVCN
+qdsFkkH4iyYTXuyBcMNEgedZQldI/kEujIH/L7FE+DF5TMzT4lHhozDoG+fy564q
+qVGUZXJn0ubc3GaPn2QOLNNM44sfYA4UJCpKBXPu85bvNObjxVQO4WqwwxU1vRnL
+UUsaGaelhSVJCo0dVPRvrfPPKZ09HTwpy40EkgQo6VriFc1EBoQDjENLbAJv9OfQ
+aCc9wiZhAoGBAP/8oEy48Zbb0P8Vdy4djf5tfBW8yXFLWzXewJ4l3itKS1r42nbX
+9q3cJsgRTQm8uRcMIpWxsc3n6zG+lREvTkoTB3ViI7+uQPiqA+BtWyNy7jzufFke
+ONKZfg7QxxmYRWZBRnoNGNbMpNeERuLmhvQuom9D1WbhzAYJbfs/O4WTAoGBAPds
+2FNDU0gaesFDdkIUGq1nIJqRQDW485LXZm4pFqBFxdOpbdWRuYT2XZjd3fD0XY98
+Nhkpb7NTMCuK3BdKcqIptt+cK+quQgYid0hhhgZbpCQ5AL6c6KgyjgpYlh2enzU9
+Zo3yg8ej1zbbA11sBlhX+5iO2P1u5DG+JHLwUUbZAoGAUwaU102EzfEtsA4+QW7E
+hyjrfgFlNKHES4yb3K9bh57pIfBkqvcQwwMMcQdrfSUAw0DkVrjzel0mI1Q09QXq
+1ould6UFAz55RC2gZEITtUOpkYmoOx9aPrQZ9qQwb1S77ZZuTVfCHqjxLhVxCFbM
+npYhiQTvShciHTMhwMOZgpECgYAVV5EtVXBYltgh1YTc3EkUzgF087R7LdHsx6Gx
+POATwRD4WfP8aQ58lpeqOPEM+LcdSlSMRRO6fyF3kAm+BJDwxfJdRWZQXumZB94M
+I0VhRQRaj4Qt7PDwmTPBVrTUJzuKZxpyggm17b8Bn1Ch/VBqzGQKW8AB1E/grosM
+UwhfuQKBgQC2JO/iqTQScHClf0qlItCJsBuVukFmSAVCkpOD8YdbdlPdOOwSk1wQ
+C0eAlsC3BCMvkpidKQmra6IqIrvTGI6EFgkrb3aknWdup2w8j2udYCNqyE3W+fVe
+p8FdYQ1FkACQ+daO5VlClL/9l0sGjKXlNKbpmJ2H4ngZmXj5uGmxuQ==
+-----END RSA PRIVATE KEY-----`
+
+	// rsaPublicKey is a RSA Public Key in PEM encoded format
+	// openssl rsa -in rsa2048.pem -pubout -out rsa2048pub.pem
+	rsaPublicKey = `-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA92mVjhBKOFsdxFzb/Pjq
++7b5TJlODAdY5hK+WxLZTIrfhDPqFWrGKdjSNiHbXrdEtwJh9V+RqPZVSN3aWy12
+24RgkyNdMJsXhJKuCC24ZKY8SXtWxuTYmMRaMnCsv6QBGRTIbZ2EFbAObVM7lDyv
+1VqY3amZIWFQMlZ9CNpxDSPa5yi43gopbXkne0oGNmey9X0qtpk7NMZIgAL6Zz4r
+Z30bcfC2ag6RLOFI2E/c4n8c38R89MfXfLkj8/Cxo4JfI9NvRCpPOpFO8d/ZtWVU
+uIrBQN+Y7tkN2T60Qq/TkKXUrhDefwlTlktZVJ/GztLYU41b2GcWsh/XO+PH831r
+mwIDAQAB
+-----END PUBLIC KEY-----`
+
+	// certificate is an x509 certificate in PEM encoded format
+	// openssl req -new -key rsa2048.pem -sha256 -nodes -x509 -days 1826 -out x509certificate.pem -subj "/C=US/CN=not-valid"
+	certificate = `-----BEGIN CERTIFICATE-----
+MIIDFTCCAf2gAwIBAgIJAN8B8NOwtiUCMA0GCSqGSIb3DQEBCwUAMCExCzAJBgNV
+BAYTAlVTMRIwEAYDVQQDDAlub3QtdmFsaWQwHhcNMTcwMzIyMDI1NjM2WhcNMjIw
+MzIyMDI1NjM2WjAhMQswCQYDVQQGEwJVUzESMBAGA1UEAwwJbm90LXZhbGlkMIIB
+IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA92mVjhBKOFsdxFzb/Pjq+7b5
+TJlODAdY5hK+WxLZTIrfhDPqFWrGKdjSNiHbXrdEtwJh9V+RqPZVSN3aWy1224Rg
+kyNdMJsXhJKuCC24ZKY8SXtWxuTYmMRaMnCsv6QBGRTIbZ2EFbAObVM7lDyv1VqY
+3amZIWFQMlZ9CNpxDSPa5yi43gopbXkne0oGNmey9X0qtpk7NMZIgAL6Zz4rZ30b
+cfC2ag6RLOFI2E/c4n8c38R89MfXfLkj8/Cxo4JfI9NvRCpPOpFO8d/ZtWVUuIrB
+QN+Y7tkN2T60Qq/TkKXUrhDefwlTlktZVJ/GztLYU41b2GcWsh/XO+PH831rmwID
+AQABo1AwTjAdBgNVHQ4EFgQU1I5GfinLF7ta+dBJ6UWcrYaexLswHwYDVR0jBBgw
+FoAU1I5GfinLF7ta+dBJ6UWcrYaexLswDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
+AQsFAAOCAQEAUl0wUD4y41juHFOVMYiziPYr1ShSpQXdwp8FfaHrzI5hsr8UMe8D
+dzb9QzZ4bx3yZhiG3ahrSBh956thMTHrKTEwAfJIEXI4cuSVWQAaOJ4Em5SDFxQe
+d0E6Ui2nGh1SFGF7oyuEXyzqgRMWFNDFw9HLUNgXaO18Zfouw8+K0BgbfEWEcSi1
+JLQbyhCjz088gltrliQGPWDFAg9cHBKtJhuTzZkvuqK1CLEmBhtzP1zFiGBfOJc8
+v+aKjAwrPUNX11cXOCPxBv2qXMetxaovBem6AI2hvypCInXaVQfP+yOLubzlTDjS
+Y708SlY38hmS1uTwDpyLOn8AKkZ8jtx75g==
+-----END CERTIFICATE-----`
+
+	// ecdsaPrivateKeyWithParams is a ECDSA Private Key with included EC Parameters block
+	// openssl ecparam -name prime256v1 -genkey -out ecdsa256params.pem
+	ecdsaPrivateKeyWithParams = `-----BEGIN EC PARAMETERS-----
+BggqhkjOPQMBBw==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIAwSOWQqlMTZNqNF7tgua812Jxib1DVOgb2pHHyIEyNNoAoGCCqGSM49
+AwEHoUQDQgAEyxYNrs6a6tsNCFNYn+l+JDUZ0PnUZbcsDgJn2O62D1se8M5iQ5rY
+iIv6RpxE3VHvlHEIvYgCZkG0jHszTUopBg==
+-----END EC PRIVATE KEY-----`
+
+	// ecdsaPrivateKey is a ECDSA Private Key in ASN.1 format
+	// openssl ecparam -name prime256v1 -genkey -noout -out ecdsa256.pem
+	ecdsaPrivateKey = `-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIP6Qw6dHDiLsSnLXUhQVTPE0fTQQrj3XSbiQAZPXnk5+oAoGCCqGSM49
+AwEHoUQDQgAEZZzi1u5f2/AEGFI/HYUhU+u6cTK1q2bbtE7r1JMK+/sQA5sNAp+7
+Vdc3psr1OaNzyTyuhTECyRdFKXm63cMnGg==
+-----END EC PRIVATE KEY-----`
+
+	// ecdsaPublicKey is a ECDSA Public Key in PEM encoded format
+	// openssl ec -in ecdsa256.pem -pubout -out ecdsa256pub.pem
+	ecdsaPublicKey = `-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZZzi1u5f2/AEGFI/HYUhU+u6cTK1
+q2bbtE7r1JMK+/sQA5sNAp+7Vdc3psr1OaNzyTyuhTECyRdFKXm63cMnGg==
+-----END PUBLIC KEY-----`
+)
+
+func TestReadPrivateKey(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("error creating tmpfile: %v", err)
+	}
+	defer os.Remove(f.Name())
+
+	if _, err := PrivateKeyFromFile(f.Name()); err == nil {
+		t.Fatalf("Expected error reading key from empty file, got none")
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(rsaPrivateKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing private key to tmpfile: %v", err)
+	}
+	if _, err := PrivateKeyFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading private RSA key: %v", err)
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(ecdsaPrivateKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing private key to tmpfile: %v", err)
+	}
+	if _, err := PrivateKeyFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading private ECDSA key: %v", err)
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(ecdsaPrivateKeyWithParams), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing private key to tmpfile: %v", err)
+	}
+	if _, err := PrivateKeyFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading private ECDSA key with params: %v", err)
+	}
+}
+
+func TestReadPublicKeys(t *testing.T) {
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("error creating tmpfile: %v", err)
+	}
+	defer os.Remove(f.Name())
+
+	if _, err := PublicKeysFromFile(f.Name()); err == nil {
+		t.Fatalf("Expected error reading keys from empty file, got none")
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(rsaPublicKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing public key to tmpfile: %v", err)
+	}
+	if keys, err := PublicKeysFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading RSA public key: %v", err)
+	} else if len(keys) != 1 {
+		t.Fatalf("expected 1 key, got %d", len(keys))
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(ecdsaPublicKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing public key to tmpfile: %v", err)
+	}
+	if keys, err := PublicKeysFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading ECDSA public key: %v", err)
+	} else if len(keys) != 1 {
+		t.Fatalf("expected 1 key, got %d", len(keys))
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(rsaPublicKey+"\n"+ecdsaPublicKey), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing public key to tmpfile: %v", err)
+	}
+	if keys, err := PublicKeysFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading combined RSA/ECDSA public key file: %v", err)
+	} else if len(keys) != 2 {
+		t.Fatalf("expected 2 keys, got %d", len(keys))
+	}
+
+	if err := ioutil.WriteFile(f.Name(), []byte(certificate), os.FileMode(0600)); err != nil {
+		t.Fatalf("error writing certificate to tmpfile: %v", err)
+	}
+	if keys, err := PublicKeysFromFile(f.Name()); err != nil {
+		t.Fatalf("error reading public key from certificate file: %v", err)
+	} else if len(keys) != 1 {
+		t.Fatalf("expected 1 keys, got %d", len(keys))
+	}
+
+}

pkg/util/tls/pki/doc.go

@@ -0,0 +1 @@
+package pki // import "yunion.io/x/onecloud/pkg/util/tls/pki"

pkg/util/tls/pki/pki_helpers.go

@@ -0,0 +1,528 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pki
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"math"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/sets"
+
+	certutil "yunion.io/x/onecloud/pkg/util/tls/cert"
+	keyutil "yunion.io/x/onecloud/pkg/util/tls/key"
+)
+
+const (
+	// PrivateKeyBlockType is a possible value for pem.Block.Type.
+	PrivateKeyBlockType = "PRIVATE KEY"
+	// PublicKeyBlockType is a possible value for pem.Block.Type.
+	PublicKeyBlockType = "PUBLIC KEY"
+	// CertificateBlockType is a possible value for pem.Block.Type.
+	CertificateBlockType = "CERTIFICATE"
+	// RSAPrivateKeyBlockType is a possible value for pem.BlockType.
+	RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
+	rsaKeySize             = 2048
+)
+
+var (
+	CertificateValidity = time.Hour * 24 * 365 * 100
+)
+
+// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm.
+type CertConfig struct {
+	certutil.Config
+	PublicKeyAlgorithm x509.PublicKeyAlgorithm
+}
+
+// NewCertificateAuthority creates new certificate and private key for the certificate authority
+func NewCertificateAuthority(config *CertConfig) (*x509.Certificate, crypto.Signer, error) {
+	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to create private key while generating CA certificate")
+	}
+
+	cert, err := certutil.NewSelfSignedCACert(config.Config, key)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to create self-signed CA certificate")
+	}
+
+	return cert, key, nil
+}
+
+// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
+func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) {
+	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to create private key")
+	}
+
+	cert, err := NewSignedCert(config, key, caCert, caKey)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to sign certificate")
+	}
+
+	return cert, key, nil
+}
+
+// NewCSR creates a new CSR
+func NewCSR(cfg CertConfig, key crypto.Signer) (*x509.CertificateRequest, error) {
+	template := &x509.CertificateRequest{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:    cfg.AltNames.DNSNames,
+		IPAddresses: cfg.AltNames.IPs,
+	}
+
+	csrBytes, err := x509.CreateCertificateRequest(cryptorand.Reader, template, key)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create a CSR")
+	}
+
+	return x509.ParseCertificateRequest(csrBytes)
+}
+
+// NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate
+func NewCSRAndKey(config *CertConfig) (*x509.CertificateRequest, crypto.Signer, error) {
+	key, err := NewPrivateKey(config.PublicKeyAlgorithm)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to create private key")
+	}
+
+	csr, err := NewCSR(*config, key)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "unable to generate CSR")
+	}
+
+	return csr, key, nil
+}
+
+// HasServerAuth returns true if the given certificate is a ServerAuth
+func HasServerAuth(cert *x509.Certificate) bool {
+	for i := range cert.ExtKeyUsage {
+		if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth {
+			return true
+		}
+	}
+	return false
+}
+
+// WriteCertAndKey stores certificate and key at the specified location
+func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key crypto.Signer) error {
+	if err := WriteKey(pkiPath, name, key); err != nil {
+		return errors.Wrap(err, "couldn't write key")
+	}
+
+	return WriteCert(pkiPath, name, cert)
+}
+
+// WriteCert stores the given certificate at the given location
+func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
+	if cert == nil {
+		return errors.Error("certificate cannot be nil when writing to file")
+	}
+
+	certificatePath := pathForCert(pkiPath, name)
+	if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil {
+		return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath)
+	}
+
+	return nil
+}
+
+// WriteKey stores the given key at the given location
+func WriteKey(pkiPath, name string, key crypto.Signer) error {
+	if key == nil {
+		return errors.Error("private key cannot be nil when writing to file")
+	}
+
+	privateKeyPath := pathForKey(pkiPath, name)
+	encoded, err := keyutil.MarshalPrivateKeyToPEM(key)
+	if err != nil {
+		return errors.Wrapf(err, "unable to marshal private key to PEM")
+	}
+	if err := keyutil.WriteKey(privateKeyPath, encoded); err != nil {
+		return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath)
+	}
+
+	return nil
+}
+
+// WriteCSR writes the pem-encoded CSR data to csrPath.
+// The CSR file will be created with file mode 0600.
+// If the CSR file already exists, it will be overwritten.
+// The parent directory of the csrPath will be created as needed with file mode 0700.
+func WriteCSR(csrDir, name string, csr *x509.CertificateRequest) error {
+	if csr == nil {
+		return errors.Error("certificate request cannot be nil when writing to file")
+	}
+
+	csrPath := pathForCSR(csrDir, name)
+	if err := os.MkdirAll(filepath.Dir(csrPath), os.FileMode(0700)); err != nil {
+		return errors.Wrapf(err, "failed to make directory %s", filepath.Dir(csrPath))
+	}
+
+	if err := ioutil.WriteFile(csrPath, EncodeCSRPEM(csr), os.FileMode(0600)); err != nil {
+		return errors.Wrapf(err, "unable to write CSR to file %s", csrPath)
+	}
+
+	return nil
+}
+
+// WritePublicKey stores the given public key at the given location
+func WritePublicKey(pkiPath, name string, key crypto.PublicKey) error {
+	if key == nil {
+		return errors.Error("public key cannot be nil when writing to file")
+	}
+
+	publicKeyBytes, err := EncodePublicKeyPEM(key)
+	if err != nil {
+		return err
+	}
+	publicKeyPath := pathForPublicKey(pkiPath, name)
+	if err := keyutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil {
+		return errors.Wrapf(err, "unable to write public key to file %s", publicKeyPath)
+	}
+
+	return nil
+}
+
+// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid
+func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, crypto.Signer, error) {
+	cert, err := TryLoadCertFromDisk(pkiPath, name)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to load certificate")
+	}
+
+	key, err := TryLoadKeyFromDisk(pkiPath, name)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to load key")
+	}
+
+	return cert, key, nil
+}
+
+// TryLoadCertFromDisk tries to load the cert from the disk and validates that it is valid
+func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) {
+	certificatePath := pathForCert(pkiPath, name)
+
+	certs, err := certutil.CertsFromFile(certificatePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath)
+	}
+
+	// We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one
+	// TODO: Support multiple certs here in order to be able to rotate certs
+	cert := certs[0]
+
+	// Check so that the certificate is valid now
+	now := time.Now()
+	if now.Before(cert.NotBefore) {
+		return nil, errors.Error("the certificate is not valid yet")
+	}
+	if now.After(cert.NotAfter) {
+		return nil, errors.Error("the certificate has expired")
+	}
+
+	return cert, nil
+}
+
+// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid
+func TryLoadKeyFromDisk(pkiPath, name string) (crypto.Signer, error) {
+	privateKeyPath := pathForKey(pkiPath, name)
+
+	// Parse the private key from a file
+	privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
+	}
+
+	// Allow RSA and ECDSA formats only
+	var key crypto.Signer
+	switch k := privKey.(type) {
+	case *rsa.PrivateKey:
+		key = k
+	case *ecdsa.PrivateKey:
+		key = k
+	default:
+		return nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath)
+	}
+
+	return key, nil
+}
+
+// TryLoadCSRAndKeyFromDisk tries to load the CSR and key from the disk
+func TryLoadCSRAndKeyFromDisk(pkiPath, name string) (*x509.CertificateRequest, crypto.Signer, error) {
+	csr, err := TryLoadCSRFromDisk(pkiPath, name)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not load CSR file")
+	}
+
+	key, err := TryLoadKeyFromDisk(pkiPath, name)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not load key file")
+	}
+
+	return csr, key, nil
+}
+
+// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
+func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) {
+	privateKeyPath := pathForKey(pkiPath, name)
+
+	// Parse the private key from a file
+	privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath)
+	}
+
+	publicKeyPath := pathForPublicKey(pkiPath, name)
+
+	// Parse the public key from a file
+	pubKeys, err := keyutil.PublicKeysFromFile(publicKeyPath)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "couldn't load the public key file %s", publicKeyPath)
+	}
+
+	// Allow RSA format only
+	k, ok := privKey.(*rsa.PrivateKey)
+	if !ok {
+		return nil, nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
+	}
+
+	p := pubKeys[0].(*rsa.PublicKey)
+
+	return k, p, nil
+}
+
+// TryLoadCSRFromDisk tries to load the CSR from the disk
+func TryLoadCSRFromDisk(pkiPath, name string) (*x509.CertificateRequest, error) {
+	csrPath := pathForCSR(pkiPath, name)
+
+	csr, err := CertificateRequestFromFile(csrPath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not load the CSR %s", csrPath)
+	}
+
+	return csr, nil
+}
+
+// CertificateRequestFromFile returns the CertificateRequest from a given PEM-encoded file.
+// Returns an error if the file could not be read or if the CSR could not be parsed.
+func CertificateRequestFromFile(file string) (*x509.CertificateRequest, error) {
+	pemBlock, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read file")
+	}
+
+	csr, err := parseCSRPEM(pemBlock)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading certificate request file %s", file)
+	}
+	return csr, nil
+}
+
+func parseCSRPEM(pemCSR []byte) (*x509.CertificateRequest, error) {
+	block, _ := pem.Decode(pemCSR)
+	if block == nil {
+		return nil, errors.Error("data doesn't contain a valid certificate request")
+	}
+
+	if block.Type != certutil.CertificateRequestBlockType {
+		return nil, errors.Errorf("expected block type %q, but PEM had type %q", certutil.CertificateRequestBlockType, block.Type)
+	}
+
+	return x509.ParseCertificateRequest(block.Bytes)
+}
+
+// CertOrKeyExist returns a boolean whether the cert or the key exists
+func CertOrKeyExist(pkiPath, name string) bool {
+	certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name)
+
+	_, certErr := os.Stat(certificatePath)
+	_, keyErr := os.Stat(privateKeyPath)
+	if os.IsNotExist(certErr) && os.IsNotExist(keyErr) {
+		// The cert and the key do not exist
+		return false
+	}
+
+	// Both files exist or one of them
+	return true
+}
+
+// CSROrKeyExist returns true if one of the CSR or key exists
+func CSROrKeyExist(csrDir, name string) bool {
+	csrPath := pathForCSR(csrDir, name)
+	keyPath := pathForKey(csrDir, name)
+
+	_, csrErr := os.Stat(csrPath)
+	_, keyErr := os.Stat(keyPath)
+
+	return !(os.IsNotExist(csrErr) && os.IsNotExist(keyErr))
+}
+
+// EncodePublicKeyPEM returns PEM-encoded public data
+func EncodePublicKeyPEM(key crypto.PublicKey) ([]byte, error) {
+	der, err := x509.MarshalPKIXPublicKey(key)
+	if err != nil {
+		return []byte{}, err
+	}
+	block := pem.Block{
+		Type:  PublicKeyBlockType,
+		Bytes: der,
+	}
+	return pem.EncodeToMemory(&block), nil
+}
+
+// EncodeCSRPEM returns PEM-encoded CSR data
+func EncodeCSRPEM(csr *x509.CertificateRequest) []byte {
+	block := pem.Block{
+		Type:  certutil.CertificateRequestBlockType,
+		Bytes: csr.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+// EncodeCertPEM returns PEM-endcoded certificate data
+func EncodeCertPEM(cert *x509.Certificate) []byte {
+	block := pem.Block{
+		Type:  CertificateBlockType,
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+// NewPrivateKey creates an RSA private key
+func NewPrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) {
+	if keyType == x509.ECDSA {
+		return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	}
+
+	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
+}
+
+// PathsForCertAndKey returns the paths for the certificate and key given the path and basename.
+func PathsForCertAndKey(pkiPath, name string) (string, string) {
+	return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
+}
+
+var (
+	pathForCert = func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
+	}
+
+	pathForKey = func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
+	}
+
+	pathForPublicKey = func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name))
+	}
+
+	pathForCSR = func(pkiPath, name string) string {
+		return filepath.Join(pkiPath, fmt.Sprintf("%s.csr", name))
+	}
+)
+
+func SetPathForCert(sf func(pkiPath, name string) string) {
+	pathForCert = sf
+}
+
+func SetPathForKey(sf func(pkiPath, name string) string) {
+	pathForKey = sf
+}
+
+func SetPathForPublicKey(sf func(pkiPath, name string) string) {
+	pathForPublicKey = sf
+}
+
+func SetPathForCSR(sf func(pkiPath, name string) string) {
+	pathForCSR = sf
+}
+
+// NewSignedCert creates a signed certificate using the given CA certificate and key
+func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
+	serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.CommonName) == 0 {
+		return nil, errors.Error("must specify a CommonName")
+	}
+	if len(cfg.Usages) == 0 {
+		return nil, errors.Error("must specify at least one ExtKeyUsage")
+	}
+
+	RemoveDuplicateAltNames(&cfg.AltNames)
+
+	certTmpl := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+		NotBefore:    caCert.NotBefore,
+		NotAfter:     time.Now().Add(CertificateValidity).UTC(),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  cfg.Usages,
+	}
+	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
+	if err != nil {
+		return nil, err
+	}
+	return x509.ParseCertificate(certDERBytes)
+}
+
+// RemoveDuplicateAltNames removes duplicate items in altNames.
+func RemoveDuplicateAltNames(altNames *certutil.AltNames) {
+	if altNames == nil {
+		return
+	}
+
+	if altNames.DNSNames != nil {
+		altNames.DNSNames = sets.NewString(altNames.DNSNames...).List()
+	}
+
+	ipsKeys := make(map[string]struct{})
+	var ips []net.IP
+	for _, one := range altNames.IPs {
+		if _, ok := ipsKeys[one.String()]; !ok {
+			ipsKeys[one.String()] = struct{}{}
+			ips = append(ips, one)
+		}
+	}
+	altNames.IPs = ips
+}