From 5ffcdb0bd215c9e1bfd0cda92871dc23dfe3d97b Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Mon, 4 Jul 2022 14:57:40 +0800 Subject: [PATCH] refactor(main): pkg/hosts->utils, pkg/token->pkg/runtime/token --- cmd/sealctl/cmd/hosts.go | 3 +- cmd/sealctl/cmd/ipvs.go | 3 +- cmd/sealctl/cmd/token.go | 5 +- pkg/ipvs/lvscare.go | 3 +- .../apis}/bootstraptoken/v1/utils.go | 0 .../apis}/bootstraptoken/v1/utils_test.go | 0 pkg/runtime/kubeadm.go | 5 +- pkg/runtime/kubeadm_cmd.go | 84 +++++++ pkg/runtime/runtime.go | 4 +- pkg/runtime/token.go | 218 ++++++++++++++---- pkg/token/bootstraptoken/doc.go | 21 -- pkg/token/bootstraptoken/v1/types.go | 58 ----- .../v1/zz_generated.deepcopy.go | 66 ------ pkg/token/pins.go | 112 --------- pkg/token/token.go | 116 ---------- pkg/{ => utils}/hosts/hosts.go | 0 16 files changed, 267 insertions(+), 431 deletions(-) rename pkg/{token => runtime/apis}/bootstraptoken/v1/utils.go (100%) rename pkg/{token => runtime/apis}/bootstraptoken/v1/utils_test.go (100%) create mode 100644 pkg/runtime/kubeadm_cmd.go delete mode 100644 pkg/token/bootstraptoken/doc.go delete mode 100644 pkg/token/bootstraptoken/v1/types.go delete mode 100644 pkg/token/bootstraptoken/v1/zz_generated.deepcopy.go delete mode 100644 pkg/token/pins.go delete mode 100644 pkg/token/token.go rename pkg/{ => utils}/hosts/hosts.go (100%) diff --git a/cmd/sealctl/cmd/hosts.go b/cmd/sealctl/cmd/hosts.go index fe3afb5c8..c4a22f70d 100644 --- a/cmd/sealctl/cmd/hosts.go +++ b/cmd/sealctl/cmd/hosts.go @@ -19,11 +19,12 @@ package cmd import ( "os" + "github.com/labring/sealos/pkg/utils/hosts" + "github.com/labring/sealos/pkg/utils/constants" "github.com/spf13/cobra" - "github.com/labring/sealos/pkg/hosts" "github.com/labring/sealos/pkg/utils/logger" ) diff --git a/cmd/sealctl/cmd/ipvs.go b/cmd/sealctl/cmd/ipvs.go index 0388c7100..fdbf92e4a 100644 --- a/cmd/sealctl/cmd/ipvs.go +++ b/cmd/sealctl/cmd/ipvs.go @@ -17,9 +17,10 @@ package cmd import ( "net" + "github.com/labring/sealos/pkg/utils/hosts" + "github.com/labring/lvscare/care" "github.com/labring/lvscare/service" - "github.com/labring/sealos/pkg/hosts" "github.com/labring/sealos/pkg/utils/constants" "github.com/labring/sealos/pkg/utils/flags" "github.com/labring/sealos/pkg/utils/iputils" diff --git a/cmd/sealctl/cmd/token.go b/cmd/sealctl/cmd/token.go index ba2a67613..594c1fad7 100755 --- a/cmd/sealctl/cmd/token.go +++ b/cmd/sealctl/cmd/token.go @@ -17,10 +17,11 @@ package cmd import ( "os" + "github.com/labring/sealos/pkg/runtime" + "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/json" - "github.com/labring/sealos/pkg/token" "github.com/labring/sealos/pkg/utils/logger" ) @@ -29,7 +30,7 @@ func newTokenCmd() *cobra.Command { Use: "token", Short: "token generator", Run: func(cmd *cobra.Command, args []string) { - t, err := token.Default() + t, err := runtime.Default() if err != nil { logger.Error("exec token error: " + err.Error()) os.Exit(1) diff --git a/pkg/ipvs/lvscare.go b/pkg/ipvs/lvscare.go index 72d6f2ee1..23082ecdf 100644 --- a/pkg/ipvs/lvscare.go +++ b/pkg/ipvs/lvscare.go @@ -17,7 +17,8 @@ package ipvs import ( "fmt" - "github.com/labring/sealos/pkg/hosts" + "github.com/labring/sealos/pkg/utils/hosts" + "github.com/labring/sealos/pkg/utils/constants" "github.com/pkg/errors" diff --git a/pkg/token/bootstraptoken/v1/utils.go b/pkg/runtime/apis/bootstraptoken/v1/utils.go similarity index 100% rename from pkg/token/bootstraptoken/v1/utils.go rename to pkg/runtime/apis/bootstraptoken/v1/utils.go diff --git a/pkg/token/bootstraptoken/v1/utils_test.go b/pkg/runtime/apis/bootstraptoken/v1/utils_test.go similarity index 100% rename from pkg/token/bootstraptoken/v1/utils_test.go rename to pkg/runtime/apis/bootstraptoken/v1/utils_test.go diff --git a/pkg/runtime/kubeadm.go b/pkg/runtime/kubeadm.go index 7f190b956..56092656b 100644 --- a/pkg/runtime/kubeadm.go +++ b/pkg/runtime/kubeadm.go @@ -27,7 +27,6 @@ import ( "github.com/labring/sealos/pkg/runtime/apis/kubeadm" "github.com/labring/sealos/pkg/runtime/apis/kubeadm/v1beta2" "github.com/labring/sealos/pkg/runtime/apis/kubeadm/v1beta3" - "github.com/labring/sealos/pkg/token" "github.com/labring/sealos/pkg/utils/constants" fileutil "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/iputils" @@ -183,7 +182,7 @@ func (k *KubeadmRuntime) writeTokenFile() error { if err = fileutil.WriteFile(tokenFile, []byte(data)); err != nil { return err } - var t token.Token + var t Token err = json.Unmarshal([]byte(data), &t) if err != nil { return err @@ -206,7 +205,7 @@ func (k *KubeadmRuntime) setKubernetesToken() error { if err != nil { return err } - var t token.Token + var t Token err = json.Unmarshal(data, &t) if err != nil { return err diff --git a/pkg/runtime/kubeadm_cmd.go b/pkg/runtime/kubeadm_cmd.go new file mode 100644 index 000000000..b7ac523e3 --- /dev/null +++ b/pkg/runtime/kubeadm_cmd.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "fmt" + "path" + "strconv" + "strings" + + "github.com/labring/sealos/pkg/utils/constants" + "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/versionutil" +) + +type CommandType string + +const InitMaster CommandType = "initMaster" +const JoinMaster CommandType = "joinMaster" +const JoinNode CommandType = "joinNode" + +func vlogToStr(vlog int) string { + str := strconv.Itoa(vlog) + return " -v " + str +} + +func (k *KubeadmRuntime) Command(version string, name CommandType) (cmd string) { + const ( + InitMaster115Lower = `kubeadm init --config=%s --experimental-upload-certs` + JoinMaster115Lower = "kubeadm join %s:6443 --token %s %s --experimental-control-plane --certificate-key %s" + JoinNode115Lower = "kubeadm join %s:6443 --token %s %s" + + InitMaser115Upper = `kubeadm init --config=%s --skip-certificate-key-print --skip-token-print` // --upload-certs --skip-certificate-key-print --skip-token-print + JoinMaster115Upper = "kubeadm join --config=%s" + JoinNode115Upper = "kubeadm join --config=%s" + ) + + initConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultInitKubeadmFileName) + joinMasterConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinMasterKubeadmFileName) + joinNodeConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinNodeKubeadmFileName) + + var discoveryTokens []string + for _, data := range k.getTokenCaCertHash() { + discoveryTokens = append(discoveryTokens, "--discovery-token-ca-cert-hash "+data) + } + + cmds := map[CommandType]string{ + InitMaster: fmt.Sprintf(InitMaster115Lower, initConfigPath), + JoinMaster: fmt.Sprintf(JoinMaster115Lower, k.getMaster0IP(), k.getJoinToken(), strings.Join(discoveryTokens, " "), k.getJoinCertificateKey()), + JoinNode: fmt.Sprintf(JoinNode115Lower, k.getVip(), k.getJoinToken(), strings.Join(discoveryTokens, " ")), + } + //other version >= 1.15.x + if versionutil.Compare(version, V1150) { + cmds[InitMaster] = fmt.Sprintf(InitMaser115Upper, initConfigPath) + cmds[JoinMaster] = fmt.Sprintf(JoinMaster115Upper, joinMasterConfigPath) + cmds[JoinNode] = fmt.Sprintf(JoinNode115Upper, joinNodeConfigPath) + } + + v, ok := cmds[name] + if !ok { + logger.Error("get kubeadm command failed %v", cmds) + return "" + } + + if name == InitMaster || name == JoinMaster { + return fmt.Sprintf("%s%s%s", v, vlogToStr(k.vlog), " --ignore-preflight-errors=SystemVerification") + } + + return fmt.Sprintf("%s%s", v, vlogToStr(k.vlog)) +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index dc4d12ecf..4caf23841 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -18,8 +18,6 @@ import ( "fmt" "sync" - "github.com/labring/sealos/pkg/token" - v2 "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/logger" ) @@ -27,7 +25,7 @@ import ( type KubeadmRuntime struct { *sync.Mutex Cluster *v2.Cluster - Token *token.Token + Token *Token *KubeadmConfig *Config } diff --git a/pkg/runtime/token.go b/pkg/runtime/token.go index b7ac523e3..530f2ae10 100644 --- a/pkg/runtime/token.go +++ b/pkg/runtime/token.go @@ -17,68 +17,192 @@ limitations under the License. package runtime import ( + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" "fmt" - "path" - "strconv" "strings" + "time" - "github.com/labring/sealos/pkg/utils/constants" - "github.com/labring/sealos/pkg/utils/logger" - "github.com/labring/sealos/pkg/utils/versionutil" + v1 "github.com/labring/sealos/pkg/runtime/apis/bootstraptoken/v1" + "github.com/pkg/errors" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/cert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/labring/sealos/pkg/utils/exec" + "github.com/labring/sealos/pkg/utils/file" + "github.com/labring/sealos/pkg/utils/yaml" ) -type CommandType string - -const InitMaster CommandType = "initMaster" -const JoinMaster CommandType = "joinMaster" -const JoinNode CommandType = "joinNode" - -func vlogToStr(vlog int) string { - str := strconv.Itoa(vlog) - return " -v " + str +type Token struct { + JoinToken string `json:"joinToken,omitempty"` + DiscoveryTokenCaCertHash []string `json:"discoveryTokenCaCertHash,omitempty"` + CertificateKey string `json:"certificateKey,omitempty"` + Expires *metav1.Time `json:"expires,omitempty"` } -func (k *KubeadmRuntime) Command(version string, name CommandType) (cmd string) { - const ( - InitMaster115Lower = `kubeadm init --config=%s --experimental-upload-certs` - JoinMaster115Lower = "kubeadm join %s:6443 --token %s %s --experimental-control-plane --certificate-key %s" - JoinNode115Lower = "kubeadm join %s:6443 --token %s %s" +const defaultAdminConf = "/etc/kubernetes/admin.conf" - InitMaser115Upper = `kubeadm init --config=%s --skip-certificate-key-print --skip-token-print` // --upload-certs --skip-certificate-key-print --skip-token-print - JoinMaster115Upper = "kubeadm join --config=%s" - JoinNode115Upper = "kubeadm join --config=%s" - ) - - initConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultInitKubeadmFileName) - joinMasterConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinMasterKubeadmFileName) - joinNodeConfigPath := path.Join(k.getContentData().EtcPath(), constants.DefaultJoinNodeKubeadmFileName) - - var discoveryTokens []string - for _, data := range k.getTokenCaCertHash() { - discoveryTokens = append(discoveryTokens, "--discovery-token-ca-cert-hash "+data) +func Default() (*Token, error) { + token := &Token{} + if _, ok := exec.CheckCmdIsExist("kubeadm"); ok && file.IsExist(defaultAdminConf) { + key, _ := CreateCertificateKey() + token.CertificateKey = key + const uploadCertTemplate = "kubeadm init phase upload-certs --upload-certs --certificate-key %s" + _, _ = exec.RunBashCmd(fmt.Sprintf(uploadCertTemplate, key)) + tokens := ListToken() + const tokenTemplate = "kubeadm token create --print-join-command --certificate-key %s" + _, _ = exec.RunBashCmd(fmt.Sprintf(tokenTemplate, key)) + afterTokens := ListToken() + diff := afterTokens.ToStrings().Difference(tokens.ToStrings()) + if diff.Len() == 1 { + token.JoinToken = diff.List()[0] + hashs, err := discoveryTokenCaCertHash(defaultAdminConf) + if err != nil { + return nil, err + } + token.DiscoveryTokenCaCertHash = hashs + for _, t := range afterTokens { + if t.Token.String() == token.JoinToken { + token.Expires = t.Expires + break + } + } + return token, nil + } + return nil, fmt.Errorf("token list found more than one") } - cmds := map[CommandType]string{ - InitMaster: fmt.Sprintf(InitMaster115Lower, initConfigPath), - JoinMaster: fmt.Sprintf(JoinMaster115Lower, k.getMaster0IP(), k.getJoinToken(), strings.Join(discoveryTokens, " "), k.getJoinCertificateKey()), - JoinNode: fmt.Sprintf(JoinNode115Lower, k.getVip(), k.getJoinToken(), strings.Join(discoveryTokens, " ")), + return nil, fmt.Errorf("kubeadm command not found or /etc/kubernetes/admin.conf not exist") +} + +func ListToken() BootstrapTokens { + const tokenListShell = "kubeadm token list -o yaml" + data, _ := exec.RunBashCmd(tokenListShell) + return processTokenList(data) +} + +func processTokenList(data string) BootstrapTokens { + var slice []v1.BootstrapToken + if data != "" { + jsons := yaml.ToJSON([]byte(data)) + for _, j := range jsons { + var to v1.BootstrapToken + _ = json.Unmarshal([]byte(j), &to) + slice = append(slice, to) + } } - //other version >= 1.15.x - if versionutil.Compare(version, V1150) { - cmds[InitMaster] = fmt.Sprintf(InitMaser115Upper, initConfigPath) - cmds[JoinMaster] = fmt.Sprintf(JoinMaster115Upper, joinMasterConfigPath) - cmds[JoinNode] = fmt.Sprintf(JoinNode115Upper, joinNodeConfigPath) + var result []v1.BootstrapToken + for _, token := range slice { + if token.Expires != nil { + t := time.Now().Unix() + ex := token.Expires.Time.Unix() + if ex < t { + continue + } + if len(token.Usages) == 0 || len(token.Groups) == 0 { + continue + } + } + result = append(result, token) } + return result +} + +type BootstrapTokens []v1.BootstrapToken - v, ok := cmds[name] - if !ok { - logger.Error("get kubeadm command failed %v", cmds) - return "" +func (c BootstrapTokens) ToStrings() sets.String { + s := sets.NewString() + for _, token := range c { + s.Insert(token.Token.String()) } + return s +} - if name == InitMaster || name == JoinMaster { - return fmt.Sprintf("%s%s%s", v, vlogToStr(k.vlog), " --ignore-preflight-errors=SystemVerification") +func discoveryTokenCaCertHash(adminPath string) ([]string, error) { + tlsBootstrapCfg, err := clientcmd.LoadFromFile(adminPath) + if err != nil { + return nil, err } + // load the default cluster config + clusterConfig := GetClusterFromKubeConfig(tlsBootstrapCfg) + if clusterConfig == nil { + return nil, errors.New("failed to get default cluster config") + } + + // load CA certificates from the kubeconfig (either from PEM data or by file path) + var caCerts []*x509.Certificate + if clusterConfig.CertificateAuthorityData != nil { + caCerts, err = cert.ParseCertsPEM(clusterConfig.CertificateAuthorityData) + if err != nil { + return nil, errors.Wrap(err, "failed to parse CA certificate from kubeconfig") + } + } else if clusterConfig.CertificateAuthority != "" { + caCerts, err = cert.CertsFromFile(clusterConfig.CertificateAuthority) + if err != nil { + return nil, errors.Wrap(err, "failed to load CA certificate referenced by kubeconfig") + } + } else { + return nil, errors.New("no CA certificates found in kubeconfig") + } + + // hash all the CA certs and include their public key pins as trusted values + publicKeyPins := make([]string, 0, len(caCerts)) + for _, caCert := range caCerts { + publicKeyPins = append(publicKeyPins, Hash(caCert)) + } + return publicKeyPins, nil +} + +// CreateRandBytes returns a cryptographically secure slice of random bytes with a given size +func CreateRandBytes(size uint32) ([]byte, error) { + bytes := make([]byte, size) + if _, err := rand.Read(bytes); err != nil { + return nil, err + } + return bytes, nil +} + +const ( + CertificateKeySize = 32 +) + +//CreateCertificateKey returns a cryptographically secure random key +func CreateCertificateKey() (string, error) { + randBytes, err := CreateRandBytes(CertificateKeySize) + if err != nil { + return "", err + } + return hex.EncodeToString(randBytes), nil +} + +// GetClusterFromKubeConfig returns the default Infra of the specified KubeConfig +func GetClusterFromKubeConfig(config *clientcmdapi.Config) *clientcmdapi.Cluster { + // If there is an unnamed cluster object, use it + if config.Clusters[""] != nil { + return config.Clusters[""] + } + if config.Contexts[config.CurrentContext] != nil { + return config.Clusters[config.Contexts[config.CurrentContext].Cluster] + } + return nil +} + +const ( + // formatSHA256 is the prefix for pins that are full-length SHA-256 hashes encoded in base 16 (hex) + formatSHA256 = "sha256" +) - return fmt.Sprintf("%s%s", v, vlogToStr(k.vlog)) +// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI) +// object in an x509 certificate (in DER encoding). It returns the full hash as a +// hex encoded string (suitable for passing to Set.Allow). +func Hash(certificate *x509.Certificate) string { + spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:])) } diff --git a/pkg/token/bootstraptoken/doc.go b/pkg/token/bootstraptoken/doc.go deleted file mode 100644 index b7be16ba4..000000000 --- a/pkg/token/bootstraptoken/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +groupName=bootstraptoken.kubeadm.k8s.io - -// Package bootstraptoken contains an API and utilities wrapping the -// "bootstrap.kubernetes.io/token" Secret type to ease its usage in kubeadm. -package bootstraptoken // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken" diff --git a/pkg/token/bootstraptoken/v1/types.go b/pkg/token/bootstraptoken/v1/types.go deleted file mode 100644 index 8ebaf77f3..000000000 --- a/pkg/token/bootstraptoken/v1/types.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster -// +k8s:deepcopy-gen=true -type BootstrapToken struct { - // Token is used for establishing bidirectional trust between nodes and control-planes. - // Used for joining nodes in the cluster. - Token *BootstrapTokenString `json:"token" datapolicy:"token"` - // Description sets a human-friendly message why this token exists and what it's used - // for, so other administrators can know its purpose. - // +optional - Description string `json:"description,omitempty"` - // TTL defines the time to live for this token. Defaults to 24h. - // Expires and TTL are mutually exclusive. - // +optional - TTL *metav1.Duration `json:"ttl,omitempty"` - // Expires specifies the timestamp when this token expires. Defaults to being set - // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. - // +optional - Expires *metav1.Time `json:"expires,omitempty"` - // Usages describes the ways in which this token can be used. Can by default be used - // for establishing bidirectional trust, but that can be changed here. - // +optional - Usages []string `json:"usages,omitempty"` - // Groups specifies the extra groups that this token will authenticate as when/if - // used for authentication - // +optional - Groups []string `json:"groups,omitempty"` -} - -// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used -// for both validation of the practically of the API server from a joining node's point -// of view and as an authentication method for the node in the bootstrap phase of -// "kubeadm join". This token is and should be short-lived -type BootstrapTokenString struct { - ID string `json:"-"` - Secret string `json:"-" datapolicy:"token"` -} diff --git a/pkg/token/bootstraptoken/v1/zz_generated.deepcopy.go b/pkg/token/bootstraptoken/v1/zz_generated.deepcopy.go deleted file mode 100644 index 1dcf76cb9..000000000 --- a/pkg/token/bootstraptoken/v1/zz_generated.deepcopy.go +++ /dev/null @@ -1,66 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package v1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(BootstrapTokenString) - **out = **in - } - if in.TTL != nil { - in, out := &in.TTL, &out.TTL - *out = new(metav1.Duration) - **out = **in - } - if in.Expires != nil { - in, out := &in.Expires, &out.Expires - *out = (*in).DeepCopy() - } - if in.Usages != nil { - in, out := &in.Usages, &out.Usages - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Groups != nil { - in, out := &in.Groups, &out.Groups - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/token/pins.go b/pkg/token/pins.go deleted file mode 100644 index 2cf420083..000000000 --- a/pkg/token/pins.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2022 cuisongliu@qq.com. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package token - -import ( - "crypto/rand" - "crypto/sha256" - "crypto/x509" - "encoding/hex" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/client-go/util/cert" -) - -func discoveryTokenCaCertHash(adminPath string) ([]string, error) { - tlsBootstrapCfg, err := clientcmd.LoadFromFile(adminPath) - if err != nil { - return nil, err - } - // load the default cluster config - clusterConfig := GetClusterFromKubeConfig(tlsBootstrapCfg) - if clusterConfig == nil { - return nil, errors.New("failed to get default cluster config") - } - - // load CA certificates from the kubeconfig (either from PEM data or by file path) - var caCerts []*x509.Certificate - if clusterConfig.CertificateAuthorityData != nil { - caCerts, err = cert.ParseCertsPEM(clusterConfig.CertificateAuthorityData) - if err != nil { - return nil, errors.Wrap(err, "failed to parse CA certificate from kubeconfig") - } - } else if clusterConfig.CertificateAuthority != "" { - caCerts, err = cert.CertsFromFile(clusterConfig.CertificateAuthority) - if err != nil { - return nil, errors.Wrap(err, "failed to load CA certificate referenced by kubeconfig") - } - } else { - return nil, errors.New("no CA certificates found in kubeconfig") - } - - // hash all the CA certs and include their public key pins as trusted values - publicKeyPins := make([]string, 0, len(caCerts)) - for _, caCert := range caCerts { - publicKeyPins = append(publicKeyPins, Hash(caCert)) - } - return publicKeyPins, nil -} - -// CreateRandBytes returns a cryptographically secure slice of random bytes with a given size -func CreateRandBytes(size uint32) ([]byte, error) { - bytes := make([]byte, size) - if _, err := rand.Read(bytes); err != nil { - return nil, err - } - return bytes, nil -} - -const ( - CertificateKeySize = 32 -) - -//CreateCertificateKey returns a cryptographically secure random key -func CreateCertificateKey() (string, error) { - randBytes, err := CreateRandBytes(CertificateKeySize) - if err != nil { - return "", err - } - return hex.EncodeToString(randBytes), nil -} - -// GetClusterFromKubeConfig returns the default Infra of the specified KubeConfig -func GetClusterFromKubeConfig(config *clientcmdapi.Config) *clientcmdapi.Cluster { - // If there is an unnamed cluster object, use it - if config.Clusters[""] != nil { - return config.Clusters[""] - } - if config.Contexts[config.CurrentContext] != nil { - return config.Clusters[config.Contexts[config.CurrentContext].Cluster] - } - return nil -} - -const ( - // formatSHA256 is the prefix for pins that are full-length SHA-256 hashes encoded in base 16 (hex) - formatSHA256 = "sha256" -) - -// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI) -// object in an x509 certificate (in DER encoding). It returns the full hash as a -// hex encoded string (suitable for passing to Set.Allow). -func Hash(certificate *x509.Certificate) string { - spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) - return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:])) -} diff --git a/pkg/token/token.go b/pkg/token/token.go deleted file mode 100644 index d839c1c37..000000000 --- a/pkg/token/token.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2022 cuisongliu@qq.com. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package token - -import ( - "encoding/json" - "fmt" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - - v1 "github.com/labring/sealos/pkg/token/bootstraptoken/v1" - "github.com/labring/sealos/pkg/utils/exec" - "github.com/labring/sealos/pkg/utils/file" - "github.com/labring/sealos/pkg/utils/yaml" -) - -type Token struct { - JoinToken string `json:"joinToken,omitempty"` - DiscoveryTokenCaCertHash []string `json:"discoveryTokenCaCertHash,omitempty"` - CertificateKey string `json:"certificateKey,omitempty"` - Expires *metav1.Time `json:"expires,omitempty"` -} - -const defaultAdminConf = "/etc/kubernetes/admin.conf" - -func Default() (*Token, error) { - token := &Token{} - if _, ok := exec.CheckCmdIsExist("kubeadm"); ok && file.IsExist(defaultAdminConf) { - key, _ := CreateCertificateKey() - token.CertificateKey = key - const uploadCertTemplate = "kubeadm init phase upload-certs --upload-certs --certificate-key %s" - _, _ = exec.RunBashCmd(fmt.Sprintf(uploadCertTemplate, key)) - tokens := ListToken() - const tokenTemplate = "kubeadm token create --print-join-command --certificate-key %s" - _, _ = exec.RunBashCmd(fmt.Sprintf(tokenTemplate, key)) - afterTokens := ListToken() - diff := afterTokens.ToStrings().Difference(tokens.ToStrings()) - if diff.Len() == 1 { - token.JoinToken = diff.List()[0] - hashs, err := discoveryTokenCaCertHash(defaultAdminConf) - if err != nil { - return nil, err - } - token.DiscoveryTokenCaCertHash = hashs - for _, t := range afterTokens { - if t.Token.String() == token.JoinToken { - token.Expires = t.Expires - break - } - } - return token, nil - } - return nil, fmt.Errorf("token list found more than one") - } - - return nil, fmt.Errorf("kubeadm command not found or /etc/kubernetes/admin.conf not exist") -} - -func ListToken() BootstrapTokens { - const tokenListShell = "kubeadm token list -o yaml" - data, _ := exec.RunBashCmd(tokenListShell) - return processTokenList(data) -} - -func processTokenList(data string) BootstrapTokens { - var slice []v1.BootstrapToken - if data != "" { - jsons := yaml.ToJSON([]byte(data)) - for _, j := range jsons { - var to v1.BootstrapToken - _ = json.Unmarshal([]byte(j), &to) - slice = append(slice, to) - } - } - var result []v1.BootstrapToken - for _, token := range slice { - if token.Expires != nil { - t := time.Now().Unix() - ex := token.Expires.Time.Unix() - if ex < t { - continue - } - if len(token.Usages) == 0 || len(token.Groups) == 0 { - continue - } - } - result = append(result, token) - } - return result -} - -type BootstrapTokens []v1.BootstrapToken - -func (c BootstrapTokens) ToStrings() sets.String { - s := sets.NewString() - for _, token := range c { - s.Insert(token.Token.String()) - } - return s -} diff --git a/pkg/hosts/hosts.go b/pkg/utils/hosts/hosts.go similarity index 100% rename from pkg/hosts/hosts.go rename to pkg/utils/hosts/hosts.go