Files
teleport/lib/client/client_store.go
T
Tiago SilvaandBrian Joerger f8ab3aa037 Improve tsh kube credentials read operations (#22388)
While loading a key from storage, Teleport client transverses the
storage directory to read the secrets and reads each file one by one.
This has huge impacts on `tsh kube credentials` because when calling
`kubectl` in parallel, the disk operations become overloaded and the
operation time increases exponentially.

This PR removes the transverse when reading credentials for Kubernetes
clusters and adds a quick read operation that reads the following files:
- `$TSH_HOME/current_profile`
- `$TSH_HOME/$profile.yaml`
- `$TSH_HOME/keys/$PROXY/$USER-kube/$TELEPORT_CLUSTER/$KUBE_CLUSTER-x509.pem`
- `$TSH_HOME/keys/$PROXY/$USER`

| - | P=1 | P=10 | P=40 | P=80 | P=120 |
| --- | --- | --- | --- | --- | --- |
| old direct | 9.90s user 3.14s system 64% cpu 20.284 total | 11.08s user 3.60s system 496% cpu 2.955 total | 12.46s user 4.36s system 820% cpu 2.050 total | 12.81s user 4.76s system 811% cpu 2.164 total | 13.02s user 5.07s system 861% cpu 2.100 total|
| old teleport | 18.45s user 7.80s system 51% cpu 51.085 total | 21.58s user 10.03s system 434% cpu 7.270 total | 22.33s user 10.52s system 178% cpu 18.402 total | 22.04s user 10.83s system 96% cpu 33.977 total | 22.57s user 11.45s system 55% cpu 1:01.34 total|
| new direct | 9.51s user 2.90s system 146% cpu 8.460 total | 13.75s user 4.95s system 718% cpu 2.603 total | 12.21s user 4.43s system 832% cpu 1.999 total | 11.72s user 4.50s system 839% cpu 1.933 total | 12.06s user 4.62s system 830% cpu 2.009 total |
| new teleport | 16.10s user 5.97s system 42% cpu 52.104 total | 21.62s user 9.01s system 381% cpu 8.023 total | 23.22s user 10.11s system 540% cpu 6.162 total | 21.64s user 9.81s system 603% cpu 5.208 total | 20.70s user 9.13s system 602% cpu 4.948 total |

---

| - | P=1 | P=10 | P=40 | P=80 | P=120 |
| --- | --- | --- | --- | --- | --- |
| old no tsh kube credential | 8.61s user 2.99s system 31% cpu 36.336 total | 9.57s user 3.39s system 283% cpu 4.565 total | 10.22s user 4.03s system 553% cpu 2.574 total | 9.32s user 3.82s system 559% cpu 2.351 total | 11.49s user 4.85s system 624% cpu 2.617 total
| new no tsh kube credential | 8.27s user 2.94s system 24% cpu 45.041 total | 10.85s user 4.31s system 195% cpu 7.767 total | 11.32s user 4.64s system 469% cpu 3.401 total | 10.83s user 4.51s system 553% cpu 2.773 total | 10.90s user 4.84s system 537% cpu 2.930 total |

---

| instances | old total | old avg | **new total** | **total avg** |
| --- | --- | --- | --- | --- |
| 1 | 0.145 | 0.145 | 0.074 | 0.074 |
| 10 | 0.430 | 0.043 | 0.146 | 0.0146 |
| 20 | 0.740 | 0.037 | 0.284 | 0.0142 |
| 30 | 1.108 | 0.037 | 0.417 | 0.0139 |
| 40 | 1.483 | 0.037 | 0.580 | 0.0145 |
| 60 | 13.604 | 0.228 | 0.825 | 0.0137 |
| 80 | 30.236 | 0.378 | 1.143 | 0.0142 |
| 106 | 46.681 | 0.440 | 1.413 | 0.0133 |

Fixes #19313

Co-authored-by: Brian Joerger <bjoerger@goteleport.com>
2023-03-01 19:54:36 +00:00

267 lines
8.3 KiB
Go

/*
Copyright 2022 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"net/url"
"os"
"time"
"github.com/gravitational/trace"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/profile"
"github.com/gravitational/teleport/api/utils/keys"
"github.com/gravitational/teleport/lib/auth"
"github.com/gravitational/teleport/lib/utils"
)
// Store is a storage interface for client data. Store is made up of three
// partial data stores; KeyStore, TrustedCertsStore, and ProfileStore.
//
// A Store can be made up of partial data stores with different backends. For example,
// when using `tsh --add-keys-to-agent=only`, Store will be made up of an in-memory
// key store and an FS (~/.tsh) profile and trusted certs store.
type Store struct {
log *logrus.Entry
KeyStore
TrustedCertsStore
ProfileStore
}
// NewMemClientStore initializes an FS backed client store with the given base dir.
func NewFSClientStore(dirPath string) *Store {
dirPath = profile.FullProfilePath(dirPath)
return &Store{
log: logrus.WithField(trace.Component, teleport.ComponentKeyStore),
KeyStore: NewFSKeyStore(dirPath),
TrustedCertsStore: NewFSTrustedCertsStore(dirPath),
ProfileStore: NewFSProfileStore(dirPath),
}
}
// NewMemClientStore initializes a new in-memory client store.
func NewMemClientStore() *Store {
return &Store{
log: logrus.WithField(trace.Component, teleport.ComponentKeyStore),
KeyStore: NewMemKeyStore(),
TrustedCertsStore: NewMemTrustedCertsStore(),
ProfileStore: NewMemProfileStore(),
}
}
// AddKey adds the given key to the key store. The key's trusted certificates are
// added to the trusted certs store.
func (s *Store) AddKey(key *Key) error {
if err := s.KeyStore.AddKey(key); err != nil {
return trace.Wrap(err)
}
if err := s.TrustedCertsStore.SaveTrustedCerts(key.ProxyHost, key.TrustedCerts); err != nil {
return trace.Wrap(err)
}
return nil
}
// GetKey gets the requested key with trusted the requested certificates. The key's
// trusted certs will be retrieved from the trusted certs store.
func (s *Store) GetKey(idx KeyIndex, opts ...CertOption) (*Key, error) {
key, err := s.KeyStore.GetKey(idx, opts...)
if err != nil {
return nil, trace.Wrap(err)
}
tlsCertExpiration, err := key.TeleportTLSCertValidBefore()
if err != nil {
return nil, trace.Wrap(err)
}
s.log.Debugf("Teleport TLS certificate valid until %q.", tlsCertExpiration)
// Validate the SSH certificate.
if key.Cert != nil {
if err := key.CheckCert(); err != nil {
if !utils.IsCertExpiredError(err) {
return nil, trace.Wrap(err)
}
}
}
trustedCerts, err := s.TrustedCertsStore.GetTrustedCerts(idx.ProxyHost)
if err != nil {
return nil, trace.Wrap(err)
}
key.TrustedCerts = trustedCerts
return key, nil
}
// AddTrustedHostKeys is a helper function to add ssh host keys directly, rather than through SaveTrustedCerts.
func (s *Store) AddTrustedHostKeys(proxyHost string, clusterName string, hostKeys ...ssh.PublicKey) error {
var authorizedKeys [][]byte
for _, hostKey := range hostKeys {
authorizedKeys = append(authorizedKeys, ssh.MarshalAuthorizedKey(hostKey))
}
err := s.SaveTrustedCerts(proxyHost, []auth.TrustedCerts{
{
ClusterName: clusterName,
AuthorizedKeys: authorizedKeys,
},
})
return trace.Wrap(err)
}
// ReadProfileStatus returns the profile status for the given profile name.
// If no profile name is provided, return the current profile.
func (s *Store) ReadProfileStatus(profileName string) (*ProfileStatus, error) {
var err error
if profileName == "" {
profileName, err = s.CurrentProfile()
if err != nil {
return nil, trace.Wrap(err)
}
} else {
// remove ports from proxy host, because profile name is stored by host name
profileName, err = utils.Host(profileName)
if err != nil {
return nil, trace.Wrap(err)
}
}
profile, err := s.GetProfile(profileName)
if err != nil {
return nil, trace.Wrap(err)
}
idx := KeyIndex{
ProxyHost: profileName,
ClusterName: profile.SiteName,
Username: profile.Username,
}
key, err := s.GetKey(idx, WithAllCerts...)
if err != nil {
if trace.IsNotFound(err) || trace.IsConnectionProblem(err) {
// If we can't find a key to match the profile, or can't connect to
// the key (hardware key), return a partial status. This is used for
// some superficial functions `tsh logout` and `tsh status`.
return &ProfileStatus{
Name: profileName,
Dir: profile.Dir,
ProxyURL: url.URL{
Scheme: "https",
Host: profile.WebProxyAddr,
},
Username: profile.Username,
Cluster: profile.SiteName,
KubeEnabled: profile.KubeProxyAddr != "",
// Set ValidUntil to now to show that the keys are not available.
ValidUntil: time.Now(),
}, nil
}
return nil, trace.Wrap(err)
}
_, onDisk := s.KeyStore.(*FSKeyStore)
return profileStatusFromKey(key, profileOptions{
ProfileName: profileName,
ProfileDir: profile.Dir,
WebProxyAddr: profile.WebProxyAddr,
Username: profile.Username,
SiteName: profile.SiteName,
KubeProxyAddr: profile.KubeProxyAddr,
IsVirtual: !onDisk,
})
}
// FullProfileStatus returns the name of the current profile with a
// a list of all profile statuses.
func (s *Store) FullProfileStatus() (*ProfileStatus, []*ProfileStatus, error) {
currentProfileName, err := s.CurrentProfile()
if err != nil {
return nil, nil, trace.Wrap(err)
}
currentProfile, err := s.ReadProfileStatus(currentProfileName)
if err != nil {
return nil, nil, trace.Wrap(err)
}
profileNames, err := s.ListProfiles()
if err != nil {
return nil, nil, trace.Wrap(err)
}
var profiles []*ProfileStatus
for _, profileName := range profileNames {
if profileName == currentProfileName {
// already loaded this one
continue
}
status, err := s.ReadProfileStatus(profileName)
if err != nil {
return nil, nil, trace.Wrap(err)
}
profiles = append(profiles, status)
}
return currentProfile, profiles, nil
}
// LoadKeysToKubeFromStore loads the keys for a given teleport cluster and kube cluster from the store.
// It returns the certificate and private key to be used for the kube cluster.
// If the keys are not found, it returns an error.
// This function is used to speed up the credentials loading process since Teleport
// Store transverses the entire store to find the keys. This operation takes a long time
// when the store has a lot of keys and when we call the function multiple times in
// parallel.
// Although this function speeds up the process since it removes all transversals,
// it still has to read 4 different files
// - $TSH_HOME/current_profile
// - $TSH_HOME/$profile.yaml
// - $TSH_HOME/keys/$PROXY/$USER-kube/$TELEPORT_CLUSTER/$KUBE_CLUSTER-x509.pem
// - $TSH_HOME/keys/$PROXY/$USER
func LoadKeysToKubeFromStore(dirPath string, teleportCluster, kubeCluster string) ([]byte, []byte, error) {
dirPath = profile.FullProfilePath(dirPath)
profileStore := NewFSProfileStore(dirPath)
currentProfile, err := profileStore.CurrentProfile()
if err != nil {
return nil, nil, trace.Wrap(err)
}
profile, err := profileStore.GetProfile(currentProfile)
if err != nil {
return nil, nil, trace.Wrap(err)
}
fsKeyStore := NewFSKeyStore(dirPath)
certPath := fsKeyStore.kubeCertPath(KeyIndex{ProxyHost: profile.SiteName, ClusterName: teleportCluster, Username: profile.Username}, kubeCluster)
kubeCert, err := os.ReadFile(certPath)
if err != nil {
return nil, nil, trace.Wrap(err)
}
privKeyPath := fsKeyStore.userKeyPath(KeyIndex{ProxyHost: profile.SiteName, Username: profile.Username})
privKey, err := os.ReadFile(privKeyPath)
if err != nil {
return nil, nil, trace.Wrap(err)
}
if ok := keys.IsRSAPrivateKey(privKey); !ok {
return nil, nil, trace.BadParameter("unsupported private key type")
}
return kubeCert, privKey, nil
}