Add resiliency in accessing EKS cluster after creating access entry for it (#37958)

* Add resiliency in accessing EKS cluster after creating access entry for it

* Allow canceling the check through context.
This commit is contained in:
Anton Miniailo
2024-02-13 15:34:55 +00:00
committed by GitHub
parent bd47f0e02a
commit 86efc77e04
2 changed files with 30 additions and 14 deletions
+26 -10
View File
@@ -42,6 +42,7 @@ import (
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
helmCli "helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/release"
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/rest"
@@ -106,7 +107,7 @@ type EnrollEKSCLusterClient interface {
GetCallerIdentity(ctx context.Context, params *sts.GetCallerIdentityInput, optFns ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error)
// CheckAgentAlreadyInstalled checks if teleport-kube-agent Helm chart is already installed on the EKS cluster.
CheckAgentAlreadyInstalled(genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error)
CheckAgentAlreadyInstalled(context.Context, genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error)
// InstallKubeAgent installs teleport-kube-agent Helm chart to the EKS cluster.
InstallKubeAgent(context.Context, *eksTypes.Cluster, string, string, string, genericclioptions.RESTClientGetter, logrus.FieldLogger, EnrollEKSClustersRequest) error
@@ -127,13 +128,13 @@ func (d *defaultEnrollEKSClustersClient) GetCallerIdentity(ctx context.Context,
}
// CheckAgentAlreadyInstalled checks if teleport-kube-agent Helm chart is already installed on the EKS cluster.
func (d *defaultEnrollEKSClustersClient) CheckAgentAlreadyInstalled(clientGetter genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) {
func (d *defaultEnrollEKSClustersClient) CheckAgentAlreadyInstalled(ctx context.Context, clientGetter genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) {
actionConfig, err := getHelmActionConfig(clientGetter, log)
if err != nil {
return false, trace.Wrap(err)
}
return checkAgentAlreadyInstalled(actionConfig)
return checkAgentAlreadyInstalled(ctx, actionConfig)
}
func getToken(ctx context.Context, clock clockwork.Clock, tokenCreator TokenCreator) (string, string, error) {
@@ -328,7 +329,7 @@ func enrollEKSCluster(ctx context.Context, log logrus.FieldLogger, clock clockwo
return "", trace.Wrap(err, "unable to build kubernetes client for EKS cluster %q", clusterName)
}
if alreadyInstalled, err := clt.CheckAgentAlreadyInstalled(kubeClientGetter, log); err != nil {
if alreadyInstalled, err := clt.CheckAgentAlreadyInstalled(ctx, kubeClientGetter, log); err != nil {
return "", trace.Wrap(err, "could not check if teleport-kube-agent is already installed.")
} else if alreadyInstalled {
return "", trace.AlreadyExists("teleport-kube-agent is already installed on the cluster %q", clusterName)
@@ -466,15 +467,30 @@ func getHelmSettings() (*helmCli.EnvSettings, error) {
}
// checkAgentAlreadyInstalled checks through the Helm if teleport-kube-agent chart was already installed in the EKS cluster.
func checkAgentAlreadyInstalled(actionConfig *action.Configuration) (bool, error) {
listCmd := action.NewList(actionConfig)
releases, err := listCmd.Run()
func checkAgentAlreadyInstalled(ctx context.Context, actionConfig *action.Configuration) (bool, error) {
var releases []*release.Release
var err error
// We setup a little backoff loop because sometimes access entry auth needs a bit more time to propagate and take
// effect, so we could get errors when trying to access cluster right after giving us permissions to do so.
for attempt := 1; attempt <= 3; attempt++ {
listCmd := action.NewList(actionConfig)
releases, err = listCmd.Run()
if err != nil {
select {
case <-time.After(time.Duration(attempt) * time.Second):
case <-ctx.Done():
return false, trace.NewAggregate(err, ctx.Err())
}
} else {
break
}
}
if err != nil {
return false, trace.Wrap(err)
}
for _, release := range releases {
if release.Name == agentName {
for _, r := range releases {
if r.Name == agentName {
return true, nil
}
}
@@ -199,7 +199,7 @@ func TestEnrollEKSClusters(t *testing.T) {
clt := baseClient(t, clusters)
mockClt, ok := clt.(*mockEnrollEKSClusterClient)
require.True(t, ok)
mockClt.checkAgentAlreadyInstalled = func(getter genericclioptions.RESTClientGetter, logger logrus.FieldLogger) (bool, error) {
mockClt.checkAgentAlreadyInstalled = func(ctx context.Context, getter genericclioptions.RESTClientGetter, logger logrus.FieldLogger) (bool, error) {
return true, nil
}
return mockClt
@@ -418,7 +418,7 @@ type mockEnrollEKSClusterClient struct {
deleteAccessEntry func(context.Context, *eks.DeleteAccessEntryInput, ...func(*eks.Options)) (*eks.DeleteAccessEntryOutput, error)
describeCluster func(context.Context, *eks.DescribeClusterInput, ...func(*eks.Options)) (*eks.DescribeClusterOutput, error)
getCallerIdentity func(context.Context, *sts.GetCallerIdentityInput, ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error)
checkAgentAlreadyInstalled func(genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error)
checkAgentAlreadyInstalled func(context.Context, genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error)
installKubeAgent func(context.Context, *eksTypes.Cluster, string, string, string, genericclioptions.RESTClientGetter, logrus.FieldLogger, EnrollEKSClustersRequest) error
createToken func(ctx context.Context, token types.ProvisionToken) error
}
@@ -465,9 +465,9 @@ func (m *mockEnrollEKSClusterClient) GetCallerIdentity(ctx context.Context, para
return &sts.GetCallerIdentityOutput{}, nil
}
func (m *mockEnrollEKSClusterClient) CheckAgentAlreadyInstalled(kubeconfig genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) {
func (m *mockEnrollEKSClusterClient) CheckAgentAlreadyInstalled(ctx context.Context, kubeconfig genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) {
if m.checkAgentAlreadyInstalled != nil {
return m.checkAgentAlreadyInstalled(kubeconfig, log)
return m.checkAgentAlreadyInstalled(ctx, kubeconfig, log)
}
return false, nil
}