From 86efc77e042cc2a00a2da87559bf164a1e6910b2 Mon Sep 17 00:00:00 2001 From: Anton Miniailo Date: Tue, 13 Feb 2024 10:34:55 -0500 Subject: [PATCH] Add resiliency in accessing EKS cluster after creating access entry for it (#37958) * Add resiliency in accessing EKS cluster after creating access entry for it * Allow canceling the check through context. --- .../awsoidc/eks_enroll_clusters.go | 36 +++++++++++++------ .../awsoidc/eks_enroll_clusters_test.go | 8 ++--- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/integrations/awsoidc/eks_enroll_clusters.go b/lib/integrations/awsoidc/eks_enroll_clusters.go index c71975378eb..07d7644b50b 100644 --- a/lib/integrations/awsoidc/eks_enroll_clusters.go +++ b/lib/integrations/awsoidc/eks_enroll_clusters.go @@ -42,6 +42,7 @@ import ( "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" helmCli "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/release" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" @@ -106,7 +107,7 @@ type EnrollEKSCLusterClient interface { GetCallerIdentity(ctx context.Context, params *sts.GetCallerIdentityInput, optFns ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error) // CheckAgentAlreadyInstalled checks if teleport-kube-agent Helm chart is already installed on the EKS cluster. - CheckAgentAlreadyInstalled(genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error) + CheckAgentAlreadyInstalled(context.Context, genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error) // InstallKubeAgent installs teleport-kube-agent Helm chart to the EKS cluster. InstallKubeAgent(context.Context, *eksTypes.Cluster, string, string, string, genericclioptions.RESTClientGetter, logrus.FieldLogger, EnrollEKSClustersRequest) error @@ -127,13 +128,13 @@ func (d *defaultEnrollEKSClustersClient) GetCallerIdentity(ctx context.Context, } // CheckAgentAlreadyInstalled checks if teleport-kube-agent Helm chart is already installed on the EKS cluster. -func (d *defaultEnrollEKSClustersClient) CheckAgentAlreadyInstalled(clientGetter genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) { +func (d *defaultEnrollEKSClustersClient) CheckAgentAlreadyInstalled(ctx context.Context, clientGetter genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) { actionConfig, err := getHelmActionConfig(clientGetter, log) if err != nil { return false, trace.Wrap(err) } - return checkAgentAlreadyInstalled(actionConfig) + return checkAgentAlreadyInstalled(ctx, actionConfig) } func getToken(ctx context.Context, clock clockwork.Clock, tokenCreator TokenCreator) (string, string, error) { @@ -328,7 +329,7 @@ func enrollEKSCluster(ctx context.Context, log logrus.FieldLogger, clock clockwo return "", trace.Wrap(err, "unable to build kubernetes client for EKS cluster %q", clusterName) } - if alreadyInstalled, err := clt.CheckAgentAlreadyInstalled(kubeClientGetter, log); err != nil { + if alreadyInstalled, err := clt.CheckAgentAlreadyInstalled(ctx, kubeClientGetter, log); err != nil { return "", trace.Wrap(err, "could not check if teleport-kube-agent is already installed.") } else if alreadyInstalled { return "", trace.AlreadyExists("teleport-kube-agent is already installed on the cluster %q", clusterName) @@ -466,15 +467,30 @@ func getHelmSettings() (*helmCli.EnvSettings, error) { } // checkAgentAlreadyInstalled checks through the Helm if teleport-kube-agent chart was already installed in the EKS cluster. -func checkAgentAlreadyInstalled(actionConfig *action.Configuration) (bool, error) { - listCmd := action.NewList(actionConfig) - - releases, err := listCmd.Run() +func checkAgentAlreadyInstalled(ctx context.Context, actionConfig *action.Configuration) (bool, error) { + var releases []*release.Release + var err error + // We setup a little backoff loop because sometimes access entry auth needs a bit more time to propagate and take + // effect, so we could get errors when trying to access cluster right after giving us permissions to do so. + for attempt := 1; attempt <= 3; attempt++ { + listCmd := action.NewList(actionConfig) + releases, err = listCmd.Run() + if err != nil { + select { + case <-time.After(time.Duration(attempt) * time.Second): + case <-ctx.Done(): + return false, trace.NewAggregate(err, ctx.Err()) + } + } else { + break + } + } if err != nil { return false, trace.Wrap(err) } - for _, release := range releases { - if release.Name == agentName { + + for _, r := range releases { + if r.Name == agentName { return true, nil } } diff --git a/lib/integrations/awsoidc/eks_enroll_clusters_test.go b/lib/integrations/awsoidc/eks_enroll_clusters_test.go index 584601bfc0d..c8212819e2d 100644 --- a/lib/integrations/awsoidc/eks_enroll_clusters_test.go +++ b/lib/integrations/awsoidc/eks_enroll_clusters_test.go @@ -199,7 +199,7 @@ func TestEnrollEKSClusters(t *testing.T) { clt := baseClient(t, clusters) mockClt, ok := clt.(*mockEnrollEKSClusterClient) require.True(t, ok) - mockClt.checkAgentAlreadyInstalled = func(getter genericclioptions.RESTClientGetter, logger logrus.FieldLogger) (bool, error) { + mockClt.checkAgentAlreadyInstalled = func(ctx context.Context, getter genericclioptions.RESTClientGetter, logger logrus.FieldLogger) (bool, error) { return true, nil } return mockClt @@ -418,7 +418,7 @@ type mockEnrollEKSClusterClient struct { deleteAccessEntry func(context.Context, *eks.DeleteAccessEntryInput, ...func(*eks.Options)) (*eks.DeleteAccessEntryOutput, error) describeCluster func(context.Context, *eks.DescribeClusterInput, ...func(*eks.Options)) (*eks.DescribeClusterOutput, error) getCallerIdentity func(context.Context, *sts.GetCallerIdentityInput, ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error) - checkAgentAlreadyInstalled func(genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error) + checkAgentAlreadyInstalled func(context.Context, genericclioptions.RESTClientGetter, logrus.FieldLogger) (bool, error) installKubeAgent func(context.Context, *eksTypes.Cluster, string, string, string, genericclioptions.RESTClientGetter, logrus.FieldLogger, EnrollEKSClustersRequest) error createToken func(ctx context.Context, token types.ProvisionToken) error } @@ -465,9 +465,9 @@ func (m *mockEnrollEKSClusterClient) GetCallerIdentity(ctx context.Context, para return &sts.GetCallerIdentityOutput{}, nil } -func (m *mockEnrollEKSClusterClient) CheckAgentAlreadyInstalled(kubeconfig genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) { +func (m *mockEnrollEKSClusterClient) CheckAgentAlreadyInstalled(ctx context.Context, kubeconfig genericclioptions.RESTClientGetter, log logrus.FieldLogger) (bool, error) { if m.checkAgentAlreadyInstalled != nil { - return m.checkAgentAlreadyInstalled(kubeconfig, log) + return m.checkAgentAlreadyInstalled(ctx, kubeconfig, log) } return false, nil }