Add Docker Hub login to Drone's Kubernetes pipelines (#23956)

* Add Docker Hub login to kubernetes pipelines

After moving Drone to AWS, we're seeing image pulls get rate limited
because they're all coming from the same IP (an AWS NAT gateway).

To avoid this, we refactor pipelines to cache/reuse images where
possible, as well as add authentication to dockerhub pulls.

* Drop dockerVolumes and dockerVolumeRefs

We don't actually consistently want these in all places.  E.g. parallel
pipelines cannot share a volumeRefDockerConfig, as they'll stop on each
others login information.

* Remove shared docker config from parallel pipelines

A shared volume results in the different steps racing against each
other.

* Remove docker config from relcli steps

We don't actually pull from dockerhub in these steps.

* Fix typos

Co-authored-by: Reed Loden <reed@goteleport.com>
Co-authored-by: Walt <walt@goteleport.com>

---------

Co-authored-by: Trent Clarke <trent@goteleport.com>
Co-authored-by: Reed Loden <reed@goteleport.com>

dronegen/aws.go

@@ -93,6 +93,7 @@ func kubernetesAssumeAwsRoleStep(s kubernetesRoleSettings) step {
 	return step{
 		Name:  s.name,
 		Image: "amazon/aws-cli",
+		Pull:  "if-not-exists",
 		Environment: map[string]value{
 			"AWS_ACCESS_KEY_ID":     s.awsAccessKeyID,
 			"AWS_SECRET_ACCESS_KEY": s.awsSecretAccessKey,
@@ -125,6 +126,7 @@ func kubernetesUploadToS3Step(s kubernetesS3Settings) step {
 	return step{
 		Name:  "Upload to S3",
 		Image: "amazon/aws-cli",
+		Pull:  "if-not-exists",
 		Environment: map[string]value{
 			"AWS_S3_BUCKET": {fromSecret: "AWS_S3_BUCKET"},
 			"AWS_REGION":    {raw: s.region},

dronegen/buildbox.go

@@ -69,7 +69,8 @@ func buildboxPipelineStep(buildboxName string, fips bool) step {
 	return step{
 		Name:    "Build and push " + buildboxName,
 		Image:   "docker",
-		Volumes: []volumeRef{volumeRefDocker, volumeRefAwsConfig},
+		Pull:    "if-not-exists",
+		Volumes: []volumeRef{volumeRefAwsConfig, volumeRefDocker, volumeRefDockerConfig},
 		Commands: []string{
 			`apk add --no-cache make aws-cli`,
 			`chown -R $UID:$GID /go`,
@@ -101,7 +102,7 @@ func buildboxPipeline() pipeline {
 	// only on master for now; add the release branch name when forking a new release series.
 	p.Trigger = pushTriggerForBranch("master", "branch/*")
 	p.Workspace = workspace{Path: "/go/src/github.com/gravitational/teleport"}
-	p.Volumes = []volume{volumeDocker, volumeAwsConfig}
+	p.Volumes = []volume{volumeAwsConfig, volumeDocker, volumeDockerConfig}
 	p.Services = []service{
 		dockerService(),
 	}

dronegen/common.go

@@ -86,6 +86,25 @@ var (
 		Name: "awsconfig",
 		Path: "/root/.aws",
 	}
+
+	// volumeDockerConfig is a temporary volume for storing docker
+	// credentials for use with the Docker-in-Docker service we use
+	// to isolate the host machines docker daemon from the one used
+	// during the build. Mount this any time you use `volumeDocker`
+	//
+	// Drone claims to destroy the the temp volumes after a workflow
+	// has run, so it should be safe to write credentials etc.
+	volumeDockerConfig = volume{
+		Name: "dockerconfig",
+		Temp: &volumeTemp{},
+	}
+
+	// volumeRefDockerConfig is how you reference the docker config
+	// volume in a workflow step
+	volumeRefDockerConfig = volumeRef{
+		Name: "dockerconfig",
+		Path: "/root/.docker",
+	}
 )
 
 var buildboxVersion value
@@ -242,18 +261,6 @@ func dockerRegistryService() service {
 	}
 }
 
-// dockerVolumes returns a slice of volumes
-// It includes the Docker socket volume by default, plus any extra volumes passed in
-func dockerVolumes(v ...volume) []volume {
-	return append(v, volumeDocker)
-}
-
-// dockerVolumeRefs returns a slice of volumeRefs
-// It includes the Docker socket volumeRef as a default, plus any extra volumeRefs passed in
-func dockerVolumeRefs(v ...volumeRef) []volumeRef {
-	return append(v, volumeRefDocker)
-}
-
 // releaseMakefileTarget gets the correct Makefile target for a given arch/fips/centos combo
 func releaseMakefileTarget(b buildType) string {
 	makefileTarget := fmt.Sprintf("release-%s", b.arch)
@@ -283,10 +290,16 @@ func waitForDockerStep() step {
 	return step{
 		Name:  "Wait for docker",
 		Image: "docker",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			`timeout 30s /bin/sh -c 'while [ ! -S /var/run/docker.sock ]; do sleep 1; done'`,
+			`printenv DOCKERHUB_PASSWORD | docker login -u="$DOCKERHUB_USERNAME" --password-stdin`,
+		},
+		Volumes: []volumeRef{volumeRefDocker, volumeRefDockerConfig},
+		Environment: map[string]value{
+			"DOCKERHUB_USERNAME": {fromSecret: "DOCKERHUB_USERNAME"},
+			"DOCKERHUB_PASSWORD": {fromSecret: "DOCKERHUB_READONLY_TOKEN"},
 		},
-		Volumes: []volumeRef{volumeRefDocker},
 	}
 }
 
@@ -295,6 +308,7 @@ func waitForDockerRegistryStep() step {
 	return step{
 		Name:  "Wait for docker registry",
 		Image: "alpine",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			"apk add curl",
 			fmt.Sprintf(`timeout 30s /bin/sh -c 'while [ "$(curl -s -o /dev/null -w %%{http_code} http://%s/)" != "200" ]; do sleep 1; done'`, LocalRegistrySocket),
@@ -306,6 +320,7 @@ func verifyTaggedStep() step {
 	return step{
 		Name:  "Verify build is tagged",
 		Image: "alpine:latest",
+		Pull:  "if-not-exists",
 		Commands: []string{
 			"[ -n ${DRONE_TAG} ] || (echo 'DRONE_TAG is not set. Is the commit tagged?' && exit 1)",
 		},
@@ -317,6 +332,7 @@ func cloneRepoStep(clonePath, commit string) step {
 	return step{
 		Name:     "Check out code",
 		Image:    "alpine/git:latest",
+		Pull:     "if-not-exists",
 		Commands: cloneRepoCommands(clonePath, commit),
 	}
 }

dronegen/container_image_products.go

@@ -478,7 +478,7 @@ func (p *Product) createBuildStep(arch string, version *ReleaseVersion, publicEc
 	step := step{
 		Name:        p.GetBuildStepName(arch, version),
 		Image:       "docker",
-		Volumes:     dockerVolumeRefs(volumeRefAwsConfig),
+		Volumes:     []volumeRef{volumeRefAwsConfig, volumeRefDocker}, // no docker config volume, as this will race
 		Environment: envVars,
 		Commands:    commands,
 		DependsOn:   getStepNames(publicEcrPullRegistry.SetupSteps),

dronegen/container_images_release_version.go

@@ -48,7 +48,7 @@ func (rv *ReleaseVersion) buildVersionPipeline(triggerSetupSteps []step, flags *
 		dockerService(),
 		dockerRegistryService(),
 	}
-	pipeline.Volumes = dockerVolumes(volumeAwsConfig)
+	pipeline.Volumes = []volume{volumeAwsConfig, volumeDocker, volumeDockerConfig}
 	pipeline.Environment = map[string]value{
 		"DEBIAN_FRONTEND": {
 			raw: "noninteractive",

dronegen/container_images_repos.go

@@ -62,6 +62,7 @@ func NewEcrContainerRepo(accessKeyIDSecret, secretAccessKeySecret, roleSecret, d
 	loginCommands := []string{
 		"apk add --no-cache aws-cli",
 		fmt.Sprintf("aws %s get-login-password --region=%s | docker login -u=\"AWS\" --password-stdin %s", loginSubcommand, ecrRegion, domain),
+		`printenv DOCKERHUB_PASSWORD | docker login -u="$DOCKERHUB_USERNAME" --password-stdin`,
 	}
 
 	if guaranteeUnique {
@@ -72,7 +73,9 @@ func NewEcrContainerRepo(accessKeyIDSecret, secretAccessKeySecret, roleSecret, d
 		Name:        repoName,
 		IsImmutable: isImmutable,
 		EnvironmentVars: map[string]value{
-			"AWS_PROFILE": {raw: profileName},
+			"AWS_PROFILE":        {raw: profileName},
+			"DOCKERHUB_USERNAME": {fromSecret: "DOCKERHUB_USERNAME"},
+			"DOCKERHUB_PASSWORD": {fromSecret: "DOCKERHUB_READONLY_TOKEN"},
 		},
 		RegistryDomain: domain,
 		RegistryOrg:    registryOrg,
@@ -112,17 +115,16 @@ func NewQuayContainerRepo(dockerUsername, dockerPassword string) *ContainerRepo
 		Name:        "Quay",
 		IsImmutable: false,
 		EnvironmentVars: map[string]value{
-			"QUAY_USERNAME": {
-				fromSecret: dockerUsername,
-			},
-			"QUAY_PASSWORD": {
-				fromSecret: dockerPassword,
-			},
+			"QUAY_USERNAME":      {fromSecret: dockerUsername},
+			"QUAY_PASSWORD":      {fromSecret: dockerPassword},
+			"DOCKERHUB_USERNAME": {fromSecret: "DOCKERHUB_USERNAME"},
+			"DOCKERHUB_PASSWORD": {fromSecret: "DOCKERHUB_READONLY_TOKEN"},
 		},
 		RegistryDomain: ProductionRegistryQuay,
 		RegistryOrg:    registryOrg,
 		LoginCommands: []string{
 			fmt.Sprintf("docker login -u=\"$QUAY_USERNAME\" -p=\"$QUAY_PASSWORD\" %q", ProductionRegistryQuay),
+			`printenv DOCKERHUB_PASSWORD | docker login -u="$DOCKERHUB_USERNAME" --password-stdin`,
 		},
 	}
 }
@@ -254,7 +256,7 @@ func (cr *ContainerRepo) pullPushStep(image *Image, dependencySteps []string) (s
 	return step{
 		Name:        fmt.Sprintf("Pull %s and push it to %s", image.GetDisplayName(), localRepo.Name),
 		Image:       "docker",
-		Volumes:     dockerVolumeRefs(volumeRefAwsConfig),
+		Volumes:     []volumeRef{volumeRefAwsConfig, volumeRefDocker}, // no docker config volume, as this will race
 		Environment: cr.EnvironmentVars,
 		Commands:    commands,
 		DependsOn:   dependencySteps,
@@ -304,7 +306,7 @@ func (cr *ContainerRepo) tagAndPushStep(buildStepDetails *buildStepOutput, image
 	step := step{
 		Name:        fmt.Sprintf("Tag and push image %q to %s", buildStepDetails.BuiltImage.GetDisplayName(), cr.Name),
 		Image:       "docker",
-		Volumes:     dockerVolumeRefs(volumeRefAwsConfig),
+		Volumes:     []volumeRef{volumeRefAwsConfig, volumeRefDocker}, // no docker config volume, as this will race
 		Environment: cr.EnvironmentVars,
 		Commands:    commands,
 		DependsOn:   dependencySteps,
@@ -332,7 +334,7 @@ func (cr *ContainerRepo) createAndPushManifestStep(manifestImage *Image, pushSte
 	return step{
 		Name:        fmt.Sprintf("Create manifest and push %q to %s", manifestImage.GetDisplayName(), cr.Name),
 		Image:       "docker",
-		Volumes:     dockerVolumeRefs(volumeRefAwsConfig),
+		Volumes:     []volumeRef{volumeRefAwsConfig, volumeRefDocker}, // no docker config volume, as this will race
 		Environment: cr.EnvironmentVars,
 		Commands:    cr.buildCommandsWithLogin(commands),
 		DependsOn:   pushStepNames,

dronegen/gha.go

@@ -62,6 +62,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline {
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -70,6 +71,7 @@ func ghaBuildPipeline(b ghaBuildType) pipeline {
 		{
 			Name:  "Delegate build to GitHub",
 			Image: fmt.Sprintf("golang:%s-alpine", GoVersion),
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GHA_APP_KEY": {fromSecret: "GITHUB_WORKFLOW_APP_PRIVATE_KEY"},
 			},

dronegen/main.go

@@ -39,6 +39,22 @@ func main() {
 	pipelines = append(pipelines, buildContainerImagePipelines()...)
 	pipelines = append(pipelines, publishReleasePipeline())
 
+	// Inject the Drone-level dockerhub credentials into all non-exec
+	// pipelines. Drone will then use the docker credentials file in
+	// the named secret as its credentials when pulling images from
+	// dockerhub.
+	//
+	// Exec pipelines do not have the `image_pull_secrets` option, as
+	// their steps are invoked directly on the host runner and not
+	// into a per-step container.
+	for pidx := range pipelines {
+		p := &pipelines[pidx]
+		if p.Type == "exec" {
+			continue
+		}
+		p.ImagePullSecrets = append(p.ImagePullSecrets, "DOCKERHUB_CREDENTIALS")
+	}
+
 	if err := writePipelines(".drone.yml", pipelines); err != nil {
 		fmt.Println("failed writing drone pipelines:", err)
 		os.Exit(1)

dronegen/push.go

@@ -126,7 +126,7 @@ func pushPipeline(b buildType) pipeline {
 	}
 	p.Trigger = triggerPush
 	p.Workspace = workspace{Path: "/go"}
-	p.Volumes = []volume{volumeDocker}
+	p.Volumes = []volume{volumeDocker, volumeDockerConfig}
 	p.Services = []service{
 		dockerService(),
 	}
@@ -134,6 +134,7 @@ func pushPipeline(b buildType) pipeline {
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -143,8 +144,9 @@ func pushPipeline(b buildType) pipeline {
 		{
 			Name:        "Build artifacts",
 			Image:       "docker",
+			Pull:        "if-not-exists",
 			Environment: pushEnvironment,
-			Volumes:     []volumeRef{volumeRefDocker},
+			Volumes:     []volumeRef{volumeRefDocker, volumeRefDockerConfig},
 			Commands:    pushBuildCommands(b),
 		},
 		sendErrorToSlackStep(),

dronegen/relcli.go

@@ -44,11 +44,7 @@ func relcliPipeline(trigger trigger, name string, stepName string, command strin
 	}
 
 	p.Services = []service{dockerService(volumeRefTmpfs)}
-	p.Volumes = []volume{
-		volumeDocker,
-		volumeTmpfs,
-		volumeAwsConfig,
-	}
+	p.Volumes = []volume{volumeTmpfs, volumeAwsConfig, volumeDocker, volumeDockerConfig}
 
 	return p
 }
@@ -60,10 +56,7 @@ func pullRelcliStep(awsConfigVolumeRef volumeRef) step {
 		Environment: map[string]value{
 			"AWS_DEFAULT_REGION": {raw: "us-west-2"},
 		},
-		Volumes: []volumeRef{
-			volumeRefDocker,
-			volumeRefAwsConfig,
-		},
+		Volumes: []volumeRef{volumeRefDocker, volumeRefAwsConfig},
 		Commands: []string{
 			`apk add --no-cache aws-cli`,
 			`aws ecr get-login-password | docker login -u="AWS" --password-stdin 146628656107.dkr.ecr.us-west-2.amazonaws.com`,
@@ -83,11 +76,7 @@ func executeRelcliStep(name string, command string) step {
 			"RELCLI_CERT":     {raw: "/tmpfs/creds/releases.crt"},
 			"RELCLI_KEY":      {raw: "/tmpfs/creds/releases.key"},
 		},
-		Volumes: []volumeRef{
-			volumeRefDocker,
-			volumeRefTmpfs,
-			volumeRefAwsConfig,
-		},
+		Volumes: []volumeRef{volumeRefDocker, volumeRefTmpfs, volumeRefAwsConfig},
 		Commands: []string{
 			`mkdir -p /tmpfs/creds`,
 			`echo "$RELEASES_CERT" | base64 -d > "$RELCLI_CERT"`,

dronegen/tag.go

@@ -274,7 +274,7 @@ func tagPipeline(b buildType) pipeline {
 	p.Trigger = triggerTag
 	p.DependsOn = []string{tagCleanupPipelineName}
 	p.Workspace = workspace{Path: "/go"}
-	p.Volumes = []volume{volumeAwsConfig, volumeDocker}
+	p.Volumes = []volume{volumeAwsConfig, volumeDocker, volumeDockerConfig}
 	p.Services = []service{
 		dockerService(),
 	}
@@ -282,6 +282,7 @@ func tagPipeline(b buildType) pipeline {
 		{
 			Name:  "Check out code",
 			Image: "docker:git",
+			Pull:  "if-not-exists",
 			Environment: map[string]value{
 				"GITHUB_PRIVATE_KEY": {fromSecret: "GITHUB_PRIVATE_KEY"},
 			},
@@ -291,13 +292,15 @@ func tagPipeline(b buildType) pipeline {
 		{
 			Name:        "Build artifacts",
 			Image:       "docker",
+			Pull:        "if-not-exists",
 			Environment: tagEnvironment,
-			Volumes:     []volumeRef{volumeRefDocker},
+			Volumes:     []volumeRef{volumeRefDocker, volumeRefDockerConfig},
 			Commands:    tagBuildCommands(b),
 		},
 		{
 			Name:     "Copy artifacts",
 			Image:    "docker",
+			Pull:     "if-not-exists",
 			Commands: tagCopyArtifactCommands(b),
 		},
 		kubernetesAssumeAwsRoleStep(kubernetesRoleSettings{
@@ -317,6 +320,7 @@ func tagPipeline(b buildType) pipeline {
 		{
 			Name:     "Register artifacts",
 			Image:    "docker",
+			Pull:     "if-not-exists",
 			Commands: tagCreateReleaseAssetCommands(b, "", extraQualifications),
 			Environment: map[string]value{
 				"RELEASES_CERT": {fromSecret: "RELEASES_CERT"},
@@ -465,12 +469,10 @@ func tagPackagePipeline(packageType string, b buildType) pipeline {
 		environment["OSS_TARBALL_PATH"] = value{raw: "/go/artifacts"}
 	}
 
-	packageDockerVolumes := []volume{
-		volumeDocker,
-		volumeAwsConfig,
-	}
+	packageDockerVolumes := []volume{volumeAwsConfig, volumeDocker, volumeDockerConfig}
 	packageDockerVolumeRefs := []volumeRef{
 		volumeRefDocker,
+		volumeRefDockerConfig,
 		volumeRefAwsConfig,
 	}
 	packageDockerService := dockerService()

dronegen/types.go

@@ -29,20 +29,21 @@ import (
 type pipeline struct {
 	comment string
 
-	Kind        string           `yaml:"kind"`
-	Type        string           `yaml:"type"`
-	Name        string           `yaml:"name"`
-	Environment map[string]value `yaml:"environment,omitempty"`
-	Trigger     trigger          `yaml:"trigger"`
-	Workspace   workspace        `yaml:"workspace,omitempty"`
-	Platform    platform         `yaml:"platform,omitempty"`
-	Node        map[string]value `yaml:"node,omitempty"`
-	Clone       clone            `yaml:"clone,omitempty"`
-	DependsOn   []string         `yaml:"depends_on,omitempty"`
-	Concurrency concurrency      `yaml:"concurrency,omitempty"`
-	Steps       []step           `yaml:"steps"`
-	Services    []service        `yaml:"services,omitempty"`
-	Volumes     []volume         `yaml:"volumes,omitempty"`
+	Kind             string           `yaml:"kind"`
+	Type             string           `yaml:"type"`
+	Name             string           `yaml:"name"`
+	Environment      map[string]value `yaml:"environment,omitempty"`
+	Trigger          trigger          `yaml:"trigger"`
+	Workspace        workspace        `yaml:"workspace,omitempty"`
+	Platform         platform         `yaml:"platform,omitempty"`
+	Node             map[string]value `yaml:"node,omitempty"`
+	Clone            clone            `yaml:"clone,omitempty"`
+	DependsOn        []string         `yaml:"depends_on,omitempty"`
+	Concurrency      concurrency      `yaml:"concurrency,omitempty"`
+	Steps            []step           `yaml:"steps"`
+	Services         []service        `yaml:"services,omitempty"`
+	Volumes          []volume         `yaml:"volumes,omitempty"`
+	ImagePullSecrets []string         `yaml:"image_pull_secrets,omitempty"`
 }
 
 func newKubePipeline(name string) pipeline {
@@ -169,6 +170,7 @@ type volumeRef struct {
 type step struct {
 	Name        string              `yaml:"name"`
 	Image       string              `yaml:"image,omitempty"`
+	Pull        string              `yaml:"pull,omitempty"`
 	Commands    []string            `yaml:"commands,omitempty"`
 	Environment map[string]value    `yaml:"environment,omitempty"`
 	Volumes     []volumeRef         `yaml:"volumes,omitempty"`