loadtesting automation improvements (#25366)

* move existing loadtest assets * new loadtest automation helpers * ignore sourcing uncovered files
2026-10-25 18:59:51 +08:00 · 2023-06-05 23:11:56 +00:00
parent e5ba338cea
commit 2ec7490283
85 changed files with 997 additions and 2150 deletions
@@ -949,6 +949,7 @@ lint-sh:
 	find . -type f \( -name '*.sh' -or -name '*.sh.tmpl' \) -not -path "*/node_modules/*" | xargs \
 		shellcheck \
 		--exclude=SC2086 \
+		--exclude=SC1091 \
 		$(SH_LINT_FLAGS)

 	# lint AWS AMI scripts
@@ -15,4 +15,6 @@ k8s/secrets/**
 **/terraform.tfstate*
 **/terraform.tfvars

-.secrets
+.secrets
+
+control-plane/vars.env
@@ -1,75 +1,78 @@
-SOAK_TEST_DURATION ?= 30m
-USE_CERT_MANAGER ?= yes
-TELEPORT_IMAGE ?= public.ecr.aws/gravitational/teleport-ent:12.0.0
-NAMESPACE ?= loadtest
+# creates an eks cluster
+.PHONY: create-aws-cluster
+create-aws-cluster:
+	$(MAKE) -C cluster/aws create-cluster

-.PHONY: reserve-ips
-reserve-ips:
-	@make -C network apply
+# creates an gks cluster
+.PHONY: create-gcp-cluster
+create-gcp-cluster:
+	$(MAKE) -C cluster/gcp create-cluster

-# create gke cluster 
-.PHONY: create-cluster
-create-cluster:
-	@make -C ./cluster apply
-	@make -C ./cluster get-creds
+# deletes an eks cluster
+.PHONY: delete-aws-cluster
+delete-aws-cluster:
+	$(MAKE) -C cluster/aws destroy

-# delete gke cluster
-.PHONY: delete-cluster
-delete-cluster:
-	@make -C cluster destroy USE_CERT_MANAGER=$(USE_CERT_MANAGER)
+# deletes a gks cluster
+.PHONY: delete-gcp-cluster
+delete-gcp-cluster:
+	$(MAKE) -C cluster/gcp destroy

-# deploy teleport with etcd backend to loadtest namespace
-.PHONY: deploy-etcd-cluster
-deploy-etcd-cluster:
-	@make -C k8s apply BACKEND=etcd USE_CERT_MANAGER=$(USE_CERT_MANAGER) TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+# cleans up all deployed resources in the cluster and
+# any resources created in the cloud provider
+.PHONY: destroy-deploy
+destroy-deploy:
+	$(MAKE) -C control-plane destroy

-# deploy teleport with firestore backend to loadtest namespace
-.PHONY: deploy-firestore-cluster
-deploy-firestore-cluster:
-	@make -C k8s apply BACKEND=firestore USE_CERT_MANAGER=$(USE_CERT_MANAGER) TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+# deploys teleport auth and proxy
+.PHONY: deploy-control-plane
+deploy-control-plane:
+	$(MAKE) -C control-plane deploy

-# deploy teleport with dynamo backend to loadtest namespace
-.PHONY: deploy-dynamo-cluster
-deploy-dynamo-cluster:
-	@make -C k8s apply BACKEND=dynamo USE_CERT_MANAGER=$(USE_CERT_MANAGER) TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+# deletes the teleport auth and proxy deployments
+.PHONY: delete-teleport
+delete-teleport:
+	$(MAKE) -C control-plane delete-teleport

-# delete the loadtest namespace
-.PHONY: delete-deploy
-delete-deploy:
-	@make -C k8s clean USE_CERT_MANAGER=$(USE_CERT_MANAGER)
+# deploys teleport auth and proxy to the cluster
+.PHONY: deploy-teleport
+deploy-teleport:
+	$(MAKE) -C control-plane deploy-teleport

-# run soak tests
-.PHONY: run-soak-tests
-run-soak-tests:
-	@make -C k8s run-soak-tests SOAK_TEST_DURATION=$(SOAK_TEST_DURATION) TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+NODE_REPLICAS ?= 1000
+PROXY_SERVER ?=
+NODE_TOKEN ?=

-# run 500 node trusted cluster scaling test
-# This installs 500 trusted clusters, waits for a period of time and then
-# deletes all 500. When they are all removed from auth the process is repeated once more.
-# The test helps identify any potential memory leaks caused by remote clusters
-.PHONY: run-tc-scaling-test
-run-tc-scaling-test:
-	@make -C k8s run-tc-scaling-test TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+# deploys nodes and adds them to the teleport cluster
+.PHONY: deploy-nodes
+deploy-nodes:
+	helm upgrade --install node-agents -n agents --create-namespace ./helm/node-agent/ --values ./helm/values/node-agents.yaml --set replicaCount=${NODE_REPLICAS} --set agentsPerPod=10 --set proxyServer=${PROXY_SERVER} --set joinParams.token_name=${NODE_TOKEN}

-# run the 10k scaling test
-# This tests installs 10,000 IoT nodes and then removes all 10,000 nodes after a fixed period of time.
-# The process repeats once more and then the same is done for non-IoT nodes. The test helps identify any
-# potential memory leaks caused by large clusters for both non-IoT and IoT nodes.
-.PHONY: run-scaling-test
-run-scaling-test:
-	@make -C k8s run-scaling-test TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
+# deletes all nodes
+.PHONY: delete-nodes
+delete-nodes:
+	helm delete -n agents node-agents

-# list pods in loadtest namespace
-.PHONY: pods
-pods:
-	@make -C k8s pods NAMESPACE=$(NAMESPACE)
+TELEPORT_USER ?= test

-# get cluster credentials
-.PHONY: get-creds
-get-creds:
-	@make -C cluster get-creds
+# creates a local teleport user
+.PHONY: create-user
+create-user:
+	kubectl --namespace teleport exec deploy/teleport-auth -- tctl users add ${TELEPORT_USER} --roles=access,editor,auditor
+
+# creates a token that can be used by nodes to join the
+# teleport cluster
+.PHONY: create-token
+create-token:
+	kubectl --namespace teleport exec deploy/teleport-auth -- tctl tokens add --type=node --ttl=8h
+
+# forwards the grafana web app to localhost:6060
+.PHONY: forward-grafana
+forward-grafana:
+	$(MAKE) -C control-plane forward-monitor
+
+# scales teleport auth pods up/down
+.PHONY: scale-auth
+scale-auth:
+	kubectl scale deployment/teleport-auth --replicas=$(REPLICAS) -n teleport

-# collect heap and goroutine profiles
-.PHONY: collect-profiles
-collect-profiles:
-	@make -C k8s collect-profiles PROFILE_LOCATION=$(shell pwd) NAMESPACE=$(NAMESPACE)
@@ -1,119 +0,0 @@
-# loadtest
-
-Automation for the `loadtest` kuberentes cluster and performing Teleport load tests.
-
-## About
-
-This automation sets up a kubernetes cluster named `loadtest` in the configured GCP project.  
-This cluster is used for, among other things, the `1k`/`10k` scaling tests that are performed as part of
-Teleports manual release test plan.
-
-## Setup
-
-### Prerequisites
- Make sure you have the following tools installed:
-    - `terraform`
-    - `gcloud`
-    - `kubectl`
- Make sure that you have a GCP service account key with `Compute Admin`, `Compute Network Admin`,
-      `Kubernetes Engine Admin`, `Kubernetes Engine Cluster Admin`, and `Service Account User`
-  - To authenticate as the service account follow these [instructions](https://cloud.google.com/docs/authentication/production)
- Make sure you have reserved static ip addresses for the proxy
-  - This only needs to be done once per GCP project, see the [network docs](./network/README.md) for details
-
-### Creating the Cluster
-
-First create a cluster, if you are running this automation for the first time, you may be asked to run
-`terraform init` from the cluster directory before continuing. To resize the cluster, edit [`terraform.tfvars`](cluster/terraform.tfvars) as needed.
-
-```bash
-$ make create-cluster
-```
-
-### DNS Entries
-Before deploying anything to the cluster you first need to set `PROXY_HOST`. These variables should
-be the DNS names to be used for the [`proxy`](./k8s/proxy.yaml). When everything is successfully deployed you should be able
-to navigate to `https://PROXY_HOST:3080` in your browser.
-
-```bash
-$ export PROXY_HOST=proxy.loadtest.com 
-```
-
-### TLS Certificates
-
-Certificates can be provisioned automatically via [cert-manager](https://cert-manager.io/) or by hand.
-
-#### cert-manager
-If you would like to use cert-manager to automatically retrieve TLS certificates for you, create 
-[`cetificate.yaml`](./k8s/certificate.yaml) with your `cert-manager.io/v1/ClusterIssuer`, `cert-manager.io/v1/Certificate` and any
-secrets required for your solver.
-
-#### Kubernetes secret
-
-To manual supply TLS certificates create a tls secret, run the following:
-
-```bash
-$ kubectl create secret tls teleport-tls -n loadtest \
-    --cert=path/to/cert/file \
-    --key=path/to/key/file
-```
-
-You **must** also provide `USE_CERT_MANAGER=no` to all make commands below.
-
-### Teleport Configuration
-You must supply an [OIDC Connector](https://goteleport.com/docs/enterprise/sso/oidc/) that will be used for authentication. Create [`oidc.yaml`](./teleport/oidc.yaml)
-before attempting to deploy Teleport to the cluster.
-
-
-## Deploy Teleport
-
-### etcd Backend
-
-```bash
-$ make deploy-etcd-cluster
-```
-
-### Firestore Backend
-
-To use the firestore backend you must have a GCP service account key with `Cloud Datastore User`, `Cloud Datastore Index Admin`
-`Storage Object Admin` permissions. Set `GCP_CREDS_LOCATION` to the location that you saved the service account key.
-
-```bash
-$ export GCP_CREDS_LOCATION=/path/to/service/account/key
-$ make deploy-firestore-cluster
-```
-
-## Running Tests
-
-To run soak tests:
-
-```bash
-$ make run-soak-tests
-```
-
-
-**Note:** You must have enough nodes in the cluster to run the following tests. Ensure your `node_count` in [`terraform.tfvars`](cluster/terraform.tfvars) is correctly set.
-
-To run the 10k node scaling tests:
-
-```bash
-$ make run-scaling-test
-```
-
-To run the trusted cluster scaling test:
-```bash
-$ make run-tc-scaling-test
-```
-
-## Cleanup
-
-To delete the loadtest deployment:
-```bash
-$ make delete-deploy
-```
-
-
-To delete the entire cluster:
-```bash
-$ make delete-cluster
-```
@@ -0,0 +1,14 @@
+# creates an eks cluster
+.PHONY: create-cluster
+create-cluster:
+	eksctl create cluster -f cluster.yaml
+
+# gets kubernetes credentials
+.PHONY: get-creds
+get-creds:
+	eksctl utils write-kubeconfig
+
+# deletes the eks cluster
+.PHONY: destroy
+destroy:
+	eksctl delete cluster -f cluster.yaml --wait --disable-nodegroup-eviction
@@ -0,0 +1,21 @@
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+metadata:
+  name:
+  region:
+  version: "1.23"
+
+iam:
+  withOIDC: true
+
+addons:
+  - name: aws-ebs-csi-driver
+    version: v1.11.4-eksbuild.1
+    attachPolicyARNs:
+      - arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy
+
+managedNodeGroups:
+  - name:
+    instanceType: m5.4xlarge
+    minSize: 2
+    maxSize: 10
@@ -4,8 +4,8 @@ plan:
 	terraform plan

 # apply cluster configuration
-.PHONY: apply
-apply:
+.PHONY: create-cluster
+create-cluster:
 	terraform apply

 # authorize kubectl to manage the cluster
@@ -0,0 +1,2 @@
+vars.env
+state
@@ -0,0 +1,26 @@
+# initializes helm repos and install teleport
+.PHONY: deploy
+deploy:
+	./init.sh
+	./apply.sh
+
+# initializes helm repos and install teleport
+.PHONY: destroy
+destroy:
+	./clean-non-kube.sh
+
+.PHONY: delete-teleport
+delete-teleport:
+	./teleport/uninstall-teleport.sh
+
+.PHONY: deploy-teleport
+deploy-teleport:
+	./teleport/install-teleport.sh
+	./teleport/wait.sh auth
+	./dns/update-record.sh UPSERT
+	./teleport/wait.sh proxy
+
+# forwards the in cluster grafana to localhost:6060
+.PHONY: forward-monitor
+forward-monitor:
+	./monitoring/port-forward.sh
@@ -0,0 +1,24 @@
+# control-plane
+
+A collection of scripts used to automate the process of setting up a teleport control plane using the
+`teleport-cluster` helm chart in `aws`.
+
+*note:* This is an experimental tool used for internal testing and may be buggy/brittle.
+
+## Quickstart
+
+- Ensure aws cli is authenticated.
+
+- Set up eks cluster and load credentials into kubectl.
+
+- Create a `vars.env` in this directory (see `example-vars.env` for required contents and explanation).
+Note that the name of the eks cluster must match the `CLUSTER_NAME` variable.
+
+- If needed, run `init.sh` to add/update all necessary helm repos.
+
+- Invoke `./apply.sh` to spin up the teleport control plane and monitoring stack.
+
+- Use `./monitoring/port-forward.sh` to forward the grafana ui.
+
+- When finished, first invoke `./clean-non-kube.sh`, then destroy the eks cluster (this ordering
+is important, as some of the resources created by these scripts interfere with eks cluster teardown).
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# set up cluster resources (kube cluster must exist, aws and kubectl must be authenticated,
+# and helm repos must be up to date).
+
+source vars.env
+
+log_info() {
+    echo "[i] $* [ $(caller | awk '{print $1}') ]" >&2
+}
+
+
+case "$TELEPORT_BACKEND" in
+    dynamo)
+        ;;
+    etcd)
+        ;;
+    *)
+        echo "invalid teleport backend '$TELEPORT_BACKEND', expected one of 'dynamo' or 'etcd'" >&2
+        exit 1
+        ;;
+esac
+
+log_info "generating iam policies..."
+
+./policies/gen-policies.sh
+
+log_info "creating iam policies..."
+
+./policies/create-policies.sh
+
+log_info "attaching iam policies..."
+
+./policies/attach-policies.sh attach
+
+log_info "installing monitoring stack..."
+
+./monitoring/install-monitoring.sh
+
+log_info "setting up cert-manager..."
+
+./dns/init-cert-manager.sh
+
+case "$TELEPORT_BACKEND" in
+    dynamo)
+        log_info "generating helm values for dynamo-backed control plane..."
+        ./teleport/gen-dynamo-teleport.sh
+        ;;
+    etcd)
+        log_info "installing etcd..."
+        make -C ../etcd deploy
+
+        log_info "generating helm values for etcd-backed control plane..."
+        ./teleport/gen-etcd-teleport.sh
+        ;;
+    *)
+        echo "invalid teleport backend '$TELEPORT_BACKEND', expected one of 'dynamo' or 'etcd'" >&2
+        exit 1
+        ;;
+esac
+
+log_info "installing control plane chart..."
+
+./teleport/install-teleport.sh
+
+log_info "waiting for auths to report ready..."
+
+./teleport/wait.sh auth
+
+log_info "setting up dns record..."
+
+./dns/update-record.sh UPSERT # CREATE|UPSERT|DELETE
+
+log_info "waiting for proxies to report ready..."
+
+./teleport/wait.sh proxy
+
+
+if [[ "$TELEPORT_BACKEND" == "dynamo" ]]; then
+    log_info "switching dynamo to on-demand mode..."
+
+    ./storage/set-on-demand.sh
+fi
+
+log_info "setting grafana admin password..."
+
+./monitoring/set-password.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# clean up the resources not automatically destroyed by bringing down the kube cluster.
+
+log_info() {
+    echo "[i] $* [ $(caller | awk '{print $1}') ]" >&2
+}
+
+log_info "detaching iam policies..."
+
+./policies/attach-policies.sh detach
+
+log_info "deleting iam policies..."
+
+./policies/delete-policies.sh
+
+log_info "destroying cluster storage..."
+
+./storage/delete-storage.sh
+
+log_info "deleting dns records..."
+
+./dns/update-record.sh DELETE
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+issuer_yaml="$STATE_DIR/aws-issuer.yaml"
+
+mkdir -p "$STATE_DIR"
+
+cat > "$issuer_yaml" <<EOF
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-production
+  namespace: teleport
+spec:
+  acme:
+    email: ${EMAIL}
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-production
+    solvers:
+    - selector:
+        dnsZones:
+          - ${ROUTE53_ZONE}
+      dns01:
+        route53:
+          region: ${AWS_REGION}
+          hostedZoneID: ${ROUTE53_ZONE_ID}
+EOF
+
+# install cert-manager in the kube cluster
+helm install cert-manager jetstack/cert-manager \
+    --create-namespace \
+    --namespace cert-manager \
+    --set installCRDs=true \
+    --set extraArgs="{--issuer-ambient-credentials}" # required to automount ambient AWS credentials when using an Issuer
+
+
+# set up namespace
+kubectl create namespace teleport
+kubectl label namespace teleport 'pod-security.kubernetes.io/enforce=baseline'
+
+# configure cert manager
+kubectl --namespace teleport create -f "$issuer_yaml"
+
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+action="${1}"
+
+case "$action" in
+    CREATE)
+        ;;
+    UPSERT)
+        ;;
+    DELETE)
+        ;;
+    *)
+        echo "invalid dns record update action '${action}', expected CREATE|UPSERT|DELETE" >&2
+        exit 1
+esac
+
+records_json="$STATE_DIR/records.json"
+
+NAMESPACE='teleport'
+RELEASE_NAME='teleport'
+MYZONE_DNS="${ROUTE53_ZONE}"
+MYDNS="${CLUSTER_NAME}.${ROUTE53_ZONE}"
+MY_CLUSTER_REGION="${AWS_REGION}"
+MYZONE="$(aws route53 list-hosted-zones-by-name --dns-name="${MYZONE_DNS?}" | jq -r '.HostedZones[0].Id' | sed s_/hostedzone/__)"
+MYELB="$(kubectl --namespace "${NAMESPACE?}" get "service/${RELEASE_NAME?}" -o jsonpath='{.status.loadBalancer.ingress[*].hostname}')"
+MYELB_NAME="${MYELB%%-*}"
+MYELB_ZONE="$(aws elbv2 describe-load-balancers --region "${MY_CLUSTER_REGION?}" --names "${MYELB_NAME?}" | jq -r '.LoadBalancers[0].CanonicalHostedZoneId')"
+jq -n --arg dns "${MYDNS?}" --arg elb "${MYELB?}" --arg elbz "${MYELB_ZONE?}" --arg act "$action" \
+    '{
+        "Comment": "Change records",
+        "Changes": [
+          {
+            "Action": $act,
+            "ResourceRecordSet": {
+              "Name": $dns,
+              "Type": "A",
+              "AliasTarget": {
+                "HostedZoneId": $elbz,
+                "DNSName": ("dualstack." + $elb),
+                "EvaluateTargetHealth": false
+              }
+            }
+          },
+          {
+            "Action": $act,
+            "ResourceRecordSet": {
+              "Name": ("*." + $dns),
+              "Type": "A",
+              "AliasTarget": {
+                "HostedZoneId": $elbz,
+                "DNSName": ("dualstack." + $elb),
+                "EvaluateTargetHealth": false
+              }
+            }
+          }
+      ]
+    }' > "$records_json"
+jq < "$records_json"
+CHANGEID="$(aws route53 change-resource-record-sets --hosted-zone-id "${MYZONE?}" --change-batch "file://${records_json}" | jq -r '.ChangeInfo.Id')"
+aws route53 get-change --id "${CHANGEID?}" | jq '.ChangeInfo.Status'
@@ -0,0 +1,32 @@
+# generated file output directory (relative)
+STATE_DIR="state"
+
+# backend variant to use (must be one of "dynamo" or "etcd")
+TELEPORT_BACKEND="dynamo"
+
+# the short name of the teleport cluster (full name will end up being <cluster-name>.<route53-zone>).
+CLUSTER_NAME="my-cluster"
+
+# session recording bucket name
+SESSION_BUCKET="${CLUSTER_NAME}-sess-deadbeefdeadbeef"
+
+# route53 zone name
+ROUTE53_ZONE="my-route53.zone"
+
+# route53 zone id
+ROUTE53_ZONE_ID="0123456789"
+
+# email for use with letsencrypt
+EMAIL="alice@example.com"
+
+# aws account id
+ACCOUNT_ID="0123456789"
+
+# grafana password
+GRAFANA_PASS="insecure"
+
+# sets the version of teleport to install
+TELEPORT_VERSION="13.0.0-alpha.2"
+
+# aws region to use for resources
+AWS_REGION="$(aws configure get region)"
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# initialize dependencies
+
+helm repo add teleport https://charts.releases.teleport.dev
+helm repo add jetstack https://charts.jetstack.io
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+monitor_yaml="$STATE_DIR/monitoring.yaml"
+
+# source: examples/load-tests/values/kube-prometheus-stack.yaml
+cat > "$monitor_yaml" <<EOF
+prometheus:
+  prometheusSpec:
+    scrapeInterval: 15s
+    retention: 30d
+    resources:
+      requests:
+        memory: 16Gi
+        cpu: "4"
+      limits:
+        memory: 16Gi
+        # cpu: 4
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 50Gi
+    podMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+EOF
+
+helm install monitoring \
+    -n monitoring \
+    --create-namespace prometheus-community/kube-prometheus-stack \
+    -f "$monitor_yaml"
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+localport="6060"
+
+echo "port forwarding grafana on $localport..."
+
+kubectl --namespace monitoring port-forward svc/monitoring-grafana "${localport}:80"
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+GRAFANA_POD=$(kubectl get pods --selector=app.kubernetes.io/name=grafana -n monitoring | tail -n 1 | awk '{print $1}')
+
+if test -z "$GRAFANA_POD"; then
+    echo "failed to discover grafana pod" >&2
+    exit 1
+fi
+
+kubectl exec --stdin --namespace="monitoring" --tty --container grafana $GRAFANA_POD -- grafana-cli admin reset-admin-password $GRAFANA_PASS
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -euo pipefail
+
+mode="${1:?'mode (one of attach or detach)'}"
+
+case "$mode" in
+    attach)
+        ;;
+    detach)
+        ;;
+    *)
+        echo "ERROR: unknown mode $mode, expected one of 'attach' or 'detach'."
+        ;;
+esac
+
+source vars.env
+
+dynamo_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-dynamo"
+
+s3_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-s3"
+
+route53_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-route53"
+
+
+# used for stripping out node role name
+role_arn_prefix="arn:aws:iam::${ACCOUNT_ID}:role/"
+
+
+log_info() {
+    echo "[i] $* [ $(caller | awk '{print $1}') ]" >&2
+}
+
+# discover the node groups associated with our eks cluster
+nodegroups="$(aws eks list-nodegroups --cluster-name="$CLUSTER_NAME" | jq -r .nodegroups[])"
+
+
+if test -z "$nodegroups"; then
+    log_info "failed to discover node group!"
+    exit 1
+fi
+
+while read -r ngroup; do
+    noderole_arn="$(aws eks describe-nodegroup \
+        --cluster-name="$CLUSTER_NAME" \
+        --nodegroup-name="$ngroup" \
+        | jq -r .nodegroup.nodeRole)"
+
+    if test -z "$noderole_arn"; then
+        log_info "failed to discover node role for group '$ngroup'!"
+        exit 1
+    fi
+
+    noderole_name="${noderole_arn#"$role_arn_prefix"}"
+
+    log_info "${mode}ing policies to discovered node role '$noderole_name'..."
+
+    if [[ "$TELEPORT_BACKEND" == "dynamo" ]]; then
+        aws iam "$mode-role-policy" \
+            --policy-arn="$dynamo_policy_arn" \
+            --role-name="$noderole_name"
+    fi
+
+    aws iam "$mode-role-policy" \
+        --policy-arn="$s3_policy_arn" \
+        --role-name="$noderole_name"
+
+    aws iam "$mode-role-policy" \
+        --policy-arn="$route53_policy_arn" \
+        --role-name="$noderole_name"
+
+done <<< "$nodegroups"
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+dynamo_policy="$STATE_DIR/dynamo-iam-policy"
+
+s3_policy="$STATE_DIR/s3-iam-policy"
+
+route53_policy="$STATE_DIR/route53-policy"
+
+if [[ "$TELEPORT_BACKEND" == "dynamo" ]]; then
+  aws iam create-policy \
+      --policy-name "${CLUSTER_NAME}-dynamo" \
+      --policy-document "$(cat "$dynamo_policy")"
+fi
+
+aws iam create-policy \
+    --policy-name "${CLUSTER_NAME}-s3" \
+    --policy-document "$(cat "$s3_policy")"
+
+aws iam create-policy \
+    --policy-name "${CLUSTER_NAME}-route53" \
+    --policy-document "$(cat "$route53_policy")"
+
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+dynamo_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-dynamo"
+
+s3_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-s3"
+
+route53_policy_arn="arn:aws:iam::${ACCOUNT_ID}:policy/${CLUSTER_NAME}-route53"
+
+if [[ "$TELEPORT_BACKEND" == "dynamo" ]]; then
+    aws iam delete-policy \
+        --policy-arn "$dynamo_policy_arn"
+fi
+
+aws iam delete-policy \
+    --policy-arn "$s3_policy_arn"
+
+aws iam delete-policy \
+    --policy-arn "$route53_policy_arn"
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+dynamo_policy="$STATE_DIR/dynamo-iam-policy"
+
+s3_policy="$STATE_DIR/s3-iam-policy"
+
+route53_policy="$STATE_DIR/route53-policy"
+
+mkdir -p "$STATE_DIR"
+
+cat > "$dynamo_policy" <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "ClusterStateStorage",
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:BatchWriteItem",
+                "dynamodb:UpdateTimeToLive",
+                "dynamodb:PutItem",
+                "dynamodb:DeleteItem",
+                "dynamodb:Scan",
+                "dynamodb:Query",
+                "dynamodb:DescribeStream",
+                "dynamodb:UpdateItem",
+                "dynamodb:DescribeTimeToLive",
+                "dynamodb:CreateTable",
+                "dynamodb:DescribeTable",
+                "dynamodb:GetShardIterator",
+                "dynamodb:GetItem",
+                "dynamodb:UpdateTable",
+                "dynamodb:GetRecords",
+                "dynamodb:UpdateContinuousBackups"
+            ],
+            "Resource": [
+                "arn:aws:dynamodb:${AWS_REGION}:${ACCOUNT_ID}:table/${CLUSTER_NAME}-backend",
+                "arn:aws:dynamodb:${AWS_REGION}:${ACCOUNT_ID}:table/${CLUSTER_NAME}-backend/stream/*"
+            ]
+        },
+        {
+            "Sid": "ClusterEventsStorage",
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:CreateTable",
+                "dynamodb:BatchWriteItem",
+                "dynamodb:UpdateTimeToLive",
+                "dynamodb:PutItem",
+                "dynamodb:DescribeTable",
+                "dynamodb:DeleteItem",
+                "dynamodb:GetItem",
+                "dynamodb:Scan",
+
+                "dynamodb:Query",
+                "dynamodb:UpdateItem",
+                "dynamodb:DescribeTimeToLive",
+                "dynamodb:UpdateTable",
+                "dynamodb:UpdateContinuousBackups"
+            ],
+            "Resource": [
+                "arn:aws:dynamodb:${AWS_REGION}:${ACCOUNT_ID}:table/${CLUSTER_NAME}-events",
+                "arn:aws:dynamodb:${AWS_REGION}:${ACCOUNT_ID}:table/${CLUSTER_NAME}-events/index/*"
+            ]
+        }
+    ]
+}
+EOF
+
+cat > "$s3_policy" <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "BucketActions",
+            "Effect": "Allow",
+            "Action": [
+                "s3:PutEncryptionConfiguration",
+                "s3:PutBucketVersioning",
+                "s3:ListBucketVersions",
+                "s3:ListBucketMultipartUploads",
+                "s3:ListBucket",
+                "s3:GetEncryptionConfiguration",
+                "s3:GetBucketVersioning",
+                "s3:CreateBucket"
+            ],
+            "Resource": "arn:aws:s3:::${SESSION_BUCKET}"
+        },
+        {
+            "Sid": "ObjectActions",
+            "Effect": "Allow",
+            "Action": [
+                "s3:GetObjectVersion",
+                "s3:GetObjectRetention",
+                "s3:*Object",
+                "s3:ListMultipartUploadParts",
+                "s3:AbortMultipartUpload"
+            ],
+            "Resource": "arn:aws:s3:::${SESSION_BUCKET}/*"
+        }
+    ]
+}
+EOF
+
+
+cat > "$route53_policy" <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "route53:GetChange",
+            "Resource": "arn:aws:route53:::change/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "route53:ChangeResourceRecordSets",
+                "route53:ListResourceRecordSets"
+            ],
+            "Resource": "arn:aws:route53:::hostedzone/${ROUTE53_ZONE_ID}"
+        }
+    ]
+}
+EOF
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+# delete dynamo tables
+
+if [[ "$TELEPORT_BACKEND" == "dynamo" ]]; then
+    aws dynamodb delete-table \
+        --table-name "${CLUSTER_NAME}-backend"
+
+    aws dynamodb delete-table \
+        --table-name "${CLUSTER_NAME}-events"
+fi
+
+# empty the session bucket
+
+contents="$(aws s3 ls "$SESSION_BUCKET")"
+
+if ! test -z "$contents"; then
+    aws s3api delete-objects \
+      --bucket "$SESSION_BUCKET" \
+      --delete "$( \
+        aws s3api list-object-versions \
+        --bucket "$SESSION_BUCKET" \
+        --output=json \
+        --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}' \
+        )"
+fi
+
+# delete the session bucket
+
+aws s3api delete-bucket \
+    --bucket "$SESSION_BUCKET"
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+# update billing mode of tables
+
+aws dynamodb update-table \
+    --table-name "${CLUSTER_NAME}-backend" \
+    --billing-mode "PAY_PER_REQUEST" \
+    > /dev/null
+
+aws dynamodb update-table \
+    --table-name "${CLUSTER_NAME}-events" \
+    --billing-mode "PAY_PER_REQUEST" \
+    > /dev/null
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+values_yaml="$STATE_DIR/teleport-values.yaml"
+
+mkdir -p "$STATE_DIR"
+
+cat > "$values_yaml" <<EOF
+chartMode: aws
+clusterName: ${CLUSTER_NAME}.${ROUTE53_ZONE}      # Name of your cluster. Use the FQDN you intend to configure in DNS below.
+teleportVersionOverride: ${TELEPORT_VERSION}
+proxyListenerMode: "multiplex"
+aws:
+  region: ${AWS_REGION}                           # AWS region
+  backendTable: ${CLUSTER_NAME}-backend           # DynamoDB table to use for the Teleport backend
+  auditLogTable: ${CLUSTER_NAME}-events           # DynamoDB table to use for the Teleport audit log (must be different to the backend table)
+  auditLogMirrorOnStdout: false                   # Whether to mirror audit log entries to stdout in JSON format (useful for external log collectors)
+  sessionRecordingBucket: ${SESSION_BUCKET}       # S3 bucket to use for Teleport session recordings
+  backups: true                                   # Whether or not to turn on DynamoDB backups
+  dynamoAutoScaling: false                        # Whether Teleport should configure DynamoDB's autoscaling.
+highAvailability:
+  replicaCount: 2                                 # Number of replicas to configure
+  certManager:
+    enabled: true                                 # Enable cert-manager support to get TLS certificates
+    issuerName: letsencrypt-production            # Name of the cert-manager Issuer to use (as configured above)
+# If you are running Kubernetes 1.23 or above, disable PodSecurityPolicies
+podSecurityPolicy:
+  enabled: false
+podMonitor:
+  enabled: true
+auth:
+  teleportConfig:
+    kubernetes_service:
+      enabled: false
+EOF
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+values_yaml="$STATE_DIR/teleport-values.yaml"
+
+mkdir -p "$STATE_DIR"
+
+cat > "$values_yaml" <<EOF
+chartMode: standalone
+clusterName: ${CLUSTER_NAME}.${ROUTE53_ZONE}      # Name of your cluster. Use the FQDN you intend to configure in DNS below.
+teleportVersionOverride: ${TELEPORT_VERSION}
+
+extraArgs: ['--debug']
+
+persistence:
+    enabled: false
+
+highAvailability:
+  replicaCount: 2
+  certManager:
+    enabled: true
+    issuerName: letsencrypt-production
+
+authentication:
+  type: local
+  secondFactor: "optional"
+  webauthn:
+    rp_id: ${CLUSTER_NAME}.${ROUTE53_ZONE}
+  connector_name: passwordless
+  device_trust:
+    mode: "off"
+proxyListenerMode: separate
+auth:
+  teleportConfig:
+    version: v3
+    teleport:
+      log:
+        severity: DEBUG
+        format:
+          output: json
+      storage:
+        type: etcd
+        peers: [https://etcd.etcd.svc.cluster.local:2379]
+        insecure: true
+        prefix: teleport
+      connection_limits:
+        max_connections: 65000
+        max_users: 10000
+      advertise_ip: teleport-auth
+    auth_service:
+      enabled: true
+      cluster_name: ${CLUSTER_NAME}.${ROUTE53_ZONE}
+      listen_addr: 0.0.0.0:3025
+      routing_strategy: most_recent
+
+proxy:
+  annotations:
+    service:
+      service.beta.kubernetes.io/aws-load-balancer-type: nlb
+      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
+      service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
+  teleportConfig:
+    version: v3
+    teleport:
+      auth_server: "teleport-auth.teleport.svc.cluster.local:3025"
+      log:
+        severity: DEBUG
+        format:
+          output: json
+      connection_limits:
+        max_connections: 65000
+        max_users: 1000
+    proxy_service:
+      enabled: true
+      web_listen_addr: 0.0.0.0:3080
+      public_addr: ${CLUSTER_NAME}.${ROUTE53_ZONE}:443
+podSecurityPolicy:
+  enabled: false
+podMonitor:
+  enabled: true
+EOF
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+values_yaml="$STATE_DIR/teleport-values.yaml"
+
+helm upgrade --install teleport teleport/teleport-cluster \
+  --create-namespace \
+  --namespace teleport \
+  -f "$values_yaml"
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+helm --namespace teleport uninstall teleport
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source vars.env
+
+component="${1}"
+
+case "$component" in
+    auth)
+        ;;
+    proxy)
+        ;;
+    *)
+        echo "unknown component '$component'" >&2
+        exit 1
+        ;;
+esac
+
+kubectl wait pods \
+    --namespace teleport \
+    --timeout 10m \
+    --selector "app.kubernetes.io/component=${component}" \
+    --for "condition=Ready"
@@ -1,9 +1,33 @@
-# generates certificates
-.PHONY: all
-all:
-	make -C certs all
+# deploys etcd into the cluster
+.PHONY: deploy
+deploy:
+	kubectl create namespace etcd --dry-run=client -o yaml | kubectl apply -f -
+	kubectl create secret tls client-certs -n etcd \
+	  --cert=./certs/client-cert.pem \
+	  --key=./certs/client-key.pem

-# deletes generated certificates
-.PHONY: destroy
-destroy:
-	make -C certs clean
+	kubectl create secret tls server-certs -n etcd \
+  	  --cert=./certs/server-cert.pem \
+  	  --key=./certs/server-key.pem
+
+	kubectl create secret tls client-certs -n etcd \
+		  --cert=./certs/client-cert.pem \
+		  --key=./certs/client-key.pem
+
+	helm repo add bitnami https://charts.bitnami.com/bitnami
+	helm install etcd -n etcd --create-namespace bitnami/etcd \
+		--set replicaCount=3 \
+		--set metrics.enabled=true \
+		--set metrics.podMonitor.enabled=true \
+		--set auth.peer.secureTransport=true \
+		--set auth.peer.useAutoTLS=true \
+		--set auth.client.secureTransport=true \
+		--set auth.client.existingSecret=client-certs \
+		--set auth.client.enableAuthentication=true
+
+
+# deletes etcd from the cluster
+.PHONY: delete
+delete:
+	helm delete -n etcd etcd
+	kubectl delete ns etcd
@@ -1,14 +0,0 @@
-# etcd
-
-To generate self-signed certificates for secure connectivity to etcd run:
-
-```bash
-$ make
-```
-
-
-To delete the generated certificates run:
-
-```bash
-$ make clean
-```
@@ -6,4 +6,4 @@ type: application

 version: 0.1.0

-appVersion: "12.0.0-dev"
+appVersion: "13.0.0-alpha.1"
@@ -24,6 +24,9 @@ data:
      enabled: false
    ssh_service:
      enabled: true
+      commands:
+        - name: fullname
+          command: ['bash', '-c', 'echo "$HOSTNAME-$REPLICA"']
      # listen_addr set at runtime to avoid conflicts in the same pod
      # listen_addr: 0.0.0.0:3022
  entrypoint.sh: |2
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  namespace: agents
+  namespace: {{ .Release.Namespace }}
  name: {{ .Release.Name }}
 spec:
  replicas: {{ .Values.replicaCount }}
@@ -19,14 +19,6 @@ joinParams:
  # DO NOT USE THIS IN PRODUCTION
  token_name: qwertyuiop

-# Applied par agent (not per-pod)
-resources:
-  limits:
-    memory: 150Mi
-  requests:
-    cpu: 20m
-    memory: 150Mi
-
 tolerations: []

 affinity: {}
@@ -6,4 +6,4 @@ type: application

 version: 0.1.0

-appVersion: "12.0.0-dev"
+appVersion: "13.0.0-alpha.1"
@@ -0,0 +1,11 @@
+replicaCount: 30
+agentsPerPod: 15
+
+minReadySeconds: 30
+
+image:
+  tag: 13.0.0-alpha.2
+
+joinParams:
+  method: token
+  token_name: "qwertyuiop"
@@ -26,8 +26,7 @@ podMonitor:
  enabled: true
  interval: ""

-# TODO: change after v12 is released
-teleportVersionOverride: 12.0.0-hugochartsplit.2
+teleportVersionOverride: 13.0.0-alpha.1

 auth:
  teleportConfig:
@@ -1,499 +0,0 @@
-LICENSE_PATH ?= /var/lib/teleport/license.pem
-CERT_MANAGER_VERSION ?= v1.7.1
-SOAK_TEST_DURATION ?= 30m
-BACKEND ?= etcd
-USE_CERT_MANAGER ?= yes
-TELEPORT_IMAGE ?= public.ecr.aws/gravitational/teleport-ent:12.0.0
-NAMESPACE ?= loadtest
-NODE ?=
-
-# performs initialization needed for cluster
-# 1) generates etcd certs
-# 2) creates namespace
-# 3) installs cert-manager
-# 4) creates and applies secrets
-.PHONY: setup
-setup:
-	@if [ -z ${TELEPORT_IMAGE} ]; then \
-		echo "TELEPORT_IMAGE is not set, cannot apply cluster."; \
-		exit 1; \
-	fi
-
-
-	@echo "applying image: ${TELEPORT_IMAGE}"
-ifeq ($(BACKEND), etcd)
-	$(MAKE) -C ../etcd/certs all
-endif
-	kubectl create namespace $(NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
-	$(MAKE) -C ./secrets all
-ifeq ($(USE_CERT_MANAGER), yes)
-	kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
-endif
-	$(MAKE) generate-secrets
-
-# create kubernetes secrets
-.PHONY: generate-secrets
-generate-secrets:
-ifeq ($(BACKEND), etcd)
-	kubectl create secret generic etcd-client-certs -n $(NAMESPACE) \
-		--from-file=client-cert.pem=../etcd/certs/client-cert.pem \
-		--from-file=client-key.pem=../etcd/certs/client-key.pem \
-		--from-file=ca-cert.pem=../etcd/certs/ca-cert.pem \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	kubectl create secret generic etcd-server-certs -n $(NAMESPACE) \
-		--from-file=server-cert.pem=../etcd/certs/server-cert.pem \
-		--from-file=server-key.pem=../etcd/certs/server-key.pem \
-		--from-file=ca-cert.pem=../etcd/certs/ca-cert.pem \
-		--dry-run=client -o yaml | kubectl apply -f -
-endif
-
-ifeq ($(BACKEND), firestore)
-	kubectl create secret generic gcp-creds -n $(NAMESPACE) \
-		--from-file=gcp_creds.json=${GCP_CREDS_LOCATION} \
-		--dry-run=client -o yaml | kubectl apply -f -
-endif
-
-	kubectl create secret generic license -n $(NAMESPACE) \
-		--from-file=license.pem=$(LICENSE_PATH) \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-# deletes the loadtest and cert-manager namespaces
-.PHONY: clean
-clean:
-	$(MAKE) -C secrets clean
-	$(MAKE) -C ../etcd/certs clean
-	kubectl delete namespace $(NAMESPACE) --ignore-not-found
-	kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml --ignore-not-found
-
-
-ifeq ($(BACKEND), etcd)
-# deploys etcd and teleport to the loadtest namespace
-.PHONY: apply
-apply: setup install-etcd generate-certificates install-monitor install-teleport
-else
-# deploys and teleport to the loadtest namespace
-.PHONY: apply
-apply: setup generate-certificates install-monitor install-teleport
-endif
-
-ifeq ($(USE_CERT_MANAGER), yes)
-# generate-certificates applies a cert-manager.io/v1/ClusterIssuer and a cert-manager.io/v1/Certificate
-# that will automatically fetch tls certificates
-.PHONY: generate-certificates
-generate-certificates:
-	kubectl wait --for=condition=available --timeout=600s deploy cert-manager -n cert-manager
-	kubectl wait --for=condition=available --timeout=600s deploy cert-manager-webhook -n cert-manager
-	kubectl wait --for=condition=available --timeout=600s deploy cert-manager-cainjector -n cert-manager
-# we have to sleep here to due to issues with certmanager
-# Error from server (InternalError): error when creating "certificate.yaml": Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority
-# Error from server (InternalError): error when creating "certificate.yaml": Internal error occurred: failed calling webhook "webhook.cert-manager.io": Post "https://cert-manager-webhook.cert-manager.svc:443/mutate?timeout=10s": x509: certificate signed by unknown authority
-# See https://github.com/jetstack/cert-manager/issues/2602 and https://github.com/jetstack/cert-manager/issues/2752
-	@echo "waiting for cert-manager to be ready..."
-	@sleep 90
-	kubectl apply -f certificate.yaml
-else
-.PHONY: generate-certificates
-generate-certificates:
-	kubectl apply -f tls.yaml
-endif
-
-# installs teleport auth, proxy, one IoT node and one non-IoT node
-.PHONY: install-teleport
-install-teleport: install-auth install-proxy install-node install-iot-node
-
-# deletes teleport deployments, services, and configmaps
-.PHONY: delete-teleport
-delete-teleport: delete-tc delete-nodes delete-proxy delete-auth
-
-# installs prometheus exporter
-.PHONY: install-monitor
-install-monitor:
-	$(MAKE) -s expand-yaml FILENAME=prometheus NAMESPACE=$(NAMESPACE)
-	kubectl apply -f prometheus-gen.yaml
-
-# deletes prometheus exporter
-.PHONY: delete-monitor
-delete-monitor:
-	kubectl delete -f prometheus-gen.yaml --ignore-not-found
-
-# installs an etcd cluster
-.PHONY: install-etcd
-install-etcd:
-	kubectl create configmap etcd-config -n $(NAMESPACE) \
-		--from-file=etcd.sh=./etcd.sh \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-
-	$(MAKE) -s expand-yaml FILENAME=etcd NAMESPACE=$(NAMESPACE)
-	kubectl apply -f etcd-gen.yaml
-
-# deletes etcd deployment, services, and configmaps
-.PHONY: delete-etcd
-delete-etcd:
-	kubectl delete -f etcd-gen.yaml --ignore-not-found
-	kubectl delete configmap etcd-config -n $(NAMESPACE) --ignore-not-found
-
-
-# install auth and applies required teleport resources for loadtests
-.PHONY: install-auth
-install-auth: setup-auth
-	kubectl wait --for=condition=ready pod -l teleport-role=auth -n $(NAMESPACE) --timeout=120s
-
-	kubectl -n $(NAMESPACE) exec deploy/auth -c teleport -it \
-		-- tctl --config /etc/teleport/teleport.yaml create -f /etc/teleport/admin.yaml
-	kubectl -n $(NAMESPACE) exec deploy/auth -c teleport -it \
-		-- tctl --config /etc/teleport/teleport.yaml create -f /etc/teleport/oidc.yaml
-	kubectl -n $(NAMESPACE) exec deploy/auth -c teleport -it \
-		-- tctl --config /etc/teleport/teleport.yaml create -f /etc/teleport/user.yaml
-
-
-ifeq ($(BACKEND), etcd)
-.PHONY: setup-auth
-setup-auth:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/teleport-auth-etcd
-
-	kubectl create configmap auth-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-auth-etcd-gen.yaml \
-		--from-file=oidc.yaml=../teleport/oidc.yaml \
-		--from-file=admin.yaml=../teleport/admin.yaml \
-		--from-file=user.yaml=../teleport/soaktest-user.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=auth-etcd TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f auth-etcd-gen.yaml
-else ifeq ($(BACKEND), firestore)
-.PHONY: setup-auth
-setup-auth:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/teleport-auth-firestore
-
-	kubectl create configmap auth-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-auth-firestore-gen.yaml \
-		--from-file=oidc.yaml=../teleport/oidc.yaml \
-		--from-file=admin.yaml=../teleport/admin.yaml \
-		--from-file=user.yaml=../teleport/soaktest-user.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=auth-firestore TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f auth-firestore-gen.yaml
-else ifeq ($(BACKEND), dynamo)
-.PHONY: setup-auth
-setup-auth:
-	$(MAKE) -s expand-yaml DYNAMO_TABLE=${DYNAMO_TABLE} DYNAMO_REGION=${DYNAMO_REGION} FILENAME=../teleport/teleport-auth-dynamo
-
-	kubectl create configmap auth-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-auth-dynamo-gen.yaml \
-		--from-file=oidc.yaml=../teleport/oidc.yaml \
-		--from-file=admin.yaml=../teleport/admin.yaml \
-		--from-file=user.yaml=../teleport/soaktest-user.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=auth-dynamo TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f auth-dynamo-gen.yaml
-else
-.PHONY: setup-auth
-setup-auth:
-	@echo "unknown backend $(BACKEND)"
-	exit 1
-endif
-
-# deletes auth deployment, services and configmaps
-.PHONY: delete-auth
-delete-auth:
-	kubectl delete -f auth-etcd-gen.yaml --ignore-not-found
-	kubectl delete -f auth-dynamo-gen.yaml --ignore-not-found
-	kubectl delete -f auth-firestore-gen.yaml --ignore-not-found
-	kubectl delete configmap auth-config -n $(NAMESPACE) --ignore-not-found
-
-# install proxy
-.PHONY: install-proxy
-install-proxy:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/teleport-proxy
-	kubectl create configmap proxy-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-proxy-gen.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-
-	$(MAKE) -s expand-yaml FILENAME=proxy TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f proxy-gen.yaml
-
-# deletes proxy deployment, services and configmaps
-.PHONY: delete-proxy
-delete-proxy:
-	kubectl delete -f proxy-gen.yaml --ignore-not-found
-	kubectl delete configmap proxy-config -n $(NAMESPACE) --ignore-not-found
-
-# deletes all node deployment and configmaps
-.PHONY: delete-nodes
-delete-nodes: delete-node delete-iot-node
-
-# deletes all non-IoT nodes
-.PHONY: delete-node
-delete-node:
-	kubectl delete -f node-gen.yaml --ignore-not-found
-	kubectl delete configmap node-config -n $(NAMESPACE) --ignore-not-found
-
-# deletes all IoT nodes
-.PHONY: delete-iot-node
-delete-iot-node:
-	kubectl delete -f iot-node-gen.yaml --ignore-not-found
-	kubectl delete configmap iot-node-config -n $(NAMESPACE) --ignore-not-found
-
-# install one IoT node and one non-IoT node
-.PHONY: install-nodes
-install-nodes: install-iot-node install-node
-
-# install an IoT mode node
-.PHONY: install-iot-node
-install-iot-node:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/teleport-iot-node
-	kubectl create configmap iot-node-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-iot-node-gen.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=iot-node TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f iot-node-gen.yaml
-
-# install a non-IoT mode node
-.PHONY: install-node
-install-node:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/teleport-node
-	kubectl create configmap node-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-node-gen.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=node TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f node-gen.yaml
-
-# installs a trusted cluster
-.PHONY: install-tc
-install-tc:
-	$(MAKE) -s expand-yaml FILENAME=../teleport/tc
-	kubectl create configmap tc-config -n $(NAMESPACE) \
-		--from-file=teleport.yaml=../teleport/teleport-tc.yaml \
-		--from-file=cluster.yaml=../teleport/tc-gen.yaml \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=tc TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE) NAMESPACE=$(NAMESPACE)
-	kubectl apply -f tc-gen.yaml
-
-# deletes all rc resources from teleport and deletes trusted cluster deployments and configmaps
-.PHONY: delete-tc
-delete-tc:
-	kubectl delete -f tc-gen.yaml --ignore-not-found
-	kubectl delete configmap tc-config -n $(NAMESPACE) --ignore-not-found
-
-	kubectl -n $(NAMESPACE) exec deploy/auth -c teleport -it \
-		-- /bin/bash -c "tctl --config /etc/teleport/teleport.yaml get rc | grep ' name:' | cut -d ':' -f2- | xargs -P 20 -n 1 -I {} tctl --config /etc/teleport/teleport.yaml rm rc/{}"
-
-# joins all trusted clusters to root cluster
-.PHONY: setup-tc
-setup-tc:
-	kubectl get pod -n $(NAMESPACE) -l app="tc" -o custom-columns=name:metadata.name --no-headers \
-	| xargs -P 20 -n 1 -I {} kubectl -n $(NAMESPACE) exec {} -- tctl --config /etc/teleport/teleport.yaml create -f /etc/teleport/cluster.yaml
-
-# scales trusted clusters to 500
-.PHONY: scale-tc-500
-scale-tc-500:
-	kubectl scale --replicas=500 deploy tc -n $(NAMESPACE)
-
-# scales trusted clusters to 1
-.PHONY: scale-tc-1
-scale-tc-1:
-	kubectl scale --replicas=1 deploy tc -n $(NAMESPACE)
-
-# scales nodes to 1
-.PHONY: scale-1-non-iot
-scale-1-non-iot:
-	kubectl scale --replicas=1 deploy node -n $(NAMESPACE)
-
-# scales nodes to 1000
-.PHONY: scale-1k-non-iot
-scale-1k-non-iot:
-	kubectl scale --replicas=1000 deploy node -n $(NAMESPACE)
-
-# scales nodes to 10000
-.PHONY: scale-10k-non-iot
-scale-10k-non-iot:
-	kubectl scale --replicas=10000 deploy node -n $(NAMESPACE)
-
-# scales nodes to 1
-.PHONY: scale-1-iot
-scale-1-iot:
-	kubectl scale --replicas=1 deploy iot-node -n $(NAMESPACE)
-
-# scales nodes to 1000
-.PHONY: scale-1k-iot
-scale-1k-iot:
-	kubectl scale --replicas=1000 deploy iot-node -n $(NAMESPACE)
-
-# scales nodes to 10000
-.PHONY: scale-10k-iot
-scale-10k-iot:
-	kubectl scale --replicas=10000 deploy iot-node -n $(NAMESPACE)
-
-# gets pods in loadtest namespace
-.PHONY: pods
-pods:
-	kubectl get pods -n $(NAMESPACE)
-
-# removes all soak test jobs and configmaps
-.PHONY: delete-soaktest
-.PHONY: delete-soaktest
-delete-soaktest:
-	kubectl delete job -l app=soaktest -n $(NAMESPACE) --ignore-not-found
-
-	kubectl delete configmap soaktest-config -n $(NAMESPACE) --ignore-not-found
-
-# creates the soak test job
-.PHONY: install-soaktest
-install-soaktest:
-	kubectl create configmap soaktest-config -n $(NAMESPACE) \
-		--from-file=soaktest.sh=../teleport/soaktest.sh \
-		--from-literal=DURATION=$(SOAK_TEST_DURATION) \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	kubectl create secret generic soaktest -n $(NAMESPACE) \
-		--from-file=auth=./secrets/soaktest-auth \
-		--from-literal=PROXY_HOST=$$(cat ./secrets/secrets.env | grep PROXY_HOST | cut -d '=' -f2- ) \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-	$(MAKE) -s expand-yaml FILENAME=soaktest TELEPORT_IMAGE=$(TELEPORT_IMAGE) NAMESPACE=$(NAMESPACE)
-	kubectl create -f soaktest-gen.yaml
-
-# deploys a job to run the soak tests
-.PHONY: run-soak-tests
-run-soak-tests:
-	kubectl -n $(NAMESPACE) exec $$(kubectl get pod -n $(NAMESPACE) -l teleport-role="auth" -o jsonpath="{.items[0].metadata.name}") -it \
-		-- tctl auth sign --overwrite --user=soaktest-runner --out=/data/soaktest-auth --ttl=8760h --config /etc/teleport/teleport.yaml
-
-	kubectl cp $(NAMESPACE)/$$(kubectl get pod -n $(NAMESPACE) -l teleport-role="auth" -o jsonpath="{.items[0].metadata.name}"):/data/soaktest-auth ./secrets/soaktest-auth
-
-	kubectl wait --for=condition=available --timeout=600s deploy/node -n $(NAMESPACE)
-	kubectl wait --for=condition=available --timeout=600s deploy/iot-node -n $(NAMESPACE)
-
-	$(MAKE) -s install-soaktest
-
-	@sleep 1
-
-	kubectl wait --for=condition=ready pod $$(kubectl get pods --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}" -l app=soaktest -n $(NAMESPACE)) -n $(NAMESPACE) --timeout=120s
-	kubectl logs $$(kubectl get pods --sort-by=.metadata.creationTimestamp -o jsonpath="{.items[-1:].metadata.name}" -l app=soaktest -n $(NAMESPACE)) -n $(NAMESPACE) --tail -1 -f
-
-# runs the node scaling tests
-.PHONY: run-scaling-test
-run-scaling-test:
-	$(MAKE) -s delete-nodes
-	$(MAKE) -s install-node
-	$(MAKE) -s scale-10k-non-iot
-	@kubectl wait --for=condition=available deploy/node -n $(NAMESPACE) --timeout=60m
-	@sleep 30
-	$(MAKE) -s scale-1-non-iot
-	@sleep 15
-	$(MAKE) -s scale-10k-non-iot
-	@kubectl wait --for=condition=available deploy/node -n $(NAMESPACE) --timeout=60m
-	@sleep 15
-	$(MAKE) -s scale-1-non-iot
-
-	@sleep 15
-
-	$(MAKE) -s delete-nodes
-	$(MAKE) -s install-iot-node
-	$(MAKE) -s scale-10k-iot
-	@kubectl wait --for=condition=available deploy/iot-node -n $(NAMESPACE) --timeout=60m
-	@sleep 30
-	$(MAKE) -s scale-1-iot
-	@sleep 15
-	$(MAKE) -s scale-10k-iot
-	@kubectl wait --for=condition=available  deploy/iot-node -n $(NAMESPACE) --timeout=60m
-	@sleep 15
-	$(MAKE) -s scale-1
-
-	$(MAKE) -s delete-nodes
-	$(MAKE) -s install-nodes
-
-# runs the trusted-cluster scaling tests
-.PHONY: run-tc-scaling-test
-run-tc-scaling-test:
-	$(MAKE) -s install-tc
-	$(MAKE) -s scale-tc-500
-	kubectl wait --for=condition=available deploy/tc -n $(NAMESPACE) --timeout=60m
-	@sleep 120
-	$(MAKE) -s setup-tc
-
-	@sleep 300
-
-	$(MAKE) -s delete-tc
-
-	@sleep 180
-
-	$(MAKE) -s install-tc
-	$(MAKE) -s scale-tc-500
-	kubectl wait --for=condition=available deploy/tc -n $(NAMESPACE) --timeout=60m
-	@sleep 120
-	$(MAKE) -s setup-tc
-
-	@sleep 300
-
-	$(MAKE) -s delete-tc
-
-# collect goroutine and heap go profiles from the auth deployment
-.PHONY: collect-profiles
-collect-profiles: collect-auth collect-proxy collect-node
-
-# forwards the pods diagnostics address
-.PHONY: forward-pod-diagnostics
-forward-pod-diagnostics:
-	kubectl port-forward $(POD) 3434:3434 -n $(NAMESPACE) > /dev/null 2>&1 &
-
-# retrieves profiles from all pods matching the selector
-.PHONY: collect-pods
-collect-pods:
-	@for POD in $$(kubectl get pod -n $(NAMESPACE) -l $(SELECTOR) -o custom-columns=name:metadata.name --no-headers) ; do \
-$(MAKE) collect-pod POD=$$POD ; \
-    done
-
-# retrieves profiles from all proxy pods
-.PHONY: collect-proxy
-collect-proxy:
-	$(MAKE) -s collect-pods SELECTOR=teleport-role="proxy"
-
-# retrieves profiles from all auth pods
-.PHONY: collect-auth
-collect-auth:
-	$(MAKE) -s collect-pods SELECTOR=teleport-role="auth"
-
-# retrieves profiles from a node
-.PHONY: collect-node
-collect-node:
-	$(MAKE) -s collect-pod POD=$(NODE)
-
-# retrieves profiles from a particular pod
-ifeq ($(POD),)
-.PHONY: collect-pod
-collect-pod:
-	@echo "No pod provided; $(MAKE) collect-pod POD=<pod-name>"
-	exit 1
-else
-.PHONY: collect-pod
-collect-pod:
-	$(MAKE) -s forward-pod-diagnostics POD=$(POD)
-	@echo "waiting for $(POD) to be available..."
-	@timeout 30 sh -c 'until nc -z localhost 3434; do sleep 0.5; done'
-	@if [ $$? -ne 0 ]; then echo "unable to reach $$POD"; exit 1; fi
-	$(MAKE) -s fetch-profiles LOCATION=$(POD)-$(shell date +%s)
-	kill -s kill $$(pgrep -f 3434:3434)
-endif
-
-# downloads the remote profiles
-.PHONY: fetch-profiles
-fetch-profiles:
-	mkdir -p $(PWD)/profiles/$(LOCATION)/
-	curl -o $(PWD)/profiles/$(LOCATION)/goroutine.profile http://127.0.0.1:3434/debug/pprof/goroutine
-	curl -o $(PWD)/profiles/$(LOCATION)/heap.profile http://127.0.0.1:3434/debug/pprof/heap
-	curl -o $(PWD)/profiles/$(LOCATION)/trace.profile http://127.0.0.1:3434/debug/pprof/trace?seconds=10
-
-# expands any placeholders in the provided yaml file with the value in the matching environment variable. the
-# output file will be named the same with a -gen suffix, i.e input = test then output will be test-gen.yaml
-.PHONY: expand-yaml
-expand-yaml:
-	@bash -c "set -a && source ./secrets/secrets.env && set +a && envsubst < $(FILENAME).yaml > $(FILENAME)-gen.yaml"
@@ -1,83 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-  labels:
-    teleport-role: auth
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      teleport-role: auth
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "3434"
-      labels:
-        backend: dynamo
-        teleport-role: auth
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: auth-config
-        - name: license
-          secret:
-            secretName: license
-        - name: storage
-          emptyDir: {}
-      containers:
-        - name: teleport
-          image: ${TELEPORT_IMAGE}
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - name: config
-              mountPath: /etc/teleport/
-              readOnly: true
-            - name: license
-              mountPath: /var/lib/teleport/license.pem
-              subPath: license.pem
-              readOnly: true
-            - mountPath: /data
-              name: storage
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-spec:
-  ports:
-    - name: auth
-      port: 3025
-      targetPort: 3025
-    - name: diag
-      port: 3434
-      targetPort: 3434
-  selector:
-    teleport-role: auth
-  type: ClusterIP
@@ -1,91 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-  labels:
-    teleport-role: auth
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      teleport-role: auth
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "3434"
-      labels:
-        teleport-role: auth
-        backend: etcd
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "3434"
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: auth-config
-        - name: certs
-          secret:
-            secretName: etcd-client-certs
-        - name: license
-          secret:
-            secretName: license
-        - name: storage
-          emptyDir: {}
-      containers:
-        - name: teleport
-          image: ${TELEPORT_IMAGE}
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - name: config
-              mountPath: /etc/teleport/
-              readOnly: true
-            - name: license
-              mountPath: /var/lib/teleport/license.pem
-              subPath: license.pem
-              readOnly: true
-            - name: certs
-              mountPath: /etc/etcd/certs/
-              readOnly: true
-            - mountPath: /data
-              name: storage
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-spec:
-  ports:
-    - name: auth
-      port: 3025
-      targetPort: 3025
-    - name: diag
-      port: 3434
-      targetPort: 3434
-  selector:
-    teleport-role: auth
-  type: ClusterIP
@@ -1,92 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-  labels:
-    teleport-role: auth
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      teleport-role: auth
-  template:
-  metadata:
-    annotations:
-      prometheus.io/scrape: "true"
-      prometheus.io/port: "3434"
-    labels:
-      teleport-role: auth
-      backend: firestore
-      prometheus.io/scrape: "true"
-      prometheus.io/port: "3434"
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: auth-config
-        - name: creds
-          secret:
-            secretName: gcp-creds
-        - name: license
-          secret:
-            secretName: license
-        - name: storage
-          emptyDir: {}
-      containers:
-        - name: teleport
-          image: ${TELEPORT_IMAGE}
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - name: config
-              mountPath: /etc/teleport/
-              readOnly: true
-            - name: license
-              mountPath: /var/lib/teleport/license.pem
-              subPath: license.pem
-              readOnly: true
-            - name: creds
-              mountPath: /var/lib/teleport/gcp_creds.json
-              subPath: gcp_creds.json
-              readOnly: true
-            - mountPath: /data
-              name: storage
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: auth
-  namespace: ${NAMESPACE}
-spec:
-  ports:
-    - name: auth
-      port: 3025
-      targetPort: 3025
-    - name: diag
-      port: 3434
-      targetPort: 3434
-  selector:
-    teleport-role: auth
-  type: ClusterIP
@@ -1,22 +0,0 @@
-#!/bin/sh
-# This script runs etcd.
-set -e
-set -x
-
-PEERS="etcd-0=https://etcd-0.etcd:2380,etcd-1=https://etcd-1.etcd:2380,etcd-2=https://etcd-2.etcd:2380"
-exec etcd \
-  --name ${POD_NAME} \
-  --advertise-client-urls https://${POD_NAME}.etcd:2379 \
-  --listen-client-urls https://0.0.0.0:2379 \
-  --initial-advertise-peer-urls https://${POD_NAME}.etcd:2380 \
-  --listen-peer-urls https://0.0.0.0:2380 \
-  --initial-cluster ${PEERS} \
-  --trusted-ca-file=/etc/etcd/certs/ca-cert.pem \
-  --cert-file=/etc/etcd/certs/server-cert.pem \
-  --key-file=/etc/etcd/certs/server-key.pem \
-  --peer-cert-file=/etc/etcd/certs/server-cert.pem \
-  --peer-key-file=/etc/etcd/certs/server-key.pem \
-  --peer-trusted-ca-file=/etc/etcd/certs/ca-cert.pem \
-  --client-cert-auth \
-  --peer-client-cert-auth \
-  --auto-compaction-retention=1
@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: etcd
-  namespace: ${NAMESPACE}
-  labels:
-    app: etcd
-spec:
-  serviceName: etcd
-  replicas: 3
-  selector:
-    matchLabels:
-      app: etcd
-  template:
-    metadata:
-      name: etcd
-      labels:
-        app: etcd
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: etcd-config
-            defaultMode: 0777
-        - name: server-certs
-          secret:
-            secretName: etcd-server-certs
-        - name: client-certs
-          secret:
-            secretName: etcd-client-certs
-      containers:
-        - name: etcd
-          image: quay.io/coreos/etcd:v3.3.25
-          ports:
-            - containerPort: 2379
-              name: client
-            - containerPort: 2380
-              name: peer
-          volumeMounts:
-            - mountPath: /scripts
-              name: config
-              readOnly: true
-            - name: server-certs
-              mountPath: /etc/etcd/certs/
-              readOnly: true
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-          command:
-            - /bin/sh
-            - -c
-            - |
-              cp /scripts/etcd.sh /tmp
-              chmod +x /tmp/etcd.sh
-              /tmp/etcd.sh
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd
-  namespace: ${NAMESPACE}
-  labels:
-    app: etcd
-spec:
-  clusterIP: None
-  ports:
-    - port: 2379
-      name: client
-    - port: 2380
-      name: peer
-  selector:
-    app: etcd
@@ -1,56 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    teleport-role: node
-  name: iot-node
-  namespace: ${NAMESPACE}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      teleport-role: node
-      node: iot
-  template:
-    metadata:
-      labels:
-        teleport-role: node
-        node: iot
-    spec:
-      containers:
-        - image: ${TELEPORT_IMAGE}
-          name: teleport
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: nodessh
-              containerPort: 3022
-              protocol: TCP
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - mountPath: /etc/teleport
-              name: config
-              readOnly: true
-      volumes:
-        - configMap:
-            name: iot-node-config
-          name: config
@@ -1,56 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    teleport-role: node
-  name: node
-  namespace: ${NAMESPACE}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      teleport-role: node
-      node: regular
-  template:
-    metadata:
-      labels:
-        teleport-role: node
-        node: regular
-    spec:
-      containers:
-        - image: ${TELEPORT_IMAGE}
-          name: teleport
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: nodessh
-              containerPort: 3022
-              protocol: TCP
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - mountPath: /etc/teleport
-              name: config
-              readOnly: true
-      volumes:
-        - configMap:
-            name: node-config
-          name: config
@@ -1,291 +0,0 @@
---
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: prometheus-loadtest
---
-# Source: prometheus/templates/server/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-  namespace: prometheus-loadtest
-  annotations:
-    {}
---
-# Source: prometheus/templates/server/cm.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-  namespace: prometheus-loadtest
-data:
-  alerting_rules.yml: |
-    {}
-  alerts: |
-    {}
-  prometheus.yml: |
-    global:
-      evaluation_interval: 1m
-      scrape_interval: 1m
-      scrape_timeout: 10s
-    remote_write:
-    - url: ${PROM_REMOTE_URL}
-      basic_auth:
-        username: ${PROM_USER}
-        password: ${PROM_PASSWORD}
-    rule_files:
-    - /etc/config/recording_rules.yml
-    - /etc/config/alerting_rules.yml
-    - /etc/config/rules
-    - /etc/config/alerts
-    scrape_configs:
-    - job_name: kubernetes-pods
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - action: keep
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape
-      - action: drop
-        regex: true
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow
-      - action: replace
-        regex: (https?)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_scheme
-        target_label: __scheme__
-      - action: replace
-        regex: (.+)
-        source_labels:
-        - __meta_kubernetes_pod_annotation_prometheus_io_path
-        target_label: __metrics_path__
-      - action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        source_labels:
-        - __address__
-        - __meta_kubernetes_pod_annotation_prometheus_io_port
-        target_label: __address__
-      - action: labelmap
-        regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-        replacement: __param_$1
-      - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_namespace
-        target_label: namespace
-      - action: replace
-        source_labels:
-        - __meta_kubernetes_pod_name
-        target_label: pod
-      - action: drop
-        regex: Pending|Succeeded|Failed|Completed
-        source_labels:
-        - __meta_kubernetes_pod_phase
-  recording_rules.yml: |
-    {}
-  rules: |
-    {}
---
-# Source: prometheus/templates/server/clusterrole.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-      - nodes/proxy
-      - nodes/metrics
-      - services
-      - endpoints
-      - pods
-      - ingresses
-      - configmaps
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - "extensions"
-      - "networking.k8s.io"
-    resources:
-      - ingresses/status
-      - ingresses
-    verbs:
-      - get
-      - list
-      - watch
-  - nonResourceURLs:
-      - "/metrics"
-    verbs:
-      - get
---
-# Source: prometheus/templates/server/clusterrolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-subjects:
-  - kind: ServiceAccount
-    name: prometheus-server
-    namespace: prometheus-loadtest
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: prometheus-server
---
-# Source: prometheus/templates/server/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-  namespace: prometheus-loadtest
-spec:
-  ports:
-    - name: http
-      port: 80
-      protocol: TCP
-      targetPort: 9090
-  selector:
-    component: "server"
-    app: prometheus
-    release: prometheus
-  sessionAffinity: None
-  type: "ClusterIP"
---
-# Source: prometheus/templates/server/deploy.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    component: "server"
-    app: prometheus
-    release: prometheus
-    chart: prometheus-15.4.0
-    heritage: Helm
-  name: prometheus-server
-  namespace: prometheus-loadtest
-spec:
-  selector:
-    matchLabels:
-      component: "server"
-      app: prometheus
-      release: prometheus
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        component: "server"
-        app: prometheus
-        release: prometheus
-        chart: prometheus-15.4.0
-        heritage: Helm
-    spec:
-      enableServiceLinks: true
-      serviceAccountName: prometheus-server
-      containers:
-        - name: prometheus-server-configmap-reload
-          image: "jimmidyson/configmap-reload:v0.5.0"
-          imagePullPolicy: "IfNotPresent"
-          args:
-            - --volume-dir=/etc/config
-            - --webhook-url=http://127.0.0.1:9090/-/reload
-          resources:
-            {}
-          volumeMounts:
-            - name: config-volume
-              mountPath: /etc/config
-              readOnly: true
-
-        - name: prometheus-server
-          image: "quay.io/prometheus/prometheus:v2.31.1"
-          imagePullPolicy: "IfNotPresent"
-          args:
-            - --storage.tsdb.retention.time=15d
-            - --config.file=/etc/config/prometheus.yml
-            - --storage.tsdb.path=/data
-            - --web.console.libraries=/etc/prometheus/console_libraries
-            - --web.console.templates=/etc/prometheus/consoles
-            - --web.enable-lifecycle
-          ports:
-            - containerPort: 9090
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9090
-              scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            timeoutSeconds: 4
-            failureThreshold: 3
-            successThreshold: 1
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: 9090
-              scheme: HTTP
-            initialDelaySeconds: 30
-            periodSeconds: 15
-            timeoutSeconds: 10
-            failureThreshold: 3
-            successThreshold: 1
-          resources:
-            {}
-          volumeMounts:
-            - name: config-volume
-              mountPath: /etc/config
-            - name: storage-volume
-              mountPath: /data
-              subPath: ""
-      hostNetwork: false
-      dnsPolicy: ClusterFirst
-      securityContext:
-        fsGroup: 65534
-        runAsGroup: 65534
-        runAsNonRoot: true
-        runAsUser: 65534
-      terminationGracePeriodSeconds: 300
-      volumes:
-        - name: config-volume
-          configMap:
-            name: prometheus-server
-        - name: storage-volume
-          emptyDir:
-            {}
@@ -1,102 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: proxy
-  namespace: ${NAMESPACE}
-  labels:
-    teleport-role: proxy
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      teleport-role: proxy
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "3434"
-      labels:
-        teleport-role: proxy
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "3434"
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: proxy-config
-        - name: teleport-tls
-          secret:
-            secretName: teleport-tls
-      containers:
-        - name: teleport
-          image: ${TELEPORT_IMAGE}
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          ports:
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-            - name: web
-              containerPort: 3080
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - name: config
-              mountPath: /etc/teleport/
-              readOnly: true
-            - name: teleport-tls
-              mountPath: /etc/teleport-tls/
-              readOnly: true
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: proxy
-  namespace: ${NAMESPACE}
-  labels:
-    teleport-role: proxy
-spec:
-  type: LoadBalancer
-  loadBalancerIP: ${PROXY_IP}
-  ports:
-  - name: https
-    port: 3080
-    targetPort: 3080
-    protocol: TCP
-  - name: sshproxy
-    port: 3023
-    targetPort: 3023
-    protocol: TCP
-  - name: k8s
-    port: 3026
-    targetPort: 3026
-    protocol: TCP
-  - name: sshtun
-    port: 3024
-    targetPort: 3024
-    protocol: TCP
-  - name: mysql
-    port: 3036
-    targetPort: 3036
-    protocol: TCP
-  - name: diag
-    port: 3434
-    targetPort: 3434
-  selector:
-    teleport-role: proxy
@@ -1,58 +0,0 @@
-
-# output:
-#   node-token      : static join token for teleport nodes
-#   proxy-token     : static join token for teleport proxies
-#   tc-token        : static join token for teleport trusted clusters
-#   secrets.env     : env file to be used to replace placeholder values in yaml files
-.PHONY:all
-all: join-tokens env
-	@rm -rf *csr
-
-.PHONY: env
-env:
-	@if [ -z ${PROXY_HOST} ]; then \
-		echo "PROXY_HOST is not set, cannot apply cluster."; \
-		exit 1; \
-	fi
-
-	@if [ -z ${PROM_REMOTE_URL} ]; then \
-		echo "PROM_REMOTE_URL is not set, cannot apply cluster."; \
-		exit 1; \
-	fi
-
-	@if [ -z ${PROM_USER} ]; then \
-		echo "PROM_USER is not set, cannot apply cluster."; \
-		exit 1; \
-	fi
-
-	@if [ -z ${PROM_PASSWORD} ]; then \
-		echo "PROM_PASSWORD is not set, cannot apply cluster."; \
-		exit 1; \
-	fi
-
-	@echo PROXY_IP=$(shell make -C ../../network get-proxy-ip) > secrets.env
-	@echo PROXY_HOST=${PROXY_HOST} >> secrets.env
-	@echo NODE_TOKEN=$(shell cat node-token) >> secrets.env
-	@echo PROXY_TOKEN=$(shell cat proxy-token) >> secrets.env
-	@echo TC_TOKEN=$(shell cat tc-token) >> secrets.env
-	@echo GCP_PROJECT=$(shell make -C ../../cluster get-project) >> secrets.env
-	@echo PROM_REMOTE_URL=${PROM_REMOTE_URL} >> secrets.env
-	@echo PROM_USER=${PROM_USER} >> secrets.env
-	@echo PROM_PASSWORD=${PROM_PASSWORD} >> secrets.env
-
-node-token:
-	openssl rand -base64 32 | tr -d '\n' > node-token
-
-proxy-token:
-	openssl rand -base64 32 | tr -d '\n' > proxy-token
-
-tc-token:
-	openssl rand -base64 32 | tr -d '\n' > tc-token
-
-.PHONY: join-tokens
-join-tokens: node-token proxy-token tc-token
-
-# removes everything
-.PHONY:clean
-clean:
-	rm -rf *-pass *-token *-auth *.env
@@ -1,47 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  generateName: soaktest-
-  namespace: ${NAMESPACE}
-  labels:
-    app: soaktest
-spec:      
-  completions: 1
-  parallelism: 1
-  backoffLimit: 2
-  template:
-    metadata:
-      labels:
-        app: soaktest
-    spec:
-      restartPolicy: Never
-      volumes:
-        - name: config
-          configMap:
-            name: soaktest-config
-            defaultMode: 0777
-        - name: soaktest-auth
-          secret:
-            secretName: soaktest
-      containers:
-        - image: ${TELEPORT_IMAGE}
-          name: teleport
-          envFrom:
-          - configMapRef:
-              name: soaktest-config
-          - secretRef:
-              name: soaktest
-          command:
-            - /bin/sh
-            - -c
-            - |
-              cp /scripts/soaktest.sh /tmp
-              chmod +x /tmp/soaktest.sh
-              /tmp/soaktest.sh
-          volumeMounts:
-            - mountPath: /scripts
-              name: config
-              readOnly: true
-            - mountPath: /etc/teleport
-              name: soaktest-auth
-              readOnly: true
@@ -1,61 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: tc
-  name: tc
-  namespace: ${NAMESPACE}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: tc
-  template:
-    metadata:
-      labels:
-        app: tc
-    spec:
-      volumes:
-        - name: config
-          configMap:
-            name: tc-config
-        - name: license
-          secret:
-            secretName: license
-      containers:
-        - image: ${TELEPORT_IMAGE}
-          args: ["-d", "--insecure", "--diag-addr=0.0.0.0:3434"]
-          name: tc
-          ports:
-            - containerPort: 3022
-              name: nodessh
-              protocol: TCP
-            - name: diag
-              containerPort: 3434
-              protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 3434
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            tcpSocket:
-              port: 3434
-            timeoutSeconds: 1
-          volumeMounts:
-            - name: config
-              mountPath: /etc/teleport/
-              readOnly: true
-            - name: license
-              mountPath: /var/lib/teleport/license.pem
-              subPath: license.pem
-              readOnly: true
@@ -1,29 +0,0 @@
-# preview address configuration
-.PHONY: plan
-plan:
-	terraform plan
-
-# reserve static ips
-.PHONY: apply
-apply:
-	terraform apply
-
-# release static ips
-.PHONY: destroy
-destroy:
-	terraform destroy
-
-# print static ips
-.PHONY: get-ips
-get-ips:
-	@terraform output
-
-# print proxy static ip
-.PHONY: get-proxy-ip
-get-proxy-ip:
-	@terraform output -raw proxy_ip
-
-# print grafana static ip
-.PHONY: get-grafana-ip
-get-grafana-ip:
-	@terraform output -raw grafana_ip
@@ -1,30 +0,0 @@
-# Network
-
-This reserves external static ip addresses to be used
-for the [`k8s/proxy`](../k8s/proxy.yaml) and [`k8s/grafana`](../k8s/grafana.yaml) services.
-Once you have the ips you **must** setup the DNS records accordingly.
-
-**NOTE**: This only needs to be run once for a GCP project.
-
-## Usage
-
-Confirm that everything is working correctly:
-
-```bash
-$ make plan
-```
-
-If you are running this automation for the first time, you may be asked to run
-`terraform init` before continuing.
-
-```bash
-$ make apply
-```
-
---
-
-To release the static ip addresses run:
-
-```bash
-$ make destroy
-```
@@ -1,19 +0,0 @@
-provider "google" {
-  project = var.project
-  region  = var.region
-  zone    = var.zone
-}
-
-terraform {
-  required_version = ">= 0.12"
-  backend "gcs" {
-    bucket = "loadtest-tf-state"
-    prefix = "terraform/state"
-  }
-}
-
-resource "google_compute_address" "proxy_ip" {
-  name         = "proxy-ip"
-  address_type = "EXTERNAL"
-  network_tier = "PREMIUM"
-}
@@ -1,4 +0,0 @@
-output "proxy_ip" {
-  description = "The static proxy ip address"
-  value       = google_compute_address.proxy_ip.address
-}
@@ -1,16 +0,0 @@
-variable "project" {
-  type        = string
-  description = "The project to manage resources in."
-}
-
-variable "region" {
-  type        = string
-  description = "The region to manage resources in."
-  default     = "us-central1"
-}
-
-variable "zone" {
-  type        = string
-  description = "The zone to manage resources in."
-  default     = "us-central1-a"
-}
@@ -1,33 +0,0 @@
-kind: role
-version: v3
-metadata:
-  name: clusteradmin
-spec:
-  # SSH options used for user sessions 
-  options:
-    # max_session_ttl defines the TTL (time to live) of SSH certificates 
-    # issued to the users with this role.
-    max_session_ttl: 8h
-
-    # forward_agent controls either users are allowed to use SSH agent forwarding
-    forward_agent: true
-
-  # allow section declares a list of resource/verb combinations that are
-  # allowed for the users of this role. by default nothing is allowed.
-  allow:
-    # logins array defines the OS logins a user is allowed to use.
-    # A few special variables are supported here (see below)
-    logins: [root, ubuntu]
-
-    # node labels that a user can connect to. The wildcard ('*') means "any node"
-    node_labels:
-      '*': '*'
-
-    # see below.
-    rules:
-    - resources: ['*']
-      verbs: ['*']
-
-  # the deny section uses the identical format as the 'allow' section.
-  # the deny rules always override allow rules.
-  deny: {}
@@ -1,6 +0,0 @@
-kind: user
-metadata:
-  name: soaktest-runner
-spec:
-  roles: ['clusteradmin']
-version: v2
@@ -1,30 +0,0 @@
-#!/bin/bash
-# This script runs the teleport load test soak tests.
-set -e
-set -x
-
-direct_node=$(tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth -l root ls -f names | grep -v iot)
-tunnel_node=$(tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth -l root ls -f names | grep iot)
-
-echo "${direct_node}"
-echo "${tunnel_node}"
-
-if [ -z "${direct_node}" ]; then
-		echo "no direct dial nodes found to run soak test on.";
-		exit 1;
-fi
-
-if [ -z "${tunnel_node}" ]; then
-		echo "no reverse tunnel nodes found to run soak test on.";
-		exit 1;
-fi
-
-echo "----Direct Dial Node Test----"
-tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth bench --duration="${DURATION}" root@"${direct_node}" ls
-
-tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth bench --duration="${DURATION}" --interactive root@"${direct_node}" ps aux
-
-echo "----Reverse Tunnel Node Test----"
-tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth bench --duration="${DURATION}" root@"${tunnel_node}" ls
-
-tsh --insecure --proxy="${PROXY_HOST}":3080 -i /etc/teleport/auth bench --duration="${DURATION}" --interactive root@"${tunnel_node}" ps aux
@@ -1,12 +0,0 @@
-kind: trusted_cluster
-version: v2
-metadata:
-  name: one
-spec:
-  enabled: true
-  token: cluster-${TC_TOKEN}
-  tunnel_addr: "${PROXY_HOST}:3024"
-  web_proxy_addr: "${PROXY_HOST}:3080"
-  role_map:
-    - remote: '*'
-      local: ['access']
@@ -1,36 +0,0 @@
-teleport:
-  log:
-    severity: DEBUG
-
-  data_dir: /var/lib/teleport
-
-  advertise_ip: auth
-
-  storage:
-    type: dynamodb
-    table_name: ${DYNAMO_TABLE}
-    region: ${DYNAMO_REGION}
-
-  connection_limits:
-    max_connections: 65000
-    max_users: 10000
-
-auth_service:
-  enabled: yes
-
-  listen_addr: 0.0.0.0:3025
-
-  authentication:
-    type: oidc
-
-  cluster_name: one
-  tokens:
-    - "node:node-${NODE_TOKEN}"
-    - "proxy:proxy-${PROXY_TOKEN}"
-    - "trusted_cluster:cluster-${TC_TOKEN}"
-
-ssh_service:
-  enabled: no
-
-proxy_service:
-  enabled: no
@@ -1,40 +0,0 @@
-teleport:
-  log:
-    severity: DEBUG
-    format:
-      output: json
-
-  data_dir: /var/lib/teleport
-
-  advertise_ip: auth
-
-  storage:
-    type: etcd
-    peers: [https://etcd-0.etcd:2379, https://etcd-1.etcd:2379, https://etcd-2.etcd:2379]
-    tls_cert_file: /etc/etcd/certs/client-cert.pem
-    tls_key_file: /etc/etcd/certs/client-key.pem
-    tls_ca_file: /etc/etcd/certs/ca-cert.pem
-    prefix: teleport
-  connection_limits:
-    max_connections: 65000
-    max_users: 10000
-
-auth_service:
-  enabled: yes
-
-  listen_addr: 0.0.0.0:3025
-
-  authentication:
-    type: oidc
-
-  cluster_name: one
-  tokens: 
-      - "node:node-${NODE_TOKEN}"
-      - "proxy:proxy-${PROXY_TOKEN}"
-      - "trusted_cluster:cluster-${TC_TOKEN}"
-
-ssh_service:
-  enabled: no
-
-proxy_service:
-  enabled: no
@@ -1,38 +0,0 @@
-teleport:
-  log:
-    severity: DEBUG
-
-  data_dir: /var/lib/teleport
-
-  advertise_ip: auth
-
-  storage:
-    type: firestore
-    collection_name: cluster-data
-    credentials_path: /var/lib/teleport/gcp_creds.json
-    project_id: "${GCP_PROJECT}"
-    audit_events_uri: "firestore://events?projectID=${GCP_PROJECT}&credentialsPath=/var/lib/teleport/gcp_creds.json"
-    audit_sessions_uri: "gs://teleport-session-storage-2?projectID=${GCP_PROJECT}&credentialsPath=/var/lib/teleport/gcp_creds.json"
-  connection_limits:
-    max_connections: 65000
-    max_users: 10000
-
-auth_service:
-  enabled: yes
-
-  listen_addr: 0.0.0.0:3025
-
-  authentication:
-    type: oidc
-
-  cluster_name: one
-  tokens: 
-      - "node:node-${NODE_TOKEN}"
-      - "proxy:proxy-${PROXY_TOKEN}"
-      - "trusted_cluster:cluster-${TC_TOKEN}"
-
-ssh_service:
-  enabled: no
-
-proxy_service:
-  enabled: no
@@ -1,17 +0,0 @@
-version: v3
-teleport:
-  data_dir: /var/lib/teleport
-  log:
-    severity: DEBUG
-    format:
-      output: json
-  storage:
-    type: dir
-  auth_token: "node-${NODE_TOKEN}"
-  proxy_server: ${PROXY_HOST}:3080
-auth_service:
-  enabled: false
-proxy_service:
-  enabled: false
-ssh_service:
-  enabled: true
@@ -1,17 +0,0 @@
-version: v3
-teleport:
-  data_dir: /var/lib/teleport
-  log:
-    severity: DEBUG
-    format:
-      output: json
-  storage:
-    type: dir
-  auth_token: "node-${NODE_TOKEN}"
-  auth_server: auth:3025
-auth_service:
-  enabled: false
-proxy_service:
-  enabled: false
-ssh_service:
-  enabled: true
@@ -1,30 +0,0 @@
-version: v3
-teleport:
-  log:
-    severity: DEBUG
-    format:
-      output: json
-
-  data_dir: /var/lib/teleport
-  auth_server: auth:3025
-  auth_token: "proxy-${PROXY_TOKEN}"
-  cache:
-    type: in-memory
-  connection_limits:
-    max_connections: 65000
-    max_users: 10000
-
-auth_service:
-  enabled: no
-
-ssh_service:
-  enabled: no
-
-proxy_service:
-  enabled: yes
-  https_cert_file: /etc/teleport-tls/tls.crt
-  https_key_file: /etc/teleport-tls/tls.key
-  public_addr: "${PROXY_HOST}:3080"
-  tunnel_public_addr: "${PROXY_HOST}:3024"
-  tunnel_listen_addr: 0.0.0.0:3024
-  listen_addr: 0.0.0.0:3023
@@ -1,13 +0,0 @@
-auth_service:
-  enabled: true
-proxy_service:
-  enabled: true
-ssh_service:
-  enabled: true
-teleport:
-  data_dir: /var/lib/teleport
-  log:
-    output: stderr
-    severity: DEBUG
-  storage:
-    type: dir
@@ -1,29 +0,0 @@
-replicaCount: 30
-agentsPerPod: 15
-
-minReadySeconds: 30
-
-image:
-  # TODO: remove after v12 is released
-  tag: 12.0.0-hugochartsplit.2
-
-joinParams:
-  method: token
-  token_name: "qwertyuiop"
-
-tolerations:
- key: "role"
-  operator: "Equal"
-  value: "agent"
-  effect: "NoSchedule"
-
-affinity:
-  nodeAffinity:
-    requiredDuringSchedulingIgnoredDuringExecution:
-      nodeSelectorTerms:
-      - matchExpressions:
-        - key: role
-          operator: In
-          values:
-            - agent
-