feat: Add high availability for multiple replicas (#4555)

* feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* remove printlns

* close all connections on coordinator

* impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* Add replicas

* Add DERP meshing to arbitrary addresses

* Move packages to highavailability folder

* Move coordinator to high availability package

* Add flags for HA

* Rename to replicasync

* Denest packages for replicas

* Add test for multiple replicas

* Fix coordination test

* Add HA to the helm chart

* Rename function pointer

* Add warnings for HA

* Add the ability to block endpoints

* Add flag to disable P2P connections

* Wow, I made the tests pass

* Add replicas endpoint

* Ensure close kills replica

* Update sql

* Add database latency to high availability

* Pipe TLS to DERP mesh

* Fix DERP mesh with TLS

* Add tests for TLS

* Fix replica sync TLS

* Fix RootCA for replica meshing

* Remove ID from replicasync

* Fix getting certificates for meshing

* Remove excessive locking

* Fix linting

* Store mesh key in the database

* Fix replica key for tests

* Fix types gen

* Fix unlocking unlocked

* Fix race in tests

* Update enterprise/derpmesh/derpmesh.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Rename to syncReplicas

* Reuse http client

* Delete old replicas on a CRON

* Fix race condition in connection tests

* Fix linting

* Fix nil type

* Move pubsub to in-memory for twenty test

* Add comment for configuration tweaking

* Fix leak with transport

* Fix close leak in derpmesh

* Fix race when creating server

* Remove handler update

* Skip test on Windows

* Fix DERP mesh test

* Wrap HTTP handler replacement in mutex

* Fix error message for relay

* Fix API handler for normal tests

* Fix speedtest

* Fix replica resend

* Fix derpmesh send

* Ping async

* Increase wait time of template version jobd

* Fix race when closing replica sync

* Add name to client

* Log the derpmap being used

* Don't connect if DERP is empty

* Improve agent coordinator logging

* Fix lock in coordinator

* Fix relay addr

* Fix race when updating durations

* Fix client publish race

* Run pubsub loop in a queue

* Store agent nodes in order

* Fix coordinator locking

* Check for closed pipe

Co-authored-by: Colin Adler <colin1adler@gmail.com>

helm/templates/coder.yaml

@@ -14,10 +14,7 @@ metadata:
     {{- include "coder.labels" . | nindent 4 }}
   annotations: {{ toYaml .Values.coder.annotations | nindent 4}}
 spec:
-  # NOTE: this is currently not used as coder v2 does not support high
-  #       availability yet.
-  # replicas: {{ .Values.coder.replicaCount }}
-  replicas: 1
+  replicas: {{ .Values.coder.replicaCount }}
   selector:
     matchLabels:
       {{- include "coder.selectorLabels" . | nindent 6 }}
@@ -38,6 +35,13 @@ spec:
           env:
             - name: CODER_ADDRESS
               value: "0.0.0.0:{{ include "coder.port" . }}"
+            # Used for inter-pod communication with high-availability.
+            - name: KUBE_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CODER_DERP_SERVER_RELAY_ADDRESS
+              value: "{{ include "coder.portName" . }}://$(KUBE_POD_IP):{{ include "coder.port" . }}"
             {{- include "coder.tlsEnv" . | nindent 12 }}
             {{- with .Values.coder.env -}}
             {{ toYaml . | nindent 12 }}

helm/templates/service.yaml

@@ -10,6 +10,7 @@ metadata:
     {{- toYaml .Values.coder.service.annotations | nindent 4 }}
 spec:
   type: {{ .Values.coder.service.type }}
+  sessionAffinity: ClientIP
   ports:
     - name: {{ include "coder.portName" . | quote }}
       port: {{ include "coder.servicePort" . }}

helm/values.yaml

@@ -1,9 +1,9 @@
 # coder -- Primary configuration for `coder server`.
 coder:
-  # NOTE: this is currently not used as coder v2 does not support high
-  #       availability yet.
-  # # coder.replicaCount -- The number of Kubernetes deployment replicas.
-  # replicaCount: 1
+  # coder.replicaCount -- The number of Kubernetes deployment replicas.
+  # This should only be increased if High Availability is enabled.
+  # This is an Enterprise feature. Contact sales@coder.com.
+  replicaCount: 1
 
   # coder.image -- The image to use for Coder.
   image: