feat: Add high availability for multiple replicas (#4555)

* feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* remove printlns

* close all connections on coordinator

* impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* Add replicas

* Add DERP meshing to arbitrary addresses

* Move packages to highavailability folder

* Move coordinator to high availability package

* Add flags for HA

* Rename to replicasync

* Denest packages for replicas

* Add test for multiple replicas

* Fix coordination test

* Add HA to the helm chart

* Rename function pointer

* Add warnings for HA

* Add the ability to block endpoints

* Add flag to disable P2P connections

* Wow, I made the tests pass

* Add replicas endpoint

* Ensure close kills replica

* Update sql

* Add database latency to high availability

* Pipe TLS to DERP mesh

* Fix DERP mesh with TLS

* Add tests for TLS

* Fix replica sync TLS

* Fix RootCA for replica meshing

* Remove ID from replicasync

* Fix getting certificates for meshing

* Remove excessive locking

* Fix linting

* Store mesh key in the database

* Fix replica key for tests

* Fix types gen

* Fix unlocking unlocked

* Fix race in tests

* Update enterprise/derpmesh/derpmesh.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Rename to syncReplicas

* Reuse http client

* Delete old replicas on a CRON

* Fix race condition in connection tests

* Fix linting

* Fix nil type

* Move pubsub to in-memory for twenty test

* Add comment for configuration tweaking

* Fix leak with transport

* Fix close leak in derpmesh

* Fix race when creating server

* Remove handler update

* Skip test on Windows

* Fix DERP mesh test

* Wrap HTTP handler replacement in mutex

* Fix error message for relay

* Fix API handler for normal tests

* Fix speedtest

* Fix replica resend

* Fix derpmesh send

* Ping async

* Increase wait time of template version jobd

* Fix race when closing replica sync

* Add name to client

* Log the derpmap being used

* Don't connect if DERP is empty

* Improve agent coordinator logging

* Fix lock in coordinator

* Fix relay addr

* Fix race when updating durations

* Fix client publish race

* Run pubsub loop in a queue

* Store agent nodes in order

* Fix coordinator locking

* Check for closed pipe

Co-authored-by: Colin Adler <colin1adler@gmail.com>

site/src/api/api.ts

@@ -28,6 +28,7 @@ export const defaultEntitlements = (): TypesGen.Entitlements => {
   return {
     features: features,
     has_license: false,
+    errors: [],
     warnings: [],
     experimental: false,
     trial: false,

site/src/api/typesGenerated.ts

@@ -274,6 +274,7 @@ export interface DeploymentFlags {
   readonly derp_server_region_code: StringFlag
   readonly derp_server_region_name: StringFlag
   readonly derp_server_stun_address: StringArrayFlag
+  readonly derp_server_relay_address: StringFlag
   readonly derp_config_url: StringFlag
   readonly derp_config_path: StringFlag
   readonly prom_enabled: BoolFlag
@@ -337,6 +338,7 @@ export interface DurationFlag {
 export interface Entitlements {
   readonly features: Record<string, Feature>
   readonly warnings: string[]
+  readonly errors: string[]
   readonly has_license: boolean
   readonly experimental: boolean
   readonly trial: boolean
@@ -528,6 +530,17 @@ export interface PutExtendWorkspaceRequest {
   readonly deadline: string
 }
 
+// From codersdk/replicas.go
+export interface Replica {
+  readonly id: string
+  readonly hostname: string
+  readonly created_at: string
+  readonly relay_address: string
+  readonly region_id: number
+  readonly error: string
+  readonly database_latency: number
+}
+
 // From codersdk/error.go
 export interface Response {
   readonly message: string

site/src/components/LicenseBanner/LicenseBanner.tsx

@@ -8,15 +8,15 @@ export const LicenseBanner: React.FC = () => {
   const [entitlementsState, entitlementsSend] = useActor(
     xServices.entitlementsXService,
   )
-  const { warnings } = entitlementsState.context.entitlements
+  const { errors, warnings } = entitlementsState.context.entitlements
 
   /** Gets license data on app mount because LicenseBanner is mounted in App */
   useEffect(() => {
     entitlementsSend("GET_ENTITLEMENTS")
   }, [entitlementsSend])
 
-  if (warnings.length > 0) {
-    return <LicenseBannerView warnings={warnings} />
+  if (errors.length > 0 || warnings.length > 0) {
+    return <LicenseBannerView errors={errors} warnings={warnings} />
   } else {
     return null
   }

site/src/components/LicenseBanner/LicenseBannerView.stories.tsx

@@ -12,13 +12,23 @@ const Template: Story<LicenseBannerViewProps> = (args) => (
 
 export const OneWarning = Template.bind({})
 OneWarning.args = {
+  errors: [],
   warnings: ["You have exceeded the number of seats in your license."],
 }
 
 export const TwoWarnings = Template.bind({})
 TwoWarnings.args = {
+  errors: [],
   warnings: [
     "You have exceeded the number of seats in your license.",
     "You are flying too close to the sun.",
   ],
 }
+
+export const OneError = Template.bind({})
+OneError.args = {
+  errors: [
+    "You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.",
+  ],
+  warnings: [],
+}

site/src/components/LicenseBanner/LicenseBannerView.tsx

@@ -2,47 +2,56 @@ import { makeStyles } from "@material-ui/core/styles"
 import { Expander } from "components/Expander/Expander"
 import { Pill } from "components/Pill/Pill"
 import { useState } from "react"
+import { colors } from "theme/colors"
 
 export const Language = {
   licenseIssue: "License Issue",
   licenseIssues: (num: number): string => `${num} License Issues`,
-  upgrade: "Contact us to upgrade your license.",
+  upgrade: "Contact sales@coder.com.",
   exceeded: "It looks like you've exceeded some limits of your license.",
   lessDetails: "Less",
   moreDetails: "More",
 }
 
 export interface LicenseBannerViewProps {
+  errors: string[]
   warnings: string[]
 }
 
 export const LicenseBannerView: React.FC<LicenseBannerViewProps> = ({
+  errors,
   warnings,
 }) => {
   const styles = useStyles()
   const [showDetails, setShowDetails] = useState(false)
-  if (warnings.length === 1) {
+  const isError = errors.length > 0
+  const messages = [...errors, ...warnings]
+  const type = isError ? "error" : "warning"
+
+  if (messages.length === 1) {
     return (
-      <div className={styles.container}>
-        <Pill text={Language.licenseIssue} type="warning" lightBorder />
-        <span className={styles.text}>{warnings[0]}</span>
-        &nbsp;
-        <a href="mailto:sales@coder.com" className={styles.link}>
-          {Language.upgrade}
-        </a>
+      <div className={`${styles.container} ${type}`}>
+        <Pill text={Language.licenseIssue} type={type} lightBorder />
+        <div className={styles.leftContent}>
+          <span>{messages[0]}</span>
+          &nbsp;
+          <a href="mailto:sales@coder.com" className={styles.link}>
+            {Language.upgrade}
+          </a>
+        </div>
       </div>
     )
   } else {
     return (
-      <div className={styles.container}>
-        <div className={styles.flex}>
-          <div className={styles.leftContent}>
-            <Pill
-              text={Language.licenseIssues(warnings.length)}
-              type="warning"
-              lightBorder
-            />
-            <span className={styles.text}>{Language.exceeded}</span>
+      <div className={`${styles.container} ${type}`}>
+        <Pill
+          text={Language.licenseIssues(messages.length)}
+          type={type}
+          lightBorder
+        />
+        <div className={styles.leftContent}>
+          <div>
+            {Language.exceeded}
             &nbsp;
             <a href="mailto:sales@coder.com" className={styles.link}>
               {Language.upgrade}
@@ -50,9 +59,9 @@ export const LicenseBannerView: React.FC<LicenseBannerViewProps> = ({
           </div>
           <Expander expanded={showDetails} setExpanded={setShowDetails}>
             <ul className={styles.list}>
-              {warnings.map((warning) => (
-                <li className={styles.listItem} key={`${warning}`}>
-                  {warning}
+              {messages.map((message) => (
+                <li className={styles.listItem} key={`${message}`}>
+                  {message}
                 </li>
               ))}
             </ul>
@@ -67,14 +76,18 @@ const useStyles = makeStyles((theme) => ({
   container: {
     padding: theme.spacing(1.5),
     backgroundColor: theme.palette.warning.main,
+    display: "flex",
+    alignItems: "center",
+
+    "&.error": {
+      backgroundColor: colors.red[12],
+    },
   },
   flex: {
-    display: "flex",
+    display: "column",
   },
   leftContent: {
     marginRight: theme.spacing(1),
-  },
-  text: {
     marginLeft: theme.spacing(1),
   },
   link: {
@@ -83,9 +96,10 @@ const useStyles = makeStyles((theme) => ({
     fontWeight: "bold",
   },
   list: {
-    margin: theme.spacing(1.5),
+    padding: theme.spacing(1),
+    margin: 0,
   },
   listItem: {
-    margin: theme.spacing(1),
+    margin: theme.spacing(0.5),
   },
 }))

site/src/testHelpers/entities.ts

@@ -821,6 +821,7 @@ export const makeMockApiError = ({
 })
 
 export const MockEntitlements: TypesGen.Entitlements = {
+  errors: [],
   warnings: [],
   has_license: false,
   features: {},
@@ -829,6 +830,7 @@ export const MockEntitlements: TypesGen.Entitlements = {
 }
 
 export const MockEntitlementsWithWarnings: TypesGen.Entitlements = {
+  errors: [],
   warnings: ["You are over your active user limit.", "And another thing."],
   has_license: true,
   experimental: false,
@@ -852,6 +854,7 @@ export const MockEntitlementsWithWarnings: TypesGen.Entitlements = {
 }
 
 export const MockEntitlementsWithAuditLog: TypesGen.Entitlements = {
+  errors: [],
   warnings: [],
   has_license: true,
   experimental: false,

site/src/xServices/entitlements/entitlementsXService.ts

@@ -20,6 +20,7 @@ export type EntitlementsEvent =
   | { type: "HIDE_MOCK_BANNER" }
 
 const emptyEntitlements = {
+  errors: [],
   warnings: [],
   features: {},
   has_license: false,