mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
## Problem Two related symptoms of the same architectural issue: the `dbcrypt` wrapper is installed inside `enterprise/coderd.New`, so any access to `options.Database` that happens before `newAPI` runs bypasses encryption. **Symptom 1 (reads):** Provider keys added via the admin UI are encrypted at rest. `BuildProviders` was running *before* `newAPI`, against the unwrapped store, so the ciphertext was read as-is and shoved into the keypool as the upstream credential. Anthropic/OpenAI reject it, and the interception log shows: ``` coderd.aibridged.pool: interception failed ... error="all configured keys failed authentication" credential_kind=centralized credential_hint=PaPb...4A== credential_length=184 ``` **Symptom 2 (writes):** `SeedAIProvidersFromEnv` was also running before `newAPI`, against the unwrapped store, so env-derived keys (`CODER_AIBRIDGE_OPENAI_KEY`, indexed `CODER_AIBRIDGE_PROVIDER_<N>_KEY`, etc.) landed in `ai_provider_keys` as plaintext with `ApiKeyKeyID = null` even when `CODER_EXTERNAL_TOKEN_ENCRYPTION_KEYS` was set. ## Fix Move both `SeedAIProvidersFromEnv` and `BuildProviders` to after `newAPI`, where `options.Database` is the dbcrypt-wrapped store. Writes encrypt correctly; reads decrypt correctly. The enterprise closure (`enterprise/cli/server.go`) runs *inside* `newAPI` and calls `BuildProviders` for the aibridgeproxyd at that point. Once the agpl seed moves to after `newAPI`, the proxy on first boot would see no env-seeded providers. Add a matching seed call inside the enterprise closure before its `BuildProviders` to cover that case. Seeding is idempotent, so the agpl-side seed running again post-`newAPI` is a no-op when the rows already exist. ## Known shortcomings The clean version of this fix would just inherit `ctx` like every other startup step and place these calls naturally. It can't, for two reasons that are both about the surrounding handler architecture rather than this change: 1. **`dbcrypt` wrapping is positioned inside `newAPI`, not around `options.Database` at creation.** That's why both seed and build have to wait until after `newAPI` in the first place. The principled fix is to install the wrapper at the point the store is created (behind a hook the enterprise build supplies), so every consumer sees a single authoritative view and the ordering stops mattering. This would also collapse the duplicated seed call back to a single site. 2. **The handler's shutdown sequence is not deferred.** `coderAPICloser.Close()` and the other teardown steps run only if control reaches the `select` at the bottom of the handler. An early `return` from anywhere in Phase 1 (e.g. seed/build returning `context.Canceled` when the user hits ctrl-c during startup) skips that block and orphans all the goroutines `newAPI` spawned — tailnet workers, gitsync, telemetry batcher, etc. `goleak` then catches them at package teardown and `TestServer_TelemetryDisabled_FinalReport` fails. Moving the shutdown into deferred closers (with a `sync.Once`-guarded close to avoid double-close from the explicit Phase 2 call) is the principled fix. For this PR I took the smallest change that fixes the reported bugs: a detached context (`context.WithoutCancel(ctx)` + a 30s timeout) at the seed and build call sites in both the agpl and enterprise paths. It lets the calls complete even if the user cancels during startup, after which the handler reaches its shutdown select naturally and tears down through Phase 2. Both shortcomings above are worth addressing separately. ## Test plan - `make test RUN=TestServer_TelemetryDisabled_FinalReport` with `-race`; passes locally with `-count=3`. - Manually verified on a deployment with `CODER_EXTERNAL_TOKEN_ENCRYPTION_KEYS` set and env-configured providers: `ai_provider_keys.api_key_key_id` is populated, `api_key` is base64 ciphertext, and upstream auth succeeds. --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
231 lines
7.8 KiB
Go
231 lines
7.8 KiB
Go
//go:build !slim
|
|
|
|
package cli
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/base64"
|
|
"errors"
|
|
"io"
|
|
"net/url"
|
|
"time"
|
|
|
|
"golang.org/x/xerrors"
|
|
"tailscale.com/derp"
|
|
"tailscale.com/types/key"
|
|
|
|
agplcli "github.com/coder/coder/v2/cli"
|
|
agplcoderd "github.com/coder/coder/v2/coderd"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/cryptorand"
|
|
"github.com/coder/coder/v2/enterprise/audit"
|
|
"github.com/coder/coder/v2/enterprise/audit/backends"
|
|
"github.com/coder/coder/v2/enterprise/coderd"
|
|
"github.com/coder/coder/v2/enterprise/coderd/dormancy"
|
|
"github.com/coder/coder/v2/enterprise/coderd/usage"
|
|
"github.com/coder/coder/v2/enterprise/dbcrypt"
|
|
"github.com/coder/coder/v2/enterprise/trialer"
|
|
"github.com/coder/coder/v2/tailnet"
|
|
"github.com/coder/quartz"
|
|
"github.com/coder/serpent"
|
|
)
|
|
|
|
func (r *RootCmd) Server(_ func()) *serpent.Command {
|
|
cmd := r.RootCmd.Server(func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, io.Closer, error) {
|
|
if options.DeploymentValues.DERP.Server.RelayURL.String() != "" {
|
|
_, err := url.Parse(options.DeploymentValues.DERP.Server.RelayURL.String())
|
|
if err != nil {
|
|
return nil, nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err)
|
|
}
|
|
}
|
|
|
|
// Always generate a mesh key, even if the built-in DERP server is
|
|
// disabled. This mesh key is still used by workspace proxies running
|
|
// HA.
|
|
var meshKey string
|
|
err := options.Database.InTx(func(tx database.Store) error {
|
|
// This will block until the lock is acquired, and will be
|
|
// automatically released when the transaction ends.
|
|
err := tx.AcquireLock(ctx, database.LockIDEnterpriseDeploymentSetup)
|
|
if err != nil {
|
|
return xerrors.Errorf("acquire lock: %w", err)
|
|
}
|
|
|
|
meshKey, err = tx.GetDERPMeshKey(ctx)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if !errors.Is(err, sql.ErrNoRows) {
|
|
return xerrors.Errorf("get DERP mesh key: %w", err)
|
|
}
|
|
meshKey, err = cryptorand.String(32)
|
|
if err != nil {
|
|
return xerrors.Errorf("generate DERP mesh key: %w", err)
|
|
}
|
|
err = tx.InsertDERPMeshKey(ctx, meshKey)
|
|
if err != nil {
|
|
return xerrors.Errorf("insert DERP mesh key: %w", err)
|
|
}
|
|
return nil
|
|
}, nil)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if meshKey == "" {
|
|
return nil, nil, xerrors.New("mesh key is empty")
|
|
}
|
|
|
|
if options.DeploymentValues.DERP.Server.Enable {
|
|
options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp")))
|
|
options.DERPServer.SetMeshKey(meshKey)
|
|
}
|
|
|
|
options.Auditor = audit.NewAuditor(
|
|
options.Database,
|
|
audit.DefaultFilter,
|
|
backends.NewPostgres(options.Database, true),
|
|
backends.NewSlog(options.Logger),
|
|
)
|
|
|
|
options.TrialGenerator = trialer.New(options.Database, "https://v2-licensor.coder.com/trial", coderd.Keys)
|
|
|
|
o := &coderd.Options{
|
|
Options: options,
|
|
AuditLogging: true,
|
|
ConnectionLogging: true,
|
|
BrowserOnly: options.DeploymentValues.BrowserOnly.Value(),
|
|
SCIMAPIKey: []byte(options.DeploymentValues.SCIMAPIKey.Value()),
|
|
RBAC: true,
|
|
DERPServerRelayAddress: options.DeploymentValues.DERP.Server.RelayURL.String(),
|
|
DERPServerRegionID: int(options.DeploymentValues.DERP.Server.RegionID.Value()),
|
|
ProxyHealthInterval: options.DeploymentValues.ProxyHealthStatusInterval.Value(),
|
|
DefaultQuietHoursSchedule: options.DeploymentValues.UserQuietHoursSchedule.DefaultSchedule.Value(),
|
|
ProvisionerDaemonPSK: options.DeploymentValues.Provisioner.DaemonPSK.Value(),
|
|
|
|
CheckInactiveUsersCancelFunc: dormancy.CheckInactiveUsers(ctx, options.Logger, quartz.NewReal(), options.Database, options.Auditor),
|
|
}
|
|
|
|
if encKeys := options.DeploymentValues.ExternalTokenEncryptionKeys.Value(); len(encKeys) != 0 {
|
|
keys := make([][]byte, 0, len(encKeys))
|
|
for idx, ek := range encKeys {
|
|
dk, err := base64.StdEncoding.DecodeString(ek)
|
|
if err != nil {
|
|
return nil, nil, xerrors.Errorf("decode external-token-encryption-key %d: %w", idx, err)
|
|
}
|
|
keys = append(keys, dk)
|
|
}
|
|
cs, err := dbcrypt.NewCiphers(keys...)
|
|
if err != nil {
|
|
return nil, nil, xerrors.Errorf("initialize encryption: %w", err)
|
|
}
|
|
o.ExternalTokenEncryption = cs
|
|
}
|
|
|
|
if o.LicenseKeys == nil {
|
|
o.LicenseKeys = coderd.Keys
|
|
}
|
|
|
|
closers := &multiCloser{}
|
|
|
|
// Create the enterprise API.
|
|
api, err := coderd.New(ctx, o)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
closers.Add(api)
|
|
|
|
// Start the enterprise usage publisher routine. This won't do anything
|
|
// unless the deployment is licensed and one of the licenses has usage
|
|
// publishing enabled.
|
|
publisher := usage.NewTallymanPublisher(ctx, options.Logger, options.Database, o.LicenseKeys,
|
|
usage.PublisherWithHTTPClient(api.HTTPClient),
|
|
)
|
|
err = publisher.Start()
|
|
if err != nil {
|
|
_ = closers.Close()
|
|
return nil, nil, xerrors.Errorf("start usage publisher: %w", err)
|
|
}
|
|
closers.Add(publisher)
|
|
|
|
// usageCron are heartbeat events to the usage table. These events are eventually sent
|
|
// to Tallyman.
|
|
usageCron := usage.NewCron(quartz.NewReal(), options.Logger.Named("usage-cron"), options.Database, *options.UsageInserter.Load())
|
|
// ai-seats heartbeats track the number of users that have used an AI feature.
|
|
// These users consume a seat for the AI addon to our License.
|
|
_ = usageCron.Register(usage.CronJob{
|
|
Name: "ai-seats",
|
|
Interval: usage.AISeatsInterval,
|
|
Jitter: 10 * time.Minute,
|
|
Fn: usage.AISeatsHeartbeat(options.Database),
|
|
})
|
|
usageCron.Start(ctx)
|
|
closers.Add(usageCron)
|
|
|
|
// In-memory AI Bridge Proxy daemon. The bridge daemon itself is
|
|
// started unconditionally by AGPL cli/server.go (chatd uses its
|
|
// in-memory roundtripper regardless of license); only the proxy
|
|
// daemon remains enterprise-gated by config.
|
|
if options.DeploymentValues.AI.BridgeProxyConfig.Enabled.Value() {
|
|
// Seed env-derived providers before reading them back so the
|
|
// proxy observes them on first startup. options.Database is
|
|
// dbcrypt-wrapped at this point (set by coderd.New above),
|
|
// so env-seeded keys are also written encrypted. Detached
|
|
// ctx for the same reason as in agplcli below: an early
|
|
// return would orphan newAPI's goroutines. Seeding is
|
|
// idempotent; the agplcli path seeds again post-newAPI.
|
|
//nolint:gocritic // Production timeout, not a test wait.
|
|
aibridgeInitCtx, aibridgeInitCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
|
defer aibridgeInitCancel()
|
|
if err := agplcoderd.SeedAIProvidersFromEnv(
|
|
aibridgeInitCtx,
|
|
options.Database,
|
|
options.DeploymentValues.AI.BridgeConfig,
|
|
options.Logger.Named("aibridge.envseed"),
|
|
); err != nil {
|
|
return nil, nil, xerrors.Errorf("seed ai providers from env: %w", err)
|
|
}
|
|
providers, err := agplcli.BuildProviders(aibridgeInitCtx, options.Database, options.DeploymentValues.AI.BridgeConfig, options.Logger.Named("aibridge.providers"))
|
|
if err != nil {
|
|
return nil, nil, xerrors.Errorf("build AI providers: %w", err)
|
|
}
|
|
aiBridgeProxyServer, err := newAIBridgeProxyDaemon(api, providers)
|
|
if err != nil {
|
|
_ = closers.Close()
|
|
return nil, nil, xerrors.Errorf("create aibridgeproxyd: %w", err)
|
|
}
|
|
closers.Add(aiBridgeProxyServer)
|
|
|
|
// Register the handler so coderd can serve the proxy endpoints.
|
|
api.RegisterInMemoryAIBridgeProxydHTTPHandler(aiBridgeProxyServer.Handler())
|
|
}
|
|
|
|
return api.AGPL, closers, nil
|
|
})
|
|
|
|
cmd.AddSubcommands(
|
|
r.dbcryptCmd(),
|
|
)
|
|
return cmd
|
|
}
|
|
|
|
type multiCloser struct {
|
|
closers []io.Closer
|
|
}
|
|
|
|
var _ io.Closer = &multiCloser{}
|
|
|
|
func (m *multiCloser) Add(closer io.Closer) {
|
|
m.closers = append(m.closers, closer)
|
|
}
|
|
|
|
func (m *multiCloser) Close() error {
|
|
var errs []error
|
|
for _, closer := range m.closers {
|
|
if err := closer.Close(); err != nil {
|
|
errs = append(errs, xerrors.Errorf("close %T: %w", closer, err))
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|