mirror of
https://github.com/coder/coder.git
synced 2026-10-24 23:14:28 +08:00
closes CODAGT-839 closes CODAGT-843 closes CODAGT-773 ## Summary Adds a new heartbeat usage event type, `hb_agent_runtime_v1`, measuring the total agent-loop runtime of Coder Agents (chats) per UTC hour, plus a reconciler that generates one event per hour with self-healing backfill over a trailing 7-day window. Events flow to Tallyman through the existing publisher unchanged. This measures the new Coder Agents (the `chats` tables), not the deprecated Tasks counted by `dc_managed_agents_v1`. Independent of #27508, which fixes the dead ai-seats cron registration. Both PRs carry the identical `usage_event` create permission hunk for the usage-publisher subject (this feature's generator and the ai-seats cron each need it for heartbeat inserts), so they can land in either order and the overlap merges cleanly. > [!WARNING] > **Do not include this in a release until Tallyman accepts `hb_agent_runtime_v1`.** The publisher marks permanently rejected events as done-forever, and the generator then sees those buckets as complete locally, so their usage would be silently and permanently lost. ## Details Each event's payload is `{"runtime_ms": N}`: the sum of `chat_messages.runtime_ms` for messages created in the hour bucket `[H, H+1)`, across all chats (sub-agents, API-created, archived, and soft-deleted messages included). Events use deterministic IDs (`hb_agent_runtime_v1:<bucket start>`) with `created_at` set to the bucket start, so concurrent replicas race safely via `ON CONFLICT (id) DO NOTHING` without locking, and daily rollups attribute backfilled hours to the correct day. Idle hours produce zero-valued events. A bucket becomes eligible 5 minutes after it closes; hours missing for longer than the 7-day window are forfeited, which can only undercount. Note that this makes `usage_events.created_at` explicitly the *event occurrence time* rather than the row insertion time; the two only diverge for backfilled events. It already behaved as the occurrence timestamp (it drives the daily rollup day and is shipped to Tallyman/Metronome as the event timestamp), and the migration now documents this with a `COMMENT ON COLUMN`, which also surfaces as a Go doc comment on `UsageEvent.CreatedAt`. The new `usage.Generator` runs unconditionally in enterprise builds; the `publish_usage_data` license flag continues to gate egress only, so air-gapped deployments still fill their local ledger. The `aggregate_usage_event()` trigger sums `runtime_ms` per day into `usage_events_daily` (unlike `hb_ai_seats_v1`, which takes the daily max). `InsertHeartbeatUsageEvent` now takes an explicit `createdAt` so generators can backfill historical buckets; the cron passes `clock.Now()` to preserve its existing behavior. ## Tallyman follow-up <details> <summary>Prompt for the Tallyman-repo agent</summary> > **Task**: Add support for the new Coder usage event type `hb_agent_runtime_v1` so Tallyman accepts, validates, and forwards it to Metronome. > > **Background**: coder/coder PR (this PR) adds hourly heartbeat events measuring Coder Agent runtime. Events arrive via the existing `/api/v1/events/ingest` endpoint with: `event_type: "hb_agent_runtime_v1"`, `event_data: {"runtime_ms": <int64 >= 0>}`, deterministic `id` of the form `hb_agent_runtime_v1:2026-07-15_14:00:00` (UTC hour bucket start), and `created_at` set to the bucket start (may be up to ~8 days in the past due to backfill; within Metronome's 34-day dedup window). Zero-value events are normal (idle hours). > > **Work**: > 1. Update Tallyman's vendored/imported `coderd/usage/usagetypes` (or equivalent) to the coder/coder commit that adds `UsageEventTypeHBAgentRuntimeV1` and `HBAgentRuntime`. > 2. Ensure ingestion validation accepts the type (`Valid()` switches) and rejects negative `runtime_ms`. > 3. Ensure Metronome forwarding maps the event with transaction ID derived from the event `id` as for existing types, passing `runtime_ms` through as the property for a SUM-aggregated billable metric ("Coder Agent Hours" = `SUM(runtime_ms) / 3,600,000`). > 4. Do NOT permanently reject unknown-but-well-formed future `hb_*` types if avoidable; at minimum confirm current behavior for unknown types (temporary vs permanent rejection) and report it. > 5. Tests: ingest accept/validate, dedup by ID, Metronome payload mapping. > > **Constraint**: this must be deployed to tallyman-prod **before** any coder/coder release containing the event generator; coderd treats permanent rejections as terminal per event. </details>
136 lines
3.9 KiB
Go
136 lines
3.9 KiB
Go
package usage_test
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbmock"
|
|
"github.com/coder/coder/v2/coderd/database/dbtime"
|
|
"github.com/coder/coder/v2/coderd/usage/usagetypes"
|
|
"github.com/coder/coder/v2/enterprise/coderd/usage"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func TestInserter(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("OK", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
clock := quartz.NewMock(t)
|
|
inserter := usage.NewDBInserter(usage.InserterWithClock(clock))
|
|
|
|
now := dbtime.Now()
|
|
events := []struct {
|
|
time time.Time
|
|
event usagetypes.DiscreteEvent
|
|
}{
|
|
{
|
|
time: now,
|
|
event: usagetypes.DCManagedAgentsV1{
|
|
Count: 1,
|
|
},
|
|
},
|
|
{
|
|
time: now.Add(1 * time.Minute),
|
|
event: usagetypes.DCManagedAgentsV1{
|
|
Count: 2,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, e := range events {
|
|
eventJSON := jsoninate(t, e.event)
|
|
db.EXPECT().InsertUsageEvent(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(ctx interface{}, params database.InsertUsageEventParams) error {
|
|
_, err := uuid.Parse(params.ID)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, e.event.EventType(), usagetypes.UsageEventType(params.EventType))
|
|
assert.JSONEq(t, eventJSON, string(params.EventData))
|
|
assert.Equal(t, e.time, params.CreatedAt)
|
|
return nil
|
|
},
|
|
).Times(1)
|
|
|
|
clock.Set(e.time)
|
|
err := inserter.InsertDiscreteUsageEvent(ctx, db, e.event)
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
|
|
t.Run("Heartbeat", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
inserter := usage.NewDBInserter()
|
|
|
|
// Heartbeat inserts must store the provided id and createdAt
|
|
// verbatim.
|
|
event := usagetypes.HBAgentRuntime{RuntimeMs: 1234}
|
|
eventJSON := jsoninate(t, event)
|
|
id := "hb_agent_runtime_v1:2025-01-02_03:00:00"
|
|
createdAt := time.Date(2025, 1, 2, 3, 0, 0, 0, time.UTC)
|
|
|
|
db.EXPECT().InsertUsageEvent(gomock.Any(), gomock.Any()).DoAndReturn(
|
|
func(ctx interface{}, params database.InsertUsageEventParams) error {
|
|
assert.Equal(t, id, params.ID)
|
|
assert.Equal(t, event.EventType(), usagetypes.UsageEventType(params.EventType))
|
|
assert.JSONEq(t, eventJSON, string(params.EventData))
|
|
assert.Equal(t, dbtime.Time(createdAt), params.CreatedAt)
|
|
return nil
|
|
},
|
|
).Times(1)
|
|
|
|
err := inserter.InsertHeartbeatUsageEvent(ctx, db, id, createdAt, event)
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
t.Run("InvalidEvent", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
|
|
// We should get an error if the event is invalid.
|
|
inserter := usage.NewDBInserter()
|
|
err := inserter.InsertDiscreteUsageEvent(ctx, db, usagetypes.DCManagedAgentsV1{
|
|
Count: 0, // invalid
|
|
})
|
|
assert.ErrorContains(t, err, `invalid "dc_managed_agents_v1" event: count must be greater than 0`)
|
|
|
|
err = inserter.InsertHeartbeatUsageEvent(ctx, db, "some-id", time.Now(), usagetypes.HBAgentRuntime{
|
|
RuntimeMs: -1, // invalid
|
|
})
|
|
assert.ErrorContains(t, err, `invalid "hb_agent_runtime_v1" event: runtime_ms cannot be negative`)
|
|
})
|
|
|
|
t.Run("ZeroCreatedAt", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
ctrl := gomock.NewController(t)
|
|
// The mock store fails the test on any unexpected call, so no insert
|
|
// may reach the database.
|
|
db := dbmock.NewMockStore(ctrl)
|
|
|
|
inserter := usage.NewDBInserter()
|
|
err := inserter.InsertHeartbeatUsageEvent(ctx, db, "some-id", time.Time{}, usagetypes.HBAgentRuntime{
|
|
RuntimeMs: 1,
|
|
})
|
|
assert.ErrorContains(t, err, `createdAt must be set for "hb_agent_runtime_v1" event`)
|
|
})
|
|
}
|