mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: add hourly hb_agent_runtime_v1 usage events for Coder Agent runtime (#27312)
closes CODAGT-839 closes CODAGT-843 closes CODAGT-773 ## Summary Adds a new heartbeat usage event type, `hb_agent_runtime_v1`, measuring the total agent-loop runtime of Coder Agents (chats) per UTC hour, plus a reconciler that generates one event per hour with self-healing backfill over a trailing 7-day window. Events flow to Tallyman through the existing publisher unchanged. This measures the new Coder Agents (the `chats` tables), not the deprecated Tasks counted by `dc_managed_agents_v1`. Independent of #27508, which fixes the dead ai-seats cron registration. Both PRs carry the identical `usage_event` create permission hunk for the usage-publisher subject (this feature's generator and the ai-seats cron each need it for heartbeat inserts), so they can land in either order and the overlap merges cleanly. > [!WARNING] > **Do not include this in a release until Tallyman accepts `hb_agent_runtime_v1`.** The publisher marks permanently rejected events as done-forever, and the generator then sees those buckets as complete locally, so their usage would be silently and permanently lost. ## Details Each event's payload is `{"runtime_ms": N}`: the sum of `chat_messages.runtime_ms` for messages created in the hour bucket `[H, H+1)`, across all chats (sub-agents, API-created, archived, and soft-deleted messages included). Events use deterministic IDs (`hb_agent_runtime_v1:<bucket start>`) with `created_at` set to the bucket start, so concurrent replicas race safely via `ON CONFLICT (id) DO NOTHING` without locking, and daily rollups attribute backfilled hours to the correct day. Idle hours produce zero-valued events. A bucket becomes eligible 5 minutes after it closes; hours missing for longer than the 7-day window are forfeited, which can only undercount. Note that this makes `usage_events.created_at` explicitly the *event occurrence time* rather than the row insertion time; the two only diverge for backfilled events. It already behaved as the occurrence timestamp (it drives the daily rollup day and is shipped to Tallyman/Metronome as the event timestamp), and the migration now documents this with a `COMMENT ON COLUMN`, which also surfaces as a Go doc comment on `UsageEvent.CreatedAt`. The new `usage.Generator` runs unconditionally in enterprise builds; the `publish_usage_data` license flag continues to gate egress only, so air-gapped deployments still fill their local ledger. The `aggregate_usage_event()` trigger sums `runtime_ms` per day into `usage_events_daily` (unlike `hb_ai_seats_v1`, which takes the daily max). `InsertHeartbeatUsageEvent` now takes an explicit `createdAt` so generators can backfill historical buckets; the cron passes `clock.Now()` to preserve its existing behavior. ## Tallyman follow-up <details> <summary>Prompt for the Tallyman-repo agent</summary> > **Task**: Add support for the new Coder usage event type `hb_agent_runtime_v1` so Tallyman accepts, validates, and forwards it to Metronome. > > **Background**: coder/coder PR (this PR) adds hourly heartbeat events measuring Coder Agent runtime. Events arrive via the existing `/api/v1/events/ingest` endpoint with: `event_type: "hb_agent_runtime_v1"`, `event_data: {"runtime_ms": <int64 >= 0>}`, deterministic `id` of the form `hb_agent_runtime_v1:2026-07-15_14:00:00` (UTC hour bucket start), and `created_at` set to the bucket start (may be up to ~8 days in the past due to backfill; within Metronome's 34-day dedup window). Zero-value events are normal (idle hours). > > **Work**: > 1. Update Tallyman's vendored/imported `coderd/usage/usagetypes` (or equivalent) to the coder/coder commit that adds `UsageEventTypeHBAgentRuntimeV1` and `HBAgentRuntime`. > 2. Ensure ingestion validation accepts the type (`Valid()` switches) and rejects negative `runtime_ms`. > 3. Ensure Metronome forwarding maps the event with transaction ID derived from the event `id` as for existing types, passing `runtime_ms` through as the property for a SUM-aggregated billable metric ("Coder Agent Hours" = `SUM(runtime_ms) / 3,600,000`). > 4. Do NOT permanently reject unknown-but-well-formed future `hb_*` types if avoidable; at minimum confirm current behavior for unknown types (temporary vs permanent rejection) and report it. > 5. Tests: ingest accept/validate, dedup by ID, Metronome payload mapping. > > **Constraint**: this must be deployed to tallyman-prod **before** any coder/coder release containing the event generator; coderd treats permanent rejections as terminal per event. </details>
This commit is contained in:
@@ -671,9 +671,9 @@ var (
|
||||
Site: rbac.Permissions(map[string][]policy.Action{
|
||||
rbac.ResourceLicense.Type: {policy.ActionRead},
|
||||
rbac.ResourceAiSeat.Type: {policy.ActionRead}, // Required for GetActiveAISeatCount.
|
||||
// The usage publisher doesn't create events, just
|
||||
// reads/processes them.
|
||||
rbac.ResourceUsageEvent.Type: {policy.ActionRead, policy.ActionUpdate},
|
||||
// Create is required to insert heartbeat usage events
|
||||
// under this subject.
|
||||
rbac.ResourceUsageEvent.Type: {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
|
||||
}),
|
||||
User: []rbac.Permission{},
|
||||
ByOrgID: map[string]rbac.OrgPermissions{},
|
||||
@@ -4992,6 +4992,16 @@ func (q *querier) GetTemplatesWithFilter(ctx context.Context, arg database.GetTe
|
||||
return q.db.GetAuthorizedTemplates(ctx, arg, prep)
|
||||
}
|
||||
|
||||
func (q *querier) GetTotalChatMessageRuntimeMsInRange(ctx context.Context, arg database.GetTotalChatMessageRuntimeMsInRangeParams) (int64, error) {
|
||||
// This query exists solely to compute hb_agent_runtime_v1 usage event
|
||||
// payloads and returns a bare sum with no chat content, so it is gated
|
||||
// on usage event creation rather than on reading chats.
|
||||
if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceUsageEvent); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return q.db.GetTotalChatMessageRuntimeMsInRange(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) GetTotalUsageDCManagedAgentsV1(ctx context.Context, arg database.GetTotalUsageDCManagedAgentsV1Params) (int64, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceUsageEvent); err != nil {
|
||||
return 0, err
|
||||
@@ -6901,6 +6911,13 @@ func (q *querier) ListTasks(ctx context.Context, arg database.ListTasksParams) (
|
||||
return fetchWithPostFilter(q.auth, policy.ActionRead, q.db.ListTasks)(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) ListUsageEventCreatedAtsByTypeSince(ctx context.Context, arg database.ListUsageEventCreatedAtsByTypeSinceParams) ([]time.Time, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceUsageEvent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.db.ListUsageEventCreatedAtsByTypeSince(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) ListUserChatCompactionThresholds(ctx context.Context, userID uuid.UUID) ([]database.UserConfig, error) {
|
||||
u, err := q.db.GetUserByID(ctx, userID)
|
||||
if err != nil {
|
||||
|
||||
@@ -6717,6 +6717,27 @@ func (s *MethodTestSuite) TestUsageEvents() {
|
||||
EndDate: time.Time{},
|
||||
}).Asserts(rbac.ResourceUsageEvent, policy.ActionRead)
|
||||
}))
|
||||
|
||||
s.Run("ListUsageEventCreatedAtsByTypeSince", s.Mocked(func(db *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
params := database.ListUsageEventCreatedAtsByTypeSinceParams{
|
||||
EventType: "hb_agent_runtime_v1",
|
||||
Since: dbtime.Now(),
|
||||
}
|
||||
db.EXPECT().ListUsageEventCreatedAtsByTypeSince(gomock.Any(), params).Return([]time.Time{}, nil)
|
||||
check.Args(params).Asserts(rbac.ResourceUsageEvent, policy.ActionRead)
|
||||
}))
|
||||
|
||||
// GetTotalChatMessageRuntimeMsInRange exists solely to compute usage
|
||||
// event payloads, so it asserts usage event creation rather than chat
|
||||
// read permissions.
|
||||
s.Run("GetTotalChatMessageRuntimeMsInRange", s.Mocked(func(db *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
params := database.GetTotalChatMessageRuntimeMsInRangeParams{
|
||||
StartTime: time.Time{},
|
||||
EndTime: time.Time{},
|
||||
}
|
||||
db.EXPECT().GetTotalChatMessageRuntimeMsInRange(gomock.Any(), params).Return(int64(0), nil)
|
||||
check.Args(params).Asserts(rbac.ResourceUsageEvent, policy.ActionCreate)
|
||||
}))
|
||||
}
|
||||
|
||||
// Ensures that the prebuilds actor may never insert an api key.
|
||||
@@ -6735,6 +6756,41 @@ func TestInsertAPIKey_AsPrebuildsUser(t *testing.T) {
|
||||
require.True(t, dbauthz.IsNotAuthorizedError(err))
|
||||
}
|
||||
|
||||
// TestGetTotalChatMessageRuntimeMsInRange_HumanRolesDenied mechanically
|
||||
// checks the invariant the query's authz gate relies on: it exposes a
|
||||
// deployment-wide aggregate behind usage_event create at site scope, which no
|
||||
// human-assignable role holds. Owner is excluded from usage_event via
|
||||
// allPermsExcept in roles.go; org roles such as org-admin do carry
|
||||
// usage_event permissions, but only at org scope, which cannot satisfy a
|
||||
// site-scoped check. If either of those ever changes, this test fails.
|
||||
func TestGetTotalChatMessageRuntimeMsInRange_HumanRolesDenied(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
orgID := uuid.New()
|
||||
var roles []rbac.RoleIdentifier
|
||||
for _, role := range rbac.SiteBuiltInRoles() {
|
||||
roles = append(roles, role.Identifier)
|
||||
}
|
||||
for _, role := range rbac.OrganizationRoles(orgID) {
|
||||
roles = append(roles, role.Identifier)
|
||||
}
|
||||
require.NotEmpty(t, roles)
|
||||
|
||||
for _, role := range roles {
|
||||
subj := rbac.Subject{
|
||||
ID: uuid.NewString(),
|
||||
Roles: rbac.RoleIdentifiers{role},
|
||||
Scope: rbac.ScopeAll,
|
||||
}
|
||||
ctx := dbauthz.As(testutil.Context(t, testutil.WaitShort), subj)
|
||||
mDB := dbmock.NewMockStore(gomock.NewController(t))
|
||||
mDB.EXPECT().Wrappers().Times(1).Return([]string{})
|
||||
dbz := dbauthz.New(mDB, rbac.NewStrictAuthorizer(prometheus.NewRegistry()), slogtest.Make(t, nil), coderdtest.AccessControlStorePointer())
|
||||
_, err := dbz.GetTotalChatMessageRuntimeMsInRange(ctx, database.GetTotalChatMessageRuntimeMsInRangeParams{})
|
||||
require.True(t, dbauthz.IsNotAuthorizedError(err), "role %s must be denied", role)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MethodTestSuite) TestAIBridge() {
|
||||
s.Run("InsertAIBridgeInterception", s.Mocked(func(db *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
initID := uuid.UUID{3}
|
||||
|
||||
Reference in New Issue
Block a user