mirror of
https://github.com/coder/coder.git
synced 2026-10-16 14:48:56 +08:00
feat: add script ordering adoption telemetry (#30860)
Count `coder_script_order` declarations and literal `rule` blocks in coderd from the existing START plan in `CompletedJob.TemplateImport.Plan`, including child modules, without multiplying counts by resource or module instances. Persist these counts per template version on successful import after START plan graph validation, without changing the provisioner API. Report a deployment-wide `script_order_summary` with the number of templates using script ordering and total declaration and rule counts. Include only active, successfully imported versions of non-deleted templates so repeated imports do not inflate adoption. Elect one reporting replica per snapshot period and report explicit zeros when no templates use ordering. The summary contains only aggregate counts, without template identifiers or rule contents. Dynamic rule blocks are not counted because Terraform omits them from configuration JSON. Configuration-decoding failures are logged and skip updating the counts without independently failing the import. Script-order summary collection failures are logged and omit that summary while allowing the rest of the snapshot to proceed. Refs: https://linear.app/codercom/issue/PLAT-551
This commit is contained in:
Generated
+2
@@ -78,6 +78,8 @@ const (
|
||||
CheckSkillsUserAclIsObject CheckConstraint = "skills_user_acl_is_object" // skills
|
||||
CheckTelemetryLockEventTypeConstraint CheckConstraint = "telemetry_lock_event_type_constraint" // telemetry_locks
|
||||
CheckValidationMonotonicOrder CheckConstraint = "validation_monotonic_order" // template_version_parameters
|
||||
CheckTemplateVersionsScriptOrderDataSourceCountCheck CheckConstraint = "template_versions_script_order_data_source_count_check" // template_versions
|
||||
CheckTemplateVersionsScriptOrderRuleCountCheck CheckConstraint = "template_versions_script_order_rule_count_check" // template_versions
|
||||
CheckUsageEventTypeCheck CheckConstraint = "usage_event_type_check" // usage_events
|
||||
CheckUsageEventsAgentRuntimeHourAligned CheckConstraint = "usage_events_agent_runtime_hour_aligned" // usage_events
|
||||
CheckUserAIBudgetOverridesSpendLimitMicrosCheck CheckConstraint = "user_ai_budget_overrides_spend_limit_micros_check" // user_ai_budget_overrides
|
||||
|
||||
@@ -5023,6 +5023,13 @@ func (q *querier) GetRuntimeConfig(ctx context.Context, key string) (string, err
|
||||
return q.db.GetRuntimeConfig(ctx, key)
|
||||
}
|
||||
|
||||
func (q *querier) GetScriptOrderTelemetrySummary(ctx context.Context) (database.GetScriptOrderTelemetrySummaryRow, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTemplate.All()); err != nil {
|
||||
return database.GetScriptOrderTelemetrySummaryRow{}, err
|
||||
}
|
||||
return q.db.GetScriptOrderTelemetrySummary(ctx)
|
||||
}
|
||||
|
||||
func (q *querier) GetStaleChats(ctx context.Context, staleThreshold time.Time) ([]database.Chat, error) {
|
||||
// GetStaleChats is a system-level operation used by the chat processor for recovery.
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceChat); err != nil {
|
||||
@@ -8680,6 +8687,28 @@ func (q *querier) UpdateTemplateVersionFlagsByJobID(ctx context.Context, arg dat
|
||||
return q.db.UpdateTemplateVersionFlagsByJobID(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) UpdateTemplateVersionScriptOrderUsageByJobID(ctx context.Context, arg database.UpdateTemplateVersionScriptOrderUsageByJobIDParams) error {
|
||||
// An actor is allowed to update the template version script order usage data if they are authorized to update the template.
|
||||
tv, err := q.db.GetTemplateVersionByJobID(ctx, arg.JobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var obj rbac.Objecter
|
||||
if !tv.TemplateID.Valid {
|
||||
obj = rbac.ResourceTemplate.InOrg(tv.OrganizationID)
|
||||
} else {
|
||||
tpl, err := q.db.GetTemplateByID(ctx, tv.TemplateID.UUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obj = tpl
|
||||
}
|
||||
if err := q.authorizeContext(ctx, policy.ActionUpdate, obj); err != nil {
|
||||
return err
|
||||
}
|
||||
return q.db.UpdateTemplateVersionScriptOrderUsageByJobID(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) UpdateTemplateWorkspacesLastUsedAt(ctx context.Context, arg database.UpdateTemplateWorkspacesLastUsedAtParams) error {
|
||||
fetch := func(ctx context.Context, arg database.UpdateTemplateWorkspacesLastUsedAtParams) (database.Template, error) {
|
||||
return q.db.GetTemplateByID(ctx, arg.TemplateID)
|
||||
|
||||
@@ -3143,6 +3143,10 @@ func (s *MethodTestSuite) TestTemplate() {
|
||||
dbm.EXPECT().GetTemplateVersionsCreatedAfter(gomock.Any(), now.Add(-time.Hour)).Return([]database.TemplateVersion{}, nil).AnyTimes()
|
||||
check.Args(now.Add(-time.Hour)).Asserts(rbac.ResourceTemplate.All(), policy.ActionRead)
|
||||
}))
|
||||
s.Run("GetScriptOrderTelemetrySummary", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().GetScriptOrderTelemetrySummary(gomock.Any()).Return(database.GetScriptOrderTelemetrySummaryRow{}, nil).AnyTimes()
|
||||
check.Args().Asserts(rbac.ResourceTemplate.All(), policy.ActionRead)
|
||||
}))
|
||||
s.Run("GetTemplatesWithFilter", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
a := testutil.Fake(s.T(), faker, database.Template{})
|
||||
arg := database.GetTemplatesWithFilterParams{}
|
||||
@@ -3211,6 +3215,27 @@ func (s *MethodTestSuite) TestTemplate() {
|
||||
dbm.EXPECT().UpdateTemplateVersionFlagsByJobID(gomock.Any(), arg).Return(nil).AnyTimes()
|
||||
check.Args(arg).Asserts(t, policy.ActionUpdate)
|
||||
}))
|
||||
s.Run("UpdateTemplateVersionScriptOrderUsageByJobID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
template := testutil.Fake(s.T(), faker, database.Template{})
|
||||
version := testutil.Fake(s.T(), faker, database.TemplateVersion{TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true}})
|
||||
arg := database.UpdateTemplateVersionScriptOrderUsageByJobIDParams{
|
||||
JobID: version.JobID,
|
||||
ScriptOrderDataSourceCount: 2,
|
||||
ScriptOrderRuleCount: 3,
|
||||
}
|
||||
dbm.EXPECT().GetTemplateVersionByJobID(gomock.Any(), version.JobID).Return(version, nil).AnyTimes()
|
||||
dbm.EXPECT().GetTemplateByID(gomock.Any(), template.ID).Return(template, nil).AnyTimes()
|
||||
dbm.EXPECT().UpdateTemplateVersionScriptOrderUsageByJobID(gomock.Any(), arg).Return(nil).AnyTimes()
|
||||
check.Args(arg).Asserts(template, policy.ActionUpdate)
|
||||
}))
|
||||
s.Run("WithoutTemplate/UpdateTemplateVersionScriptOrderUsageByJobID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
version := testutil.Fake(s.T(), faker, database.TemplateVersion{})
|
||||
version.TemplateID = uuid.NullUUID{}
|
||||
arg := database.UpdateTemplateVersionScriptOrderUsageByJobIDParams{JobID: version.JobID}
|
||||
dbm.EXPECT().GetTemplateVersionByJobID(gomock.Any(), version.JobID).Return(version, nil).AnyTimes()
|
||||
dbm.EXPECT().UpdateTemplateVersionScriptOrderUsageByJobID(gomock.Any(), arg).Return(nil).AnyTimes()
|
||||
check.Args(arg).Asserts(rbac.ResourceTemplate.InOrg(version.OrganizationID), policy.ActionUpdate)
|
||||
}))
|
||||
s.Run("UpdateTemplateWorkspacesLastUsedAt", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
t1 := testutil.Fake(s.T(), faker, database.Template{})
|
||||
arg := database.UpdateTemplateWorkspacesLastUsedAtParams{TemplateID: t1.ID}
|
||||
|
||||
@@ -1680,6 +1680,14 @@ func TemplateVersion(t testing.TB, db database.Store, orig database.TemplateVers
|
||||
UpdatedAt: dbtime.Now(),
|
||||
}))
|
||||
}
|
||||
if orig.ScriptOrderDataSourceCount != 0 || orig.ScriptOrderRuleCount != 0 {
|
||||
require.NoError(t, db.UpdateTemplateVersionScriptOrderUsageByJobID(genCtx, database.UpdateTemplateVersionScriptOrderUsageByJobIDParams{
|
||||
JobID: jobID,
|
||||
ScriptOrderDataSourceCount: orig.ScriptOrderDataSourceCount,
|
||||
ScriptOrderRuleCount: orig.ScriptOrderRuleCount,
|
||||
UpdatedAt: dbtime.Now(),
|
||||
}))
|
||||
}
|
||||
|
||||
version, err = db.GetTemplateVersionByID(genCtx, versionID)
|
||||
if err != nil {
|
||||
|
||||
+16
@@ -3168,6 +3168,14 @@ func (m queryMetricsStore) GetRuntimeConfig(ctx context.Context, key string) (st
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetScriptOrderTelemetrySummary(ctx context.Context) (database.GetScriptOrderTelemetrySummaryRow, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetScriptOrderTelemetrySummary(ctx)
|
||||
m.queryLatencies.WithLabelValues("GetScriptOrderTelemetrySummary").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetScriptOrderTelemetrySummary").Inc()
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetStaleChats(ctx context.Context, staleThreshold time.Time) ([]database.Chat, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetStaleChats(ctx, staleThreshold)
|
||||
@@ -6121,6 +6129,14 @@ func (m queryMetricsStore) UpdateTemplateVersionFlagsByJobID(ctx context.Context
|
||||
return r0
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) UpdateTemplateVersionScriptOrderUsageByJobID(ctx context.Context, arg database.UpdateTemplateVersionScriptOrderUsageByJobIDParams) error {
|
||||
start := time.Now()
|
||||
r0 := m.s.UpdateTemplateVersionScriptOrderUsageByJobID(ctx, arg)
|
||||
m.queryLatencies.WithLabelValues("UpdateTemplateVersionScriptOrderUsageByJobID").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpdateTemplateVersionScriptOrderUsageByJobID").Inc()
|
||||
return r0
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) UpdateTemplateWorkspacesLastUsedAt(ctx context.Context, arg database.UpdateTemplateWorkspacesLastUsedAtParams) error {
|
||||
start := time.Now()
|
||||
r0 := m.s.UpdateTemplateWorkspacesLastUsedAt(ctx, arg)
|
||||
|
||||
Generated
+29
@@ -5949,6 +5949,21 @@ func (mr *MockStoreMockRecorder) GetRuntimeConfig(ctx, key any) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuntimeConfig", reflect.TypeOf((*MockStore)(nil).GetRuntimeConfig), ctx, key)
|
||||
}
|
||||
|
||||
// GetScriptOrderTelemetrySummary mocks base method.
|
||||
func (m *MockStore) GetScriptOrderTelemetrySummary(ctx context.Context) (database.GetScriptOrderTelemetrySummaryRow, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetScriptOrderTelemetrySummary", ctx)
|
||||
ret0, _ := ret[0].(database.GetScriptOrderTelemetrySummaryRow)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetScriptOrderTelemetrySummary indicates an expected call of GetScriptOrderTelemetrySummary.
|
||||
func (mr *MockStoreMockRecorder) GetScriptOrderTelemetrySummary(ctx any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScriptOrderTelemetrySummary", reflect.TypeOf((*MockStore)(nil).GetScriptOrderTelemetrySummary), ctx)
|
||||
}
|
||||
|
||||
// GetStaleChats mocks base method.
|
||||
func (m *MockStore) GetStaleChats(ctx context.Context, staleThreshold time.Time) ([]database.Chat, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -11570,6 +11585,20 @@ func (mr *MockStoreMockRecorder) UpdateTemplateVersionFlagsByJobID(ctx, arg any)
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTemplateVersionFlagsByJobID", reflect.TypeOf((*MockStore)(nil).UpdateTemplateVersionFlagsByJobID), ctx, arg)
|
||||
}
|
||||
|
||||
// UpdateTemplateVersionScriptOrderUsageByJobID mocks base method.
|
||||
func (m *MockStore) UpdateTemplateVersionScriptOrderUsageByJobID(ctx context.Context, arg database.UpdateTemplateVersionScriptOrderUsageByJobIDParams) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "UpdateTemplateVersionScriptOrderUsageByJobID", ctx, arg)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// UpdateTemplateVersionScriptOrderUsageByJobID indicates an expected call of UpdateTemplateVersionScriptOrderUsageByJobID.
|
||||
func (mr *MockStoreMockRecorder) UpdateTemplateVersionScriptOrderUsageByJobID(ctx, arg any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTemplateVersionScriptOrderUsageByJobID", reflect.TypeOf((*MockStore)(nil).UpdateTemplateVersionScriptOrderUsageByJobID), ctx, arg)
|
||||
}
|
||||
|
||||
// UpdateTemplateWorkspacesLastUsedAt mocks base method.
|
||||
func (m *MockStore) UpdateTemplateWorkspacesLastUsedAt(ctx context.Context, arg database.UpdateTemplateWorkspacesLastUsedAtParams) error {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
Generated
+12
-2
@@ -3304,7 +3304,7 @@ CREATE TABLE telemetry_items (
|
||||
CREATE TABLE telemetry_locks (
|
||||
event_type text NOT NULL,
|
||||
period_ending_at timestamp with time zone NOT NULL,
|
||||
CONSTRAINT telemetry_lock_event_type_constraint CHECK ((event_type = ANY (ARRAY['aibridge_interceptions_summary'::text, 'boundary_usage_summary'::text, 'user_secrets_summary'::text])))
|
||||
CONSTRAINT telemetry_lock_event_type_constraint CHECK ((event_type = ANY (ARRAY['aibridge_interceptions_summary'::text, 'boundary_usage_summary'::text, 'user_secrets_summary'::text, 'script_order_summary'::text])))
|
||||
);
|
||||
|
||||
COMMENT ON TABLE telemetry_locks IS 'Telemetry lock tracking table for deduplication of heartbeat events across replicas.';
|
||||
@@ -3491,13 +3491,21 @@ CREATE TABLE template_versions (
|
||||
message character varying(1048576) DEFAULT ''::character varying NOT NULL,
|
||||
archived boolean DEFAULT false NOT NULL,
|
||||
source_example_id text,
|
||||
has_external_agent boolean
|
||||
has_external_agent boolean,
|
||||
script_order_data_source_count integer DEFAULT 0 NOT NULL,
|
||||
script_order_rule_count integer DEFAULT 0 NOT NULL,
|
||||
CONSTRAINT template_versions_script_order_data_source_count_check CHECK ((script_order_data_source_count >= 0)),
|
||||
CONSTRAINT template_versions_script_order_rule_count_check CHECK ((script_order_rule_count >= 0))
|
||||
);
|
||||
|
||||
COMMENT ON COLUMN template_versions.external_auth_providers IS 'IDs of External auth providers for a specific template version';
|
||||
|
||||
COMMENT ON COLUMN template_versions.message IS 'Message describing the changes in this version of the template, similar to a Git commit message. Like a commit message, this should be a short, high-level description of the changes in this version of the template. This message is immutable and should not be updated after the fact.';
|
||||
|
||||
COMMENT ON COLUMN template_versions.script_order_data_source_count IS 'Number of coder_script_order data source declarations in the template configuration.';
|
||||
|
||||
COMMENT ON COLUMN template_versions.script_order_rule_count IS 'Number of rule blocks across coder_script_order declarations in the template configuration.';
|
||||
|
||||
CREATE VIEW template_version_with_user AS
|
||||
SELECT template_versions.id,
|
||||
template_versions.template_id,
|
||||
@@ -3513,6 +3521,8 @@ CREATE VIEW template_version_with_user AS
|
||||
template_versions.archived,
|
||||
template_versions.source_example_id,
|
||||
template_versions.has_external_agent,
|
||||
template_versions.script_order_data_source_count,
|
||||
template_versions.script_order_rule_count,
|
||||
COALESCE(visible_users.avatar_url, ''::text) AS created_by_avatar_url,
|
||||
COALESCE(visible_users.username, ''::text) AS created_by_username,
|
||||
COALESCE(visible_users.name, ''::text) AS created_by_name
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
DELETE FROM telemetry_locks WHERE event_type = 'script_order_summary';
|
||||
|
||||
ALTER TABLE telemetry_locks DROP CONSTRAINT telemetry_lock_event_type_constraint;
|
||||
ALTER TABLE telemetry_locks ADD CONSTRAINT telemetry_lock_event_type_constraint
|
||||
CHECK (event_type IN ('aibridge_interceptions_summary', 'boundary_usage_summary', 'user_secrets_summary'));
|
||||
|
||||
-- Drop the view first because it depends on the columns being removed.
|
||||
DROP VIEW template_version_with_user;
|
||||
|
||||
ALTER TABLE template_versions
|
||||
DROP COLUMN script_order_rule_count,
|
||||
DROP COLUMN script_order_data_source_count;
|
||||
|
||||
CREATE VIEW template_version_with_user AS
|
||||
SELECT
|
||||
template_versions.id,
|
||||
template_versions.template_id,
|
||||
template_versions.organization_id,
|
||||
template_versions.created_at,
|
||||
template_versions.updated_at,
|
||||
template_versions.name,
|
||||
template_versions.readme,
|
||||
template_versions.job_id,
|
||||
template_versions.created_by,
|
||||
template_versions.external_auth_providers,
|
||||
template_versions.message,
|
||||
template_versions.archived,
|
||||
template_versions.source_example_id,
|
||||
template_versions.has_external_agent,
|
||||
COALESCE(visible_users.avatar_url, ''::text) AS created_by_avatar_url,
|
||||
COALESCE(visible_users.username, ''::text) AS created_by_username,
|
||||
COALESCE(visible_users.name, ''::text) AS created_by_name
|
||||
FROM template_versions
|
||||
LEFT JOIN visible_users ON template_versions.created_by = visible_users.id;
|
||||
|
||||
COMMENT ON VIEW template_version_with_user IS 'Joins in the username + avatar url of the created by user.';
|
||||
@@ -0,0 +1,44 @@
|
||||
ALTER TABLE template_versions
|
||||
ADD COLUMN script_order_data_source_count INTEGER NOT NULL DEFAULT 0 CHECK (script_order_data_source_count >= 0),
|
||||
ADD COLUMN script_order_rule_count INTEGER NOT NULL DEFAULT 0 CHECK (script_order_rule_count >= 0);
|
||||
|
||||
COMMENT ON COLUMN template_versions.script_order_data_source_count IS
|
||||
'Number of coder_script_order data source declarations in the template configuration.';
|
||||
|
||||
COMMENT ON COLUMN template_versions.script_order_rule_count IS
|
||||
'Number of rule blocks across coder_script_order declarations in the template configuration.';
|
||||
|
||||
-- The view must expose every base-table column to keep the generated
|
||||
-- models consistent, as enforced by TestViewSubsetTemplateVersion.
|
||||
DROP VIEW template_version_with_user;
|
||||
|
||||
CREATE VIEW template_version_with_user AS
|
||||
SELECT
|
||||
template_versions.id,
|
||||
template_versions.template_id,
|
||||
template_versions.organization_id,
|
||||
template_versions.created_at,
|
||||
template_versions.updated_at,
|
||||
template_versions.name,
|
||||
template_versions.readme,
|
||||
template_versions.job_id,
|
||||
template_versions.created_by,
|
||||
template_versions.external_auth_providers,
|
||||
template_versions.message,
|
||||
template_versions.archived,
|
||||
template_versions.source_example_id,
|
||||
template_versions.has_external_agent,
|
||||
template_versions.script_order_data_source_count,
|
||||
template_versions.script_order_rule_count,
|
||||
COALESCE(visible_users.avatar_url, ''::text) AS created_by_avatar_url,
|
||||
COALESCE(visible_users.username, ''::text) AS created_by_username,
|
||||
COALESCE(visible_users.name, ''::text) AS created_by_name
|
||||
FROM template_versions
|
||||
LEFT JOIN visible_users ON template_versions.created_by = visible_users.id;
|
||||
|
||||
COMMENT ON VIEW template_version_with_user IS 'Joins in the username + avatar url of the created by user.';
|
||||
|
||||
-- Elect one replica per snapshot period to report script-order adoption.
|
||||
ALTER TABLE telemetry_locks DROP CONSTRAINT telemetry_lock_event_type_constraint;
|
||||
ALTER TABLE telemetry_locks ADD CONSTRAINT telemetry_lock_event_type_constraint
|
||||
CHECK (event_type IN ('aibridge_interceptions_summary', 'boundary_usage_summary', 'user_secrets_summary', 'script_order_summary'));
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
"github.com/coder/coder/v2/coderd/database/migrations"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
@@ -406,6 +407,7 @@ func TestMigrationChain(t *testing.T) {
|
||||
{"Migration000602RestoreAgentsAccessDefaultRole", 602, testMigration000602RestoreAgentsAccessDefaultRole},
|
||||
{"Migration000606ChatDiffStatusOriginCredentials", 606, testMigration000606ChatDiffStatusOriginCredentials},
|
||||
{"Migration000621Skills", 621, testMigration000621Skills},
|
||||
{"Migration000625TemplateVersionScriptOrderUsage", 625, testMigration000625TemplateVersionScriptOrderUsage},
|
||||
}
|
||||
for _, step := range steps {
|
||||
stepTo(step.version - 1)
|
||||
@@ -4477,3 +4479,117 @@ func TestMigration000599OAuth2RedirectURIsPrimary(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assertRows()
|
||||
}
|
||||
|
||||
func testMigration000625TemplateVersionScriptOrderUsage(t *testing.T, sqlDB *sql.DB, next migrationStepper) {
|
||||
const migrationVersion = 625
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
db := database.New(sqlDB)
|
||||
org, err := db.GetDefaultOrganization(ctx)
|
||||
require.NoError(t, err)
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
// GIVEN: An existing template version and a user-secrets telemetry lock.
|
||||
// (Avoid dbgen.TemplateVersion: its view query expects the new columns.)
|
||||
legacyID := uuid.New()
|
||||
err = db.InsertTemplateVersion(ctx, database.InsertTemplateVersionParams{
|
||||
ID: legacyID,
|
||||
OrganizationID: org.ID,
|
||||
CreatedBy: user.ID,
|
||||
JobID: uuid.New(),
|
||||
Name: testutil.GetRandomName(t),
|
||||
CreatedAt: dbtime.Now(),
|
||||
UpdatedAt: dbtime.Now(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// An existing event type's lock must survive both upgrade and rollback.
|
||||
legacyLock := database.InsertTelemetryLockParams{
|
||||
EventType: "user_secrets_summary",
|
||||
PeriodEndingAt: dbtime.Now(),
|
||||
}
|
||||
require.NoError(t, db.InsertTelemetryLock(ctx, legacyLock))
|
||||
|
||||
// WHEN: The migration is applied.
|
||||
version, more, err := next()
|
||||
require.NoError(t, err)
|
||||
require.True(t, more)
|
||||
require.EqualValues(t, migrationVersion, version)
|
||||
|
||||
// THEN: Script-order locks are accepted, existing and new versions default
|
||||
// to zero counts, and only valid counts can be stored.
|
||||
scriptOrderLock := database.InsertTelemetryLockParams{
|
||||
EventType: "script_order_summary",
|
||||
PeriodEndingAt: legacyLock.PeriodEndingAt,
|
||||
}
|
||||
require.NoError(t, db.InsertTelemetryLock(ctx, scriptOrderLock))
|
||||
|
||||
legacyVersion, err := db.GetTemplateVersionByID(ctx, legacyID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, legacyVersion.ScriptOrderDataSourceCount)
|
||||
require.Zero(t, legacyVersion.ScriptOrderRuleCount)
|
||||
|
||||
// An older provisioner omits the counts, so new rows also default to zero.
|
||||
newVersion := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
||||
OrganizationID: org.ID,
|
||||
CreatedBy: user.ID,
|
||||
})
|
||||
require.Zero(t, newVersion.ScriptOrderDataSourceCount)
|
||||
require.Zero(t, newVersion.ScriptOrderRuleCount)
|
||||
|
||||
for _, tt := range []struct {
|
||||
query string
|
||||
constraint database.CheckConstraint
|
||||
}{
|
||||
{"UPDATE template_versions SET script_order_data_source_count = -1 WHERE id = $1", database.CheckTemplateVersionsScriptOrderDataSourceCountCheck},
|
||||
{"UPDATE template_versions SET script_order_rule_count = -1 WHERE id = $1", database.CheckTemplateVersionsScriptOrderRuleCountCheck},
|
||||
} {
|
||||
_, err := sqlDB.ExecContext(ctx, tt.query, legacyVersion.ID)
|
||||
require.True(t, database.IsCheckViolation(err, tt.constraint), "%s: %v", tt.query, err)
|
||||
}
|
||||
|
||||
for _, query := range []string{
|
||||
"UPDATE template_versions SET script_order_data_source_count = NULL WHERE id = $1",
|
||||
"UPDATE template_versions SET script_order_rule_count = NULL WHERE id = $1",
|
||||
} {
|
||||
_, err := sqlDB.ExecContext(ctx, query, legacyVersion.ID)
|
||||
var pqErr *pq.Error
|
||||
require.ErrorAs(t, err, &pqErr, "%s: %v", query, err)
|
||||
require.Equal(t, "not_null_violation", pqErr.Code.Name(), "%s: %v", query, err)
|
||||
}
|
||||
|
||||
err = db.UpdateTemplateVersionScriptOrderUsageByJobID(ctx, database.UpdateTemplateVersionScriptOrderUsageByJobIDParams{
|
||||
JobID: legacyVersion.JobID,
|
||||
ScriptOrderDataSourceCount: 2,
|
||||
ScriptOrderRuleCount: 3,
|
||||
UpdatedAt: dbtime.Now(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
updated, err := db.GetTemplateVersionByID(ctx, legacyVersion.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int32(2), updated.ScriptOrderDataSourceCount)
|
||||
require.Equal(t, int32(3), updated.ScriptOrderRuleCount)
|
||||
|
||||
// Roll back the migration to verify script-order locks are rejected.
|
||||
down, err := os.ReadFile("000625_template_version_script_order_usage.down.sql")
|
||||
require.NoError(t, err)
|
||||
_, err = sqlDB.ExecContext(ctx, string(down))
|
||||
require.NoError(t, err)
|
||||
err = db.InsertTelemetryLock(ctx, scriptOrderLock)
|
||||
require.True(t, database.IsCheckViolation(err, database.CheckTelemetryLockEventTypeConstraint), "unexpected error: %v", err)
|
||||
|
||||
// Reapply the migration to verify script-order locks are accepted again
|
||||
// and the pre-existing lock survives rollback and reapplication.
|
||||
up, err := os.ReadFile("000625_template_version_script_order_usage.up.sql")
|
||||
require.NoError(t, err)
|
||||
_, err = sqlDB.ExecContext(ctx, string(up))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.InsertTelemetryLock(ctx, scriptOrderLock))
|
||||
err = db.InsertTelemetryLock(ctx, legacyLock)
|
||||
require.True(t, database.IsUniqueViolation(err, database.UniqueTelemetryLocksPkey), "unexpected error: %v", err)
|
||||
|
||||
updated, err = db.GetTemplateVersionByID(ctx, legacyVersion.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, legacyVersion.Name, updated.Name)
|
||||
require.Zero(t, updated.ScriptOrderDataSourceCount)
|
||||
require.Zero(t, updated.ScriptOrderRuleCount)
|
||||
}
|
||||
|
||||
Generated
+23
-17
@@ -6352,23 +6352,25 @@ type TemplateUsageStatsSessionApp struct {
|
||||
|
||||
// Joins in the username + avatar url of the created by user.
|
||||
type TemplateVersion struct {
|
||||
ID uuid.UUID `db:"id" json:"id"`
|
||||
TemplateID uuid.NullUUID `db:"template_id" json:"template_id"`
|
||||
OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"`
|
||||
CreatedAt time.Time `db:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
|
||||
Name string `db:"name" json:"name"`
|
||||
Readme string `db:"readme" json:"readme"`
|
||||
JobID uuid.UUID `db:"job_id" json:"job_id"`
|
||||
CreatedBy uuid.UUID `db:"created_by" json:"created_by"`
|
||||
ExternalAuthProviders json.RawMessage `db:"external_auth_providers" json:"external_auth_providers"`
|
||||
Message string `db:"message" json:"message"`
|
||||
Archived bool `db:"archived" json:"archived"`
|
||||
SourceExampleID sql.NullString `db:"source_example_id" json:"source_example_id"`
|
||||
HasExternalAgent sql.NullBool `db:"has_external_agent" json:"has_external_agent"`
|
||||
CreatedByAvatarURL string `db:"created_by_avatar_url" json:"created_by_avatar_url"`
|
||||
CreatedByUsername string `db:"created_by_username" json:"created_by_username"`
|
||||
CreatedByName string `db:"created_by_name" json:"created_by_name"`
|
||||
ID uuid.UUID `db:"id" json:"id"`
|
||||
TemplateID uuid.NullUUID `db:"template_id" json:"template_id"`
|
||||
OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"`
|
||||
CreatedAt time.Time `db:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
|
||||
Name string `db:"name" json:"name"`
|
||||
Readme string `db:"readme" json:"readme"`
|
||||
JobID uuid.UUID `db:"job_id" json:"job_id"`
|
||||
CreatedBy uuid.UUID `db:"created_by" json:"created_by"`
|
||||
ExternalAuthProviders json.RawMessage `db:"external_auth_providers" json:"external_auth_providers"`
|
||||
Message string `db:"message" json:"message"`
|
||||
Archived bool `db:"archived" json:"archived"`
|
||||
SourceExampleID sql.NullString `db:"source_example_id" json:"source_example_id"`
|
||||
HasExternalAgent sql.NullBool `db:"has_external_agent" json:"has_external_agent"`
|
||||
ScriptOrderDataSourceCount int32 `db:"script_order_data_source_count" json:"script_order_data_source_count"`
|
||||
ScriptOrderRuleCount int32 `db:"script_order_rule_count" json:"script_order_rule_count"`
|
||||
CreatedByAvatarURL string `db:"created_by_avatar_url" json:"created_by_avatar_url"`
|
||||
CreatedByUsername string `db:"created_by_username" json:"created_by_username"`
|
||||
CreatedByName string `db:"created_by_name" json:"created_by_name"`
|
||||
}
|
||||
|
||||
type TemplateVersionParameter struct {
|
||||
@@ -6457,6 +6459,10 @@ type TemplateVersionTable struct {
|
||||
Archived bool `db:"archived" json:"archived"`
|
||||
SourceExampleID sql.NullString `db:"source_example_id" json:"source_example_id"`
|
||||
HasExternalAgent sql.NullBool `db:"has_external_agent" json:"has_external_agent"`
|
||||
// Number of coder_script_order data source declarations in the template configuration.
|
||||
ScriptOrderDataSourceCount int32 `db:"script_order_data_source_count" json:"script_order_data_source_count"`
|
||||
// Number of rule blocks across coder_script_order declarations in the template configuration.
|
||||
ScriptOrderRuleCount int32 `db:"script_order_rule_count" json:"script_order_rule_count"`
|
||||
}
|
||||
|
||||
type TemplateVersionTerraformValue struct {
|
||||
|
||||
Generated
+4
@@ -885,6 +885,9 @@ type sqlcQuerier interface {
|
||||
GetReplicasUpdatedAfter(ctx context.Context, updatedAt time.Time) ([]Replica, error)
|
||||
GetRunningPrebuiltWorkspaces(ctx context.Context) ([]GetRunningPrebuiltWorkspacesRow, error)
|
||||
GetRuntimeConfig(ctx context.Context, key string) (string, error)
|
||||
// Count only the active, successfully imported version of each non-deleted
|
||||
// template, so repeated imports do not inflate adoption.
|
||||
GetScriptOrderTelemetrySummary(ctx context.Context) (GetScriptOrderTelemetrySummaryRow, error)
|
||||
// Find chats that appear stuck and need recovery:
|
||||
// 1. Running chats whose heartbeat has expired (worker crash).
|
||||
// 2. requires_action chats past the timeout threshold (client
|
||||
@@ -1719,6 +1722,7 @@ type sqlcQuerier interface {
|
||||
UpdateTemplateVersionDescriptionByJobID(ctx context.Context, arg UpdateTemplateVersionDescriptionByJobIDParams) error
|
||||
UpdateTemplateVersionExternalAuthProvidersByJobID(ctx context.Context, arg UpdateTemplateVersionExternalAuthProvidersByJobIDParams) error
|
||||
UpdateTemplateVersionFlagsByJobID(ctx context.Context, arg UpdateTemplateVersionFlagsByJobIDParams) error
|
||||
UpdateTemplateVersionScriptOrderUsageByJobID(ctx context.Context, arg UpdateTemplateVersionScriptOrderUsageByJobIDParams) error
|
||||
UpdateTemplateWorkspacesLastUsedAt(ctx context.Context, arg UpdateTemplateWorkspacesLastUsedAtParams) error
|
||||
UpdateUsageEventsPostPublish(ctx context.Context, arg UpdateUsageEventsPostPublishParams) error
|
||||
UpdateUserAIProviderKey(ctx context.Context, arg UpdateUserAIProviderKeyParams) (UserAIProviderKey, error)
|
||||
|
||||
Generated
+81
-8
@@ -30617,7 +30617,7 @@ FROM
|
||||
-- Scope an archive to a single template and ignore already archived template versions
|
||||
(
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count
|
||||
FROM
|
||||
template_versions
|
||||
WHERE
|
||||
@@ -30718,7 +30718,7 @@ func (q *sqlQuerier) ArchiveUnusedTemplateVersions(ctx context.Context, arg Arch
|
||||
|
||||
const getPreviousTemplateVersion = `-- name: GetPreviousTemplateVersion :one
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -30757,6 +30757,8 @@ func (q *sqlQuerier) GetPreviousTemplateVersion(ctx context.Context, arg GetPrev
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -30764,9 +30766,40 @@ func (q *sqlQuerier) GetPreviousTemplateVersion(ctx context.Context, arg GetPrev
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getScriptOrderTelemetrySummary = `-- name: GetScriptOrderTelemetrySummary :one
|
||||
SELECT
|
||||
COUNT(*) AS templates_with_script_order,
|
||||
COALESCE(SUM(tv.script_order_data_source_count), 0)::bigint AS data_source_count,
|
||||
COALESCE(SUM(tv.script_order_rule_count), 0)::bigint AS rule_count
|
||||
FROM templates t
|
||||
JOIN template_versions tv ON tv.id = t.active_version_id
|
||||
JOIN provisioner_jobs pj ON pj.id = tv.job_id
|
||||
WHERE t.deleted = false
|
||||
AND tv.script_order_data_source_count > 0
|
||||
AND pj.type = 'template_version_import'
|
||||
AND pj.completed_at IS NOT NULL
|
||||
AND COALESCE(pj.error, '') = ''
|
||||
AND pj.canceled_at IS NULL
|
||||
`
|
||||
|
||||
type GetScriptOrderTelemetrySummaryRow struct {
|
||||
TemplatesWithScriptOrder int64 `db:"templates_with_script_order" json:"templates_with_script_order"`
|
||||
DataSourceCount int64 `db:"data_source_count" json:"data_source_count"`
|
||||
RuleCount int64 `db:"rule_count" json:"rule_count"`
|
||||
}
|
||||
|
||||
// Count only the active, successfully imported version of each non-deleted
|
||||
// template, so repeated imports do not inflate adoption.
|
||||
func (q *sqlQuerier) GetScriptOrderTelemetrySummary(ctx context.Context) (GetScriptOrderTelemetrySummaryRow, error) {
|
||||
row := q.db.QueryRowContext(ctx, getScriptOrderTelemetrySummary)
|
||||
var i GetScriptOrderTelemetrySummaryRow
|
||||
err := row.Scan(&i.TemplatesWithScriptOrder, &i.DataSourceCount, &i.RuleCount)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getTemplateVersionByID = `-- name: GetTemplateVersionByID :one
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -30791,6 +30824,8 @@ func (q *sqlQuerier) GetTemplateVersionByID(ctx context.Context, id uuid.UUID) (
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -30800,7 +30835,7 @@ func (q *sqlQuerier) GetTemplateVersionByID(ctx context.Context, id uuid.UUID) (
|
||||
|
||||
const getTemplateVersionByJobID = `-- name: GetTemplateVersionByJobID :one
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -30825,6 +30860,8 @@ func (q *sqlQuerier) GetTemplateVersionByJobID(ctx context.Context, jobID uuid.U
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -30834,7 +30871,7 @@ func (q *sqlQuerier) GetTemplateVersionByJobID(ctx context.Context, jobID uuid.U
|
||||
|
||||
const getTemplateVersionByTemplateIDAndName = `-- name: GetTemplateVersionByTemplateIDAndName :one
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -30865,6 +30902,8 @@ func (q *sqlQuerier) GetTemplateVersionByTemplateIDAndName(ctx context.Context,
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -30874,7 +30913,7 @@ func (q *sqlQuerier) GetTemplateVersionByTemplateIDAndName(ctx context.Context,
|
||||
|
||||
const getTemplateVersionsByIDs = `-- name: GetTemplateVersionsByIDs :many
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -30905,6 +30944,8 @@ func (q *sqlQuerier) GetTemplateVersionsByIDs(ctx context.Context, ids []uuid.UU
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -30924,7 +30965,7 @@ func (q *sqlQuerier) GetTemplateVersionsByIDs(ctx context.Context, ids []uuid.UU
|
||||
|
||||
const getTemplateVersionsByTemplateID = `-- name: GetTemplateVersionsByTemplateID :many
|
||||
SELECT
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name
|
||||
id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name
|
||||
FROM
|
||||
template_version_with_user AS template_versions
|
||||
WHERE
|
||||
@@ -31002,6 +31043,8 @@ func (q *sqlQuerier) GetTemplateVersionsByTemplateID(ctx context.Context, arg Ge
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -31020,7 +31063,7 @@ func (q *sqlQuerier) GetTemplateVersionsByTemplateID(ctx context.Context, arg Ge
|
||||
}
|
||||
|
||||
const getTemplateVersionsCreatedAfter = `-- name: GetTemplateVersionsCreatedAfter :many
|
||||
SELECT id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, created_by_avatar_url, created_by_username, created_by_name FROM template_version_with_user AS template_versions WHERE created_at > $1
|
||||
SELECT id, template_id, organization_id, created_at, updated_at, name, readme, job_id, created_by, external_auth_providers, message, archived, source_example_id, has_external_agent, script_order_data_source_count, script_order_rule_count, created_by_avatar_url, created_by_username, created_by_name FROM template_version_with_user AS template_versions WHERE created_at > $1
|
||||
`
|
||||
|
||||
func (q *sqlQuerier) GetTemplateVersionsCreatedAfter(ctx context.Context, createdAt time.Time) ([]TemplateVersion, error) {
|
||||
@@ -31047,6 +31090,8 @@ func (q *sqlQuerier) GetTemplateVersionsCreatedAfter(ctx context.Context, create
|
||||
&i.Archived,
|
||||
&i.SourceExampleID,
|
||||
&i.HasExternalAgent,
|
||||
&i.ScriptOrderDataSourceCount,
|
||||
&i.ScriptOrderRuleCount,
|
||||
&i.CreatedByAvatarURL,
|
||||
&i.CreatedByUsername,
|
||||
&i.CreatedByName,
|
||||
@@ -31229,6 +31274,34 @@ func (q *sqlQuerier) UpdateTemplateVersionFlagsByJobID(ctx context.Context, arg
|
||||
return err
|
||||
}
|
||||
|
||||
const updateTemplateVersionScriptOrderUsageByJobID = `-- name: UpdateTemplateVersionScriptOrderUsageByJobID :exec
|
||||
UPDATE
|
||||
template_versions
|
||||
SET
|
||||
script_order_data_source_count = $2,
|
||||
script_order_rule_count = $3,
|
||||
updated_at = $4
|
||||
WHERE
|
||||
job_id = $1
|
||||
`
|
||||
|
||||
type UpdateTemplateVersionScriptOrderUsageByJobIDParams struct {
|
||||
JobID uuid.UUID `db:"job_id" json:"job_id"`
|
||||
ScriptOrderDataSourceCount int32 `db:"script_order_data_source_count" json:"script_order_data_source_count"`
|
||||
ScriptOrderRuleCount int32 `db:"script_order_rule_count" json:"script_order_rule_count"`
|
||||
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
func (q *sqlQuerier) UpdateTemplateVersionScriptOrderUsageByJobID(ctx context.Context, arg UpdateTemplateVersionScriptOrderUsageByJobIDParams) error {
|
||||
_, err := q.db.ExecContext(ctx, updateTemplateVersionScriptOrderUsageByJobID,
|
||||
arg.JobID,
|
||||
arg.ScriptOrderDataSourceCount,
|
||||
arg.ScriptOrderRuleCount,
|
||||
arg.UpdatedAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const getTemplateVersionTerraformValues = `-- name: GetTemplateVersionTerraformValues :one
|
||||
SELECT
|
||||
template_version_terraform_values.template_version_id, template_version_terraform_values.updated_at, template_version_terraform_values.cached_plan, template_version_terraform_values.cached_module_files, template_version_terraform_values.provisionerd_version
|
||||
|
||||
@@ -234,3 +234,30 @@ SET
|
||||
updated_at = $3
|
||||
WHERE
|
||||
job_id = $1;
|
||||
|
||||
-- name: UpdateTemplateVersionScriptOrderUsageByJobID :exec
|
||||
UPDATE
|
||||
template_versions
|
||||
SET
|
||||
script_order_data_source_count = $2,
|
||||
script_order_rule_count = $3,
|
||||
updated_at = $4
|
||||
WHERE
|
||||
job_id = $1;
|
||||
|
||||
-- name: GetScriptOrderTelemetrySummary :one
|
||||
-- Count only the active, successfully imported version of each non-deleted
|
||||
-- template, so repeated imports do not inflate adoption.
|
||||
SELECT
|
||||
COUNT(*) AS templates_with_script_order,
|
||||
COALESCE(SUM(tv.script_order_data_source_count), 0)::bigint AS data_source_count,
|
||||
COALESCE(SUM(tv.script_order_rule_count), 0)::bigint AS rule_count
|
||||
FROM templates t
|
||||
JOIN template_versions tv ON tv.id = t.active_version_id
|
||||
JOIN provisioner_jobs pj ON pj.id = tv.job_id
|
||||
WHERE t.deleted = false
|
||||
AND tv.script_order_data_source_count > 0
|
||||
AND pj.type = 'template_version_import'
|
||||
AND pj.completed_at IS NOT NULL
|
||||
AND COALESCE(pj.error, '') = ''
|
||||
AND pj.canceled_at IS NULL;
|
||||
|
||||
@@ -1874,6 +1874,15 @@ func (s *server) completeTemplateImportJob(ctx context.Context, job database.Pro
|
||||
return xerrors.Errorf("template version ID is expected: %w", err)
|
||||
}
|
||||
|
||||
// Keep plan parsing and traversal outside the transaction to avoid
|
||||
// extending its duration for large plans.
|
||||
// Failure to count script-order usage must not prevent import completion.
|
||||
orderUsage, orderUsageErr := countScriptOrderUsage(jobType.TemplateImport.Plan)
|
||||
if orderUsageErr != nil {
|
||||
s.Logger.Warn(ctx, "count script order usage, skipping",
|
||||
slog.F("job_id", jobID), slog.Error(orderUsageErr))
|
||||
}
|
||||
|
||||
// Execute all database operations in a transaction
|
||||
return s.Database.InTx(func(db database.Store) error {
|
||||
now := s.timeNow()
|
||||
@@ -2049,6 +2058,17 @@ func (s *server) completeTemplateImportJob(ctx context.Context, job database.Pro
|
||||
if err != nil {
|
||||
return xerrors.Errorf("update template version external agent: %w", err)
|
||||
}
|
||||
if orderUsageErr == nil {
|
||||
err = db.UpdateTemplateVersionScriptOrderUsageByJobID(ctx, database.UpdateTemplateVersionScriptOrderUsageByJobIDParams{
|
||||
JobID: jobID,
|
||||
ScriptOrderDataSourceCount: orderUsage.dataSourceCount,
|
||||
ScriptOrderRuleCount: orderUsage.ruleCount,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("update template version script order usage: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Process terraform values
|
||||
plan := jobType.TemplateImport.Plan
|
||||
|
||||
@@ -1803,7 +1803,10 @@ func TestCompleteJob(t *testing.T) {
|
||||
Name: "test-resource",
|
||||
Type: "aws_instance",
|
||||
}},
|
||||
Plan: []byte("{}"),
|
||||
Plan: []byte(`{"configuration":{"root_module":{"resources":[
|
||||
{"mode":"data","type":"coder_script_order","expressions":{"rule":[{},{}]}},
|
||||
{"mode":"data","type":"coder_script_order","expressions":{"rule":[{}]}}
|
||||
]}}}`),
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -1819,6 +1822,12 @@ func TestCompleteJob(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resources, 1, "Expected one resource to be created")
|
||||
require.Equal(t, "test-resource", resources[0].Name)
|
||||
|
||||
// Verify script-order counts were persisted.
|
||||
version, err := db.GetTemplateVersionByID(ctx, versionID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int32(2), version.ScriptOrderDataSourceCount)
|
||||
require.Equal(t, int32(3), version.ScriptOrderRuleCount)
|
||||
})
|
||||
|
||||
// Test TemplateDryRun transaction
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package provisionerdserver
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
tfjson "github.com/hashicorp/terraform-json"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
type scriptOrderUsage struct {
|
||||
dataSourceCount int32
|
||||
ruleCount int32
|
||||
}
|
||||
|
||||
// countScriptOrderUsage counts declarations in the supplied plan configuration,
|
||||
// so count, for_each, and module instance counts do not inflate usage.
|
||||
func countScriptOrderUsage(planJSON []byte) (scriptOrderUsage, error) {
|
||||
var usage scriptOrderUsage
|
||||
if len(planJSON) == 0 {
|
||||
return usage, nil
|
||||
}
|
||||
var plan struct {
|
||||
Config *tfjson.Config `json:"configuration"`
|
||||
}
|
||||
if err := json.Unmarshal(planJSON, &plan); err != nil {
|
||||
return usage, xerrors.Errorf("decode plan configuration: %w", err)
|
||||
}
|
||||
if plan.Config != nil {
|
||||
countScriptOrderUsageInModule(plan.Config.RootModule, &usage)
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
func countScriptOrderUsageInModule(module *tfjson.ConfigModule, usage *scriptOrderUsage) {
|
||||
if module == nil {
|
||||
return
|
||||
}
|
||||
for _, r := range module.Resources {
|
||||
if r == nil || r.Mode != tfjson.DataResourceMode || r.Type != "coder_script_order" {
|
||||
continue
|
||||
}
|
||||
usage.dataSourceCount++
|
||||
// Terraform's configuration JSON omits dynamic blocks, so only literal
|
||||
// rule blocks contribute to this count.
|
||||
if rules := r.Expressions["rule"]; rules != nil {
|
||||
// #nosec G115 - Terraform rule counts are expected to fit in int32.
|
||||
usage.ruleCount += int32(len(rules.NestedBlocks))
|
||||
}
|
||||
}
|
||||
for _, call := range module.ModuleCalls {
|
||||
if call != nil {
|
||||
countScriptOrderUsageInModule(call.Module, usage)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package provisionerdserver
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
tfjson "github.com/hashicorp/terraform-json"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCountScriptOrderUsage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
plan string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "MissingPlan"},
|
||||
{name: "NullPlan", plan: "null"},
|
||||
{name: "MissingConfiguration", plan: "{}"},
|
||||
{name: "EmptyConfiguration", plan: `{"configuration":{}}`},
|
||||
{name: "InvalidJSON", plan: "{", wantErr: true},
|
||||
{name: "InvalidConfiguration", plan: `{"configuration":[]}`, wantErr: true},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
usage, err := countScriptOrderUsage([]byte(tt.plan))
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.Empty(t, usage)
|
||||
})
|
||||
}
|
||||
|
||||
checkUsage := func(t *testing.T, config *tfjson.Config, want scriptOrderUsage) {
|
||||
t.Helper()
|
||||
// The provisioner marshals a decoded plan before forwarding it to coderd.
|
||||
planJSON, err := json.Marshal(&tfjson.Plan{Config: config})
|
||||
require.NoError(t, err)
|
||||
usage, err := countScriptOrderUsage(planJSON)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, want, usage)
|
||||
}
|
||||
|
||||
declaration := func(ruleCount int) *tfjson.ConfigResource {
|
||||
return &tfjson.ConfigResource{
|
||||
Mode: tfjson.DataResourceMode,
|
||||
Type: "coder_script_order",
|
||||
Expressions: map[string]*tfjson.Expression{
|
||||
"rule": {ExpressionData: &tfjson.ExpressionData{
|
||||
NestedBlocks: make([]map[string]*tfjson.Expression, ruleCount),
|
||||
}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("ConfigurationDeclarationsNotInstances", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Three declarations (root, child, nested) contain 2 + 3 + 1 rules.
|
||||
// Resource and module repetition must not multiply either total.
|
||||
rootOrder := declaration(2)
|
||||
rootOrder.ForEachExpression = &tfjson.Expression{ExpressionData: &tfjson.ExpressionData{
|
||||
ConstantValue: map[string]string{"first": "", "second": ""},
|
||||
}}
|
||||
childOrder := declaration(3)
|
||||
childOrder.CountExpression = &tfjson.Expression{ExpressionData: &tfjson.ExpressionData{
|
||||
ConstantValue: 3, // Three resource instances per module instance.
|
||||
}}
|
||||
config := &tfjson.Config{RootModule: &tfjson.ConfigModule{
|
||||
Resources: []*tfjson.ConfigResource{
|
||||
rootOrder,
|
||||
nil,
|
||||
{Mode: tfjson.ManagedResourceMode, Type: "coder_script_order"},
|
||||
{Mode: tfjson.DataResourceMode, Type: "coder_parameter"},
|
||||
},
|
||||
ModuleCalls: map[string]*tfjson.ModuleCall{
|
||||
"repeated": {
|
||||
ForEachExpression: &tfjson.Expression{ExpressionData: &tfjson.ExpressionData{
|
||||
ConstantValue: map[string]string{"east": "", "west": ""},
|
||||
}},
|
||||
Module: &tfjson.ConfigModule{
|
||||
Resources: []*tfjson.ConfigResource{childOrder},
|
||||
ModuleCalls: map[string]*tfjson.ModuleCall{
|
||||
"nested": {
|
||||
CountExpression: &tfjson.Expression{ExpressionData: &tfjson.ExpressionData{
|
||||
ConstantValue: 2,
|
||||
}},
|
||||
Module: &tfjson.ConfigModule{
|
||||
Resources: []*tfjson.ConfigResource{declaration(1)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"nil": nil,
|
||||
"empty": {},
|
||||
},
|
||||
}}
|
||||
checkUsage(t, config, scriptOrderUsage{dataSourceCount: 3, ruleCount: 6})
|
||||
})
|
||||
|
||||
t.Run("DeclarationWithZeroInstances", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
order := declaration(2)
|
||||
order.CountExpression = &tfjson.Expression{ExpressionData: &tfjson.ExpressionData{ConstantValue: 0}}
|
||||
config := &tfjson.Config{RootModule: &tfjson.ConfigModule{
|
||||
Resources: []*tfjson.ConfigResource{order},
|
||||
}}
|
||||
checkUsage(t, config, scriptOrderUsage{dataSourceCount: 1, ruleCount: 2})
|
||||
})
|
||||
|
||||
t.Run("NoLiteralRules", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &tfjson.Config{RootModule: &tfjson.ConfigModule{
|
||||
Resources: []*tfjson.ConfigResource{declaration(0)},
|
||||
}}
|
||||
checkUsage(t, config, scriptOrderUsage{dataSourceCount: 1})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package provisionerdserver_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
tfjson "github.com/hashicorp/terraform-json"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
"github.com/coder/coder/v2/coderd/provisionerdserver"
|
||||
"github.com/coder/coder/v2/provisionerd/proto"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
|
||||
func TestCompleteJobPersistsScriptOrderCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
srv, db, _, daemon := setup(t, false, nil)
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
|
||||
// GIVEN: A template version with an acquired import job.
|
||||
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
||||
CreatedBy: user.ID,
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
JobID: uuid.New(),
|
||||
})
|
||||
input, err := json.Marshal(provisionerdserver.TemplateVersionImportJob{
|
||||
TemplateVersionID: version.ID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
job, err := db.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
|
||||
ID: version.JobID,
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
InitiatorID: user.ID,
|
||||
Provisioner: database.ProvisionerTypeEcho,
|
||||
StorageMethod: database.ProvisionerStorageMethodFile,
|
||||
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
||||
Input: input,
|
||||
Tags: daemon.Tags,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
tags, err := json.Marshal(job.Tags)
|
||||
require.NoError(t, err)
|
||||
_, err = db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
WorkerID: uuid.NullUUID{UUID: daemon.ID, Valid: true},
|
||||
Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
|
||||
StartedAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
|
||||
ProvisionerTags: tags,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// This test calls CompleteJob directly without running Terraform or the
|
||||
// provisioner. Use a captured START plan and reproduce the provisioner's
|
||||
// decode/re-encode step to verify literal rule blocks survive serialization
|
||||
// and their counts are persisted.
|
||||
planJSON, err := os.ReadFile("../../provisioner/terraform/testdata/script-order-plan.json")
|
||||
require.NoError(t, err)
|
||||
var plan tfjson.Plan
|
||||
require.NoError(t, json.Unmarshal(planJSON, &plan))
|
||||
planJSON, err = json.Marshal(&plan)
|
||||
require.NoError(t, err)
|
||||
|
||||
// WHEN: The import completes with the serialized START plan.
|
||||
_, err = srv.CompleteJob(ctx, &proto.CompletedJob{
|
||||
JobId: job.ID.String(),
|
||||
Type: &proto.CompletedJob_TemplateImport_{
|
||||
TemplateImport: &proto.CompletedJob_TemplateImport{
|
||||
Plan: planJSON,
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// THEN: Counts are persisted but excluded from the telemetry summary.
|
||||
version, err = db.GetTemplateVersionByID(ctx, version.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int32(1), version.ScriptOrderDataSourceCount)
|
||||
require.Equal(t, int32(1), version.ScriptOrderRuleCount)
|
||||
|
||||
usage, err := db.GetScriptOrderTelemetrySummary(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, database.GetScriptOrderTelemetrySummaryRow{}, usage)
|
||||
|
||||
// WHEN: The version becomes a template's active version.
|
||||
template := dbgen.Template(t, db, database.Template{
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
CreatedBy: user.ID,
|
||||
ActiveVersionID: version.ID,
|
||||
})
|
||||
err = db.UpdateTemplateVersionByID(ctx, database.UpdateTemplateVersionByIDParams{
|
||||
ID: version.ID,
|
||||
TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true},
|
||||
UpdatedAt: dbtime.Now(),
|
||||
Name: version.Name,
|
||||
Message: version.Message,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// THEN: Its counts are included in the telemetry summary.
|
||||
usage, err = db.GetScriptOrderTelemetrySummary(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, database.GetScriptOrderTelemetrySummaryRow{
|
||||
TemplatesWithScriptOrder: 1,
|
||||
DataSourceCount: 1,
|
||||
RuleCount: 1,
|
||||
}, usage)
|
||||
}
|
||||
|
||||
// Script-order telemetry is best effort: missing or malformed plan
|
||||
// configuration must not fail an otherwise successful template import.
|
||||
func TestCompleteJobWithoutScriptOrderConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv, db, _, daemon := setup(t, false, nil)
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
plan string
|
||||
}{
|
||||
{name: "MissingPlan"},
|
||||
{name: "MissingConfiguration", plan: "{}"},
|
||||
{name: "InvalidConfiguration", plan: `{"configuration":[]}`},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
// GIVEN: A template version with an acquired import job.
|
||||
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
||||
CreatedBy: user.ID,
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
JobID: uuid.New(),
|
||||
})
|
||||
input, err := json.Marshal(provisionerdserver.TemplateVersionImportJob{
|
||||
TemplateVersionID: version.ID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
|
||||
ID: version.JobID,
|
||||
OrganizationID: daemon.OrganizationID,
|
||||
InitiatorID: user.ID,
|
||||
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
||||
Input: input,
|
||||
Tags: daemon.Tags,
|
||||
StartedAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
|
||||
WorkerID: uuid.NullUUID{UUID: daemon.ID, Valid: true},
|
||||
})
|
||||
|
||||
// WHEN: The import completes without usable plan configuration.
|
||||
_, err = srv.CompleteJob(ctx, &proto.CompletedJob{
|
||||
JobId: job.ID.String(),
|
||||
Type: &proto.CompletedJob_TemplateImport_{
|
||||
TemplateImport: &proto.CompletedJob_TemplateImport{
|
||||
Plan: []byte(tt.plan),
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// THEN: The import succeeds without script-order usage counts.
|
||||
job, err = db.GetProvisionerJobByID(ctx, job.ID)
|
||||
require.NoError(t, err)
|
||||
require.True(t, job.CompletedAt.Valid)
|
||||
require.Empty(t, job.Error.String)
|
||||
|
||||
version, err = db.GetTemplateVersionByID(ctx, version.ID)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, version.ScriptOrderDataSourceCount)
|
||||
require.Zero(t, version.ScriptOrderRuleCount)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -854,6 +854,18 @@ func (r *remoteReporter) createSnapshot() (*Snapshot, error) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
eg.Go(func() error {
|
||||
summary, err := r.collectScriptOrderSummary(ctx)
|
||||
if err != nil {
|
||||
// A script-order collection failure should not prevent
|
||||
// the rest of the telemetry snapshot from being sent,
|
||||
// as some collectors reset counters during collection.
|
||||
r.options.Logger.Warn(ctx, "collect script order summary", slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
snapshot.ScriptOrderSummary = summary
|
||||
return nil
|
||||
})
|
||||
|
||||
err := eg.Wait()
|
||||
if err != nil {
|
||||
@@ -1027,6 +1039,35 @@ func (r *remoteReporter) collectUserSecretsSummary(ctx context.Context) (*UserSe
|
||||
}, nil
|
||||
}
|
||||
|
||||
// collectScriptOrderSummary collects deployment-wide adoption counts from active,
|
||||
// successfully imported versions of non-deleted templates. It elects one replica
|
||||
// per snapshot period to avoid overrepresenting multi-replica deployments and
|
||||
// returns (nil, nil) if the period's telemetry lock is already claimed.
|
||||
func (r *remoteReporter) collectScriptOrderSummary(ctx context.Context) (*ScriptOrderSummary, error) {
|
||||
periodEndingAt := dbtime.Time(r.options.Clock.Now()).UTC().Truncate(r.options.SnapshotFrequency)
|
||||
err := r.options.Database.InsertTelemetryLock(ctx, database.InsertTelemetryLockParams{
|
||||
EventType: "script_order_summary",
|
||||
PeriodEndingAt: periodEndingAt,
|
||||
})
|
||||
if database.IsUniqueViolation(err, database.UniqueTelemetryLocksPkey) {
|
||||
r.options.Logger.Debug(ctx, "script order telemetry lock already claimed by another replica, skipping", slog.F("period_ending_at", periodEndingAt))
|
||||
return nil, nil //nolint:nilnil // Another replica reports this period's summary.
|
||||
}
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("insert script order telemetry lock (period_ending_at=%q): %w", periodEndingAt, err)
|
||||
}
|
||||
|
||||
row, err := r.options.Database.GetScriptOrderTelemetrySummary(ctx)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("get script order telemetry summary: %w", err)
|
||||
}
|
||||
return &ScriptOrderSummary{
|
||||
TemplatesWithScriptOrder: row.TemplatesWithScriptOrder,
|
||||
DataSourceCount: row.DataSourceCount,
|
||||
RuleCount: row.RuleCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ConvertAPIKey anonymizes an API key.
|
||||
func ConvertAPIKey(apiKey database.APIKey) APIKey {
|
||||
a := APIKey{
|
||||
@@ -1478,6 +1519,7 @@ type Snapshot struct {
|
||||
Licenses []License `json:"licenses"`
|
||||
ProvisionerJobs []ProvisionerJob `json:"provisioner_jobs"`
|
||||
TemplateVersions []TemplateVersion `json:"template_versions"`
|
||||
ScriptOrderSummary *ScriptOrderSummary `json:"script_order_summary"`
|
||||
Templates []Template `json:"templates"`
|
||||
Users []User `json:"users"`
|
||||
Groups []Group `json:"groups"`
|
||||
@@ -2483,6 +2525,15 @@ type UserSecretsSummary struct {
|
||||
SecretsPerUserP90 int64 `json:"secrets_per_user_p90"`
|
||||
}
|
||||
|
||||
// ScriptOrderSummary contains deployment-wide counts from active,
|
||||
// successfully imported versions of non-deleted templates, without
|
||||
// template identifiers or rule contents.
|
||||
type ScriptOrderSummary struct {
|
||||
TemplatesWithScriptOrder int64 `json:"templates_with_script_order"`
|
||||
DataSourceCount int64 `json:"data_source_count"`
|
||||
RuleCount int64 `json:"rule_count"`
|
||||
}
|
||||
|
||||
// TemplateBuilderSession tracks a single event in the template builder
|
||||
// wizard: entry, compose completion, and build failure. User-supplied
|
||||
// variable values are never included.
|
||||
|
||||
@@ -1682,6 +1682,230 @@ func TestUserSecretsTelemetry(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestTelemetryScriptOrderSummary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, _ := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
org, err := db.GetDefaultOrganization(ctx)
|
||||
require.NoError(t, err)
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
now := dbtime.Now()
|
||||
|
||||
// Seed all fixtures into one database, then verify the combined
|
||||
// telemetry summary below.
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
jobType database.ProvisionerJobType
|
||||
dataSources int32
|
||||
rules int32
|
||||
completed bool
|
||||
failed bool
|
||||
canceled bool
|
||||
inactive bool
|
||||
deleted bool
|
||||
unattached bool
|
||||
}{
|
||||
// These three templates contribute to the summary totals. A
|
||||
// script-order declaration counts even without literal rules.
|
||||
{name: "FirstTemplate", dataSources: 2, rules: 3, completed: true},
|
||||
{name: "SecondTemplate", dataSources: 1, rules: 2, completed: true},
|
||||
{name: "NoLiteralRules", dataSources: 1, completed: true},
|
||||
// These must not contribute to the summary totals.
|
||||
{name: "NoScriptOrder", completed: true},
|
||||
{name: "InactiveVersion", dataSources: 9, rules: 9, completed: true, inactive: true},
|
||||
{name: "DeletedTemplate", dataSources: 9, rules: 9, completed: true, deleted: true},
|
||||
{name: "UnattachedVersion", dataSources: 9, rules: 9, completed: true, unattached: true},
|
||||
{name: "FailedImport", dataSources: 9, rules: 9, completed: true, failed: true},
|
||||
{name: "CanceledImport", dataSources: 9, rules: 9, completed: true, canceled: true},
|
||||
{name: "IncompleteImport", dataSources: 9, rules: 9},
|
||||
{name: "NonImportJob", jobType: database.ProvisionerJobTypeTemplateVersionDryRun, dataSources: 9, rules: 9, completed: true},
|
||||
} {
|
||||
job := database.ProvisionerJob{
|
||||
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
||||
OrganizationID: org.ID,
|
||||
InitiatorID: user.ID,
|
||||
}
|
||||
if tt.jobType != "" {
|
||||
job.Type = tt.jobType
|
||||
}
|
||||
if tt.completed {
|
||||
job.CompletedAt = sql.NullTime{Time: now, Valid: true}
|
||||
}
|
||||
if tt.failed {
|
||||
job.Error = sql.NullString{String: "import failed", Valid: true}
|
||||
}
|
||||
if tt.canceled {
|
||||
job.CanceledAt = sql.NullTime{Time: now, Valid: true}
|
||||
}
|
||||
job = dbgen.ProvisionerJob(t, db, nil, job)
|
||||
var templateID uuid.NullUUID
|
||||
if !tt.unattached {
|
||||
template := dbgen.Template(t, db, database.Template{
|
||||
OrganizationID: org.ID,
|
||||
CreatedBy: user.ID,
|
||||
})
|
||||
templateID = uuid.NullUUID{UUID: template.ID, Valid: true}
|
||||
if tt.deleted {
|
||||
err := db.UpdateTemplateDeletedByID(ctx, database.UpdateTemplateDeletedByIDParams{
|
||||
ID: template.ID,
|
||||
Deleted: true,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
require.NoError(t, err, tt.name)
|
||||
}
|
||||
}
|
||||
// Active versions count regardless of when they were imported.
|
||||
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
||||
CreatedAt: now.Add(-24 * time.Hour),
|
||||
CreatedBy: user.ID,
|
||||
OrganizationID: org.ID,
|
||||
TemplateID: templateID,
|
||||
JobID: job.ID,
|
||||
ScriptOrderDataSourceCount: tt.dataSources,
|
||||
ScriptOrderRuleCount: tt.rules,
|
||||
})
|
||||
if tt.inactive {
|
||||
// A newer version without script ordering replaces one that uses it.
|
||||
activeJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
|
||||
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
||||
OrganizationID: org.ID,
|
||||
InitiatorID: user.ID,
|
||||
CompletedAt: sql.NullTime{Time: now, Valid: true},
|
||||
})
|
||||
version = dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
||||
CreatedBy: user.ID,
|
||||
OrganizationID: org.ID,
|
||||
TemplateID: templateID,
|
||||
JobID: activeJob.ID,
|
||||
// No script order counts.
|
||||
})
|
||||
}
|
||||
if templateID.Valid {
|
||||
err := db.UpdateTemplateActiveVersionByID(ctx, database.UpdateTemplateActiveVersionByIDParams{
|
||||
ID: templateID.UUID,
|
||||
ActiveVersionID: version.ID,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
require.NoError(t, err, tt.name)
|
||||
}
|
||||
}
|
||||
|
||||
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
||||
|
||||
// Keep totals distinct so swapped telemetry fields fail this test.
|
||||
require.Equal(t, &telemetry.ScriptOrderSummary{
|
||||
TemplatesWithScriptOrder: 3,
|
||||
DataSourceCount: 4,
|
||||
RuleCount: 5,
|
||||
}, snapshot.ScriptOrderSummary)
|
||||
data, err := json.Marshal(snapshot.ScriptOrderSummary)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the serialized telemetry contains only aggregate counts,
|
||||
// with no identifiers or rule contents.
|
||||
require.JSONEq(t, `{
|
||||
"templates_with_script_order":3,"data_source_count":4,"rule_count":5
|
||||
}`, string(data))
|
||||
|
||||
// Verify script-order counts are excluded from individual
|
||||
// template-version telemetry records and reported only in the
|
||||
// deployment-wide summary.
|
||||
versionData, err := json.Marshal(telemetry.ConvertTemplateVersion(database.TemplateVersion{
|
||||
ScriptOrderDataSourceCount: 1,
|
||||
ScriptOrderRuleCount: 2,
|
||||
}))
|
||||
require.NoError(t, err)
|
||||
var fields map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(versionData, &fields))
|
||||
require.NotContains(t, fields, "script_order_data_source_count")
|
||||
require.NotContains(t, fields, "script_order_rule_count")
|
||||
}
|
||||
|
||||
func TestTelemetryScriptOrderSummaryError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, _ := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
|
||||
// Boundary counters are reset during collection, so their snapshot
|
||||
// must still be delivered when script-order collection fails.
|
||||
tracker := boundaryusage.NewTracker()
|
||||
tracker.Track(uuid.New(), uuid.New(), 10, 2)
|
||||
require.NoError(t, tracker.FlushToDB(ctx, db, uuid.New()))
|
||||
|
||||
_, snapshot := collectSnapshot(ctx, t, &scriptOrderSummaryErrorStore{Store: db}, nil)
|
||||
require.Nil(t, snapshot.ScriptOrderSummary)
|
||||
require.NotNil(t, snapshot.BoundaryUsageSummary)
|
||||
require.Equal(t, int64(1), snapshot.BoundaryUsageSummary.UniqueWorkspaces)
|
||||
require.Equal(t, int64(1), snapshot.BoundaryUsageSummary.UniqueUsers)
|
||||
require.Equal(t, int64(10), snapshot.BoundaryUsageSummary.AllowedRequests)
|
||||
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.DeniedRequests)
|
||||
}
|
||||
|
||||
type scriptOrderSummaryErrorStore struct {
|
||||
database.Store
|
||||
}
|
||||
|
||||
func (*scriptOrderSummaryErrorStore) GetScriptOrderTelemetrySummary(context.Context) (database.GetScriptOrderTelemetrySummaryRow, error) {
|
||||
return database.GetScriptOrderTelemetrySummaryRow{}, sql.ErrConnDone
|
||||
}
|
||||
|
||||
func TestTelemetryScriptOrderSummaryEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, _ := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
||||
|
||||
// Verify an empty deployment reports an explicit all-zero
|
||||
// summary, rather than a missing summary.
|
||||
require.Equal(t, &telemetry.ScriptOrderSummary{}, snapshot.ScriptOrderSummary)
|
||||
data, err := json.Marshal(snapshot)
|
||||
require.NoError(t, err)
|
||||
var fields map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(data, &fields))
|
||||
require.JSONEq(t, `{
|
||||
"templates_with_script_order":0,"data_source_count":0,"rule_count":0
|
||||
}`, string(fields["script_order_summary"]))
|
||||
}
|
||||
|
||||
func TestTelemetryScriptOrderSummaryOneReplicaPerPeriod(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, _ := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitMedium)
|
||||
const snapshotFrequency = 5 * time.Minute
|
||||
clock := quartz.NewMock(t)
|
||||
clock.Set(dbtime.Now().Truncate(snapshotFrequency))
|
||||
deploymentID := uuid.NewString()
|
||||
options := func(opts telemetry.Options) telemetry.Options {
|
||||
opts.Clock = clock
|
||||
opts.SnapshotFrequency = snapshotFrequency
|
||||
opts.DeploymentID = deploymentID
|
||||
return opts
|
||||
}
|
||||
|
||||
// The elected replica reports zero counts too, so adoption can be cleared.
|
||||
_, first := collectSnapshot(ctx, t, db, options)
|
||||
require.Equal(t, &telemetry.ScriptOrderSummary{}, first.ScriptOrderSummary)
|
||||
|
||||
// Another replica later in the same period must not collect a summary.
|
||||
clock.Advance(time.Second)
|
||||
_, second := collectSnapshot(ctx, t, db, options)
|
||||
require.Nil(t, second.ScriptOrderSummary)
|
||||
data, err := json.Marshal(second)
|
||||
require.NoError(t, err)
|
||||
var fields map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(data, &fields))
|
||||
require.Equal(t, "null", string(fields["script_order_summary"]))
|
||||
|
||||
// The previous period's lock must not prevent the next collection.
|
||||
clock.Advance(snapshotFrequency - time.Second)
|
||||
_, next := collectSnapshot(ctx, t, db, options)
|
||||
require.Equal(t, &telemetry.ScriptOrderSummary{}, next.ScriptOrderSummary)
|
||||
}
|
||||
|
||||
func TestCollectAgentsVirtualDesktop(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -767,6 +767,8 @@ Actions: `create`, `write`
|
||||
<tr><td><code>name</code></td><td>Yes</td></tr>
|
||||
<tr><td><code>organization_id</code></td><td>No</td></tr>
|
||||
<tr><td><code>readme</code></td><td>Yes</td></tr>
|
||||
<tr><td><code>script_order_data_source_count</code></td><td>No</td></tr>
|
||||
<tr><td><code>script_order_rule_count</code></td><td>No</td></tr>
|
||||
<tr><td><code>source_example_id</code></td><td>No</td></tr>
|
||||
<tr><td><code>template_id</code></td><td>Yes</td></tr>
|
||||
<tr><td><code>updated_at</code></td><td>No</td></tr>
|
||||
|
||||
+19
-17
@@ -144,23 +144,25 @@ var auditableResourcesTypes = map[any]map[string]Action{
|
||||
"allow_workspace_renames": ActionTrack,
|
||||
},
|
||||
&database.TemplateVersion{}: {
|
||||
"id": ActionTrack,
|
||||
"template_id": ActionTrack,
|
||||
"organization_id": ActionIgnore, // Never changes.
|
||||
"created_at": ActionIgnore, // Never changes, but is implicit and not helpful in a diff.
|
||||
"updated_at": ActionIgnore, // Changes, but is implicit and not helpful in a diff.
|
||||
"name": ActionTrack,
|
||||
"message": ActionIgnore, // Never changes after creation.
|
||||
"readme": ActionTrack,
|
||||
"job_id": ActionIgnore, // Not helpful in a diff because jobs aren't tracked in audit logs.
|
||||
"created_by": ActionTrack,
|
||||
"external_auth_providers": ActionIgnore, // Not helpful because this can only change when new versions are added.
|
||||
"created_by_avatar_url": ActionIgnore,
|
||||
"created_by_username": ActionIgnore,
|
||||
"created_by_name": ActionIgnore,
|
||||
"archived": ActionTrack,
|
||||
"source_example_id": ActionIgnore, // Never changes.
|
||||
"has_external_agent": ActionIgnore, // Never changes.
|
||||
"id": ActionTrack,
|
||||
"template_id": ActionTrack,
|
||||
"organization_id": ActionIgnore, // Never changes.
|
||||
"created_at": ActionIgnore, // Never changes, but is implicit and not helpful in a diff.
|
||||
"updated_at": ActionIgnore, // Changes, but is implicit and not helpful in a diff.
|
||||
"name": ActionTrack,
|
||||
"message": ActionIgnore, // Never changes after creation.
|
||||
"readme": ActionTrack,
|
||||
"job_id": ActionIgnore, // Not helpful in a diff because jobs aren't tracked in audit logs.
|
||||
"created_by": ActionTrack,
|
||||
"external_auth_providers": ActionIgnore, // Not helpful because this can only change when new versions are added.
|
||||
"created_by_avatar_url": ActionIgnore,
|
||||
"created_by_username": ActionIgnore,
|
||||
"created_by_name": ActionIgnore,
|
||||
"archived": ActionTrack,
|
||||
"source_example_id": ActionIgnore, // Never changes.
|
||||
"has_external_agent": ActionIgnore, // Never changes.
|
||||
"script_order_data_source_count": ActionIgnore, // Telemetry-only metadata.
|
||||
"script_order_rule_count": ActionIgnore, // Telemetry-only metadata.
|
||||
},
|
||||
&database.User{}: {
|
||||
"id": ActionTrack,
|
||||
|
||||
@@ -152,8 +152,13 @@ func sendGraph(sess proto.DRPCProvisioner_SessionClient, source proto.GraphSourc
|
||||
}}})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // Avoid "text file busy"; see explanation below.
|
||||
func TestGraphProcessesScriptOrdersOnlyForStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
// This test and its subtests must run serially. Each subtest writes and
|
||||
// immediately executes a temporary shell script that mocks Terraform.
|
||||
// Another test forking while the script is open for writing can leave a
|
||||
// writable file descriptor in its child even after os.WriteFile returns,
|
||||
// causing execution to fail with "text file busy".
|
||||
|
||||
const selfDependencyPlanJSON = `{
|
||||
"format_version":"1.2",
|
||||
@@ -277,9 +282,8 @@ func TestGraphProcessesScriptOrdersOnlyForStart(t *testing.T) {
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
//nolint:paralleltest // Avoid the script descriptor race above.
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
planPath := filepath.Join(dir, "plan.json")
|
||||
graphPath := filepath.Join(dir, "graph.dot")
|
||||
@@ -343,8 +347,13 @@ esac
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:paralleltest // Avoid "text file busy"; see explanation below.
|
||||
func TestGraphResolvesAppliedScriptOrders(t *testing.T) {
|
||||
t.Parallel()
|
||||
// This test and its subtests must run serially. Each subtest writes and
|
||||
// immediately executes a temporary shell script that mocks Terraform.
|
||||
// Another test forking while the script is open for writing can leave a
|
||||
// writable file descriptor in its child even after os.WriteFile returns,
|
||||
// causing execution to fail with "text file busy".
|
||||
|
||||
// Terraform records the script order data source in prior state when it is
|
||||
// read during planning. The plan fixture reproduces that shape so this test
|
||||
@@ -416,9 +425,8 @@ func TestGraphResolvesAppliedScriptOrders(t *testing.T) {
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
//nolint:paralleltest // Avoid the script descriptor race above.
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
planPath := filepath.Join(dir, "plan.json")
|
||||
statePath := filepath.Join(dir, "state.json")
|
||||
|
||||
Reference in New Issue
Block a user