mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix: reduce cost of prebuild failure (#17697)
Relates to https://github.com/coder/coder/issues/17432 ### Part 1: Notes: - `GetPresetsAtFailureLimit` SQL query is added, which is similar to `GetPresetsBackoff`, they use same CTEs: `filtered_builds`, `time_sorted_builds`, but they are still different. - Query is executed on every loop iteration. We can consider marking specific preset as permanently failed as an optimization to avoid executing query on every loop iteration. But I decided don't do it for now. - By default `FailureHardLimit` is set to 3. - `FailureHardLimit` is configurable. Setting it to zero - means that hard limit is disabled. ### Part 2 Notes: - `PrebuildFailureLimitReached` notification is added. - Notification is sent to template admins. - Notification is sent only the first time, when hard limit is reached. But it will `log.Warn` on every loop iteration. - I introduced this enum: ```sql CREATE TYPE prebuild_status AS ENUM ( 'normal', -- Prebuilds are working as expected; this is the default, healthy state. 'hard_limited', -- Prebuilds have failed repeatedly and hit the configured hard failure limit; won't be retried anymore. 'validation_failed' -- Prebuilds failed due to a non-retryable validation error (e.g. template misconfiguration); won't be retried. ); ``` `validation_failed` not used in this PR, but I think it will be used in next one, so I wanted to save us an extra migration. - Notification looks like this: <img width="472" alt="image" src="https://github.com/user-attachments/assets/e10efea0-1790-4e7f-a65c-f94c40fced27" /> ### Latest notification views: <img width="463" alt="image" src="https://github.com/user-attachments/assets/11310c58-68d1-4075-a497-f76d854633fe" /> <img width="725" alt="image" src="https://github.com/user-attachments/assets/6bbfe21a-91ac-47c3-a9d1-21807bb0c53a" />
This commit is contained in:
Generated
+8
-1
@@ -153,6 +153,12 @@ CREATE TYPE port_share_protocol AS ENUM (
|
||||
'https'
|
||||
);
|
||||
|
||||
CREATE TYPE prebuild_status AS ENUM (
|
||||
'healthy',
|
||||
'hard_limited',
|
||||
'validation_failed'
|
||||
);
|
||||
|
||||
CREATE TYPE provisioner_daemon_status AS ENUM (
|
||||
'offline',
|
||||
'idle',
|
||||
@@ -1439,7 +1445,8 @@ CREATE TABLE template_version_presets (
|
||||
name text NOT NULL,
|
||||
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
desired_instances integer,
|
||||
invalidate_after_secs integer DEFAULT 0
|
||||
invalidate_after_secs integer DEFAULT 0,
|
||||
prebuild_status prebuild_status DEFAULT 'healthy'::prebuild_status NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE template_version_terraform_values (
|
||||
|
||||
Reference in New Issue
Block a user