opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(coderd/notifications): group workspace build failure report (#17306)

Browse Source 
Closes https://github.com/coder/coder/issues/15745

Instead of sending X many reports to a single template admin, we instead
send only 1.
This commit is contained in:
Danielle Maywood
2025-04-10 13:32:19 +01:00
committed by GitHub
parent 33b9487899
commit 6dd1056025
10 changed files with 552 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dbmem/dbmem.go
+1
View File
@@ -3291,6 +3291,7 @@ func (q *FakeQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context,
}
workspaceBuildStats = append(workspaceBuildStats, database.GetFailedWorkspaceBuildsByTemplateIDRow{
WorkspaceID: w.ID,
WorkspaceName: w.Name,
WorkspaceOwnerUsername: workspaceOwner.Username,
TemplateVersionName: templateVersion.Name,
coderd/database/migrations/000316_group_build_failure_notifications.down.sql
+21
View File
@@ -0,0 +1,21 @@
UPDATE notification_templates
SET
name = 'Report: Workspace Builds Failed For Template',
title_template = E'Workspace builds failed for template "{{.Labels.template_display_name}}"',
body_template = E'Template **{{.Labels.template_display_name}}** has failed to build {{.Data.failed_builds}}/{{.Data.total_builds}} times over the last {{.Data.report_frequency}}.
**Report:**
{{range $version := .Data.template_versions}}
**{{$version.template_version_name}}** failed {{$version.failed_count}} time{{if gt $version.failed_count 1.0}}s{{end}}:
{{range $build := $version.failed_builds}}
* [{{$build.workspace_owner_username}} / {{$build.workspace_name}} / #{{$build.build_number}}]({{base_url}}/@{{$build.workspace_owner_username}}/{{$build.workspace_name}}/builds/{{$build.build_number}})
{{- end}}
{{end}}
We recommend reviewing these issues to ensure future builds are successful.',
actions = '[
{
"label": "View workspaces",
"url": "{{ base_url }}/workspaces?filter=template%3A{{.Labels.template_name}}"
}
]'::jsonb
WHERE id = '34a20db2-e9cc-4a93-b0e4-8569699d7a00';
coderd/database/migrations/000316_group_build_failure_notifications.up.sql
+29
View File
@@ -0,0 +1,29 @@
UPDATE notification_templates
SET
name = 'Report: Workspace Builds Failed',
title_template = 'Failed workspace builds report',
body_template =
E'The following templates have had build failures over the last {{.Data.report_frequency}}:
{{range $template := .Data.templates}}
- **{{$template.display_name}}** failed to build {{$template.failed_builds}}/{{$template.total_builds}} times
{{end}}
**Report:**
{{range $template := .Data.templates}}
**{{$template.display_name}}**
{{range $version := $template.versions}}
- **{{$version.template_version_name}}** failed {{$version.failed_count}} time{{if gt $version.failed_count 1.0}}s{{end}}:
{{range $build := $version.failed_builds}}
- [{{$build.workspace_owner_username}} / {{$build.workspace_name}} / #{{$build.build_number}}]({{base_url}}/@{{$build.workspace_owner_username}}/{{$build.workspace_name}}/builds/{{$build.build_number}})
{{end}}
{{end}}
{{end}}
We recommend reviewing these issues to ensure future builds are successful.',
actions = '[
{
"label": "View workspaces",
"url": "{{ base_url }}/workspaces?filter={{$first := true}}{{range $template := .Data.templates}}{{range $version := $template.versions}}{{range $build := $version.failed_builds}}{{if not $first}}+{{else}}{{$first = false}}{{end}}id%3A{{$build.workspace_id}}{{end}}{{end}}{{end}}"
}
]'::jsonb
WHERE id = '34a20db2-e9cc-4a93-b0e4-8569699d7a00';
coderd/database/queries.sql.go
+7 -4
View File
@@ -16334,6 +16334,7 @@ SELECT
tv.name AS template_version_name,
u.username AS workspace_owner_username,
w.name AS workspace_name,
w.id AS workspace_id,
wb.build_number AS workspace_build_number
FROM
workspace_build_with_user AS wb
@@ -16372,10 +16373,11 @@ type GetFailedWorkspaceBuildsByTemplateIDParams struct {
}
type GetFailedWorkspaceBuildsByTemplateIDRow struct {
TemplateVersionName string `db:"template_version_name" json:"template_version_name"`
WorkspaceOwnerUsername string `db:"workspace_owner_username" json:"workspace_owner_username"`
WorkspaceName string `db:"workspace_name" json:"workspace_name"`
WorkspaceBuildNumber int32 `db:"workspace_build_number" json:"workspace_build_number"`
TemplateVersionName string `db:"template_version_name" json:"template_version_name"`
WorkspaceOwnerUsername string `db:"workspace_owner_username" json:"workspace_owner_username"`
WorkspaceName string `db:"workspace_name" json:"workspace_name"`
WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"`
WorkspaceBuildNumber int32 `db:"workspace_build_number" json:"workspace_build_number"`
}
func (q *sqlQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg GetFailedWorkspaceBuildsByTemplateIDParams) ([]GetFailedWorkspaceBuildsByTemplateIDRow, error) {
@@ -16391,6 +16393,7 @@ func (q *sqlQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, a
&i.TemplateVersionName,
&i.WorkspaceOwnerUsername,
&i.WorkspaceName,
&i.WorkspaceID,
&i.WorkspaceBuildNumber,
); err != nil {
return nil, err
coderd/database/queries/workspacebuilds.sql
+1
View File
@@ -213,6 +213,7 @@ SELECT
tv.name AS template_version_name,
u.username AS workspace_owner_username,
w.name AS workspace_name,
w.id AS workspace_id,
wb.build_number AS workspace_build_number
FROM
workspace_build_with_user AS wb
coderd/notifications/notifications_test.go
+84 -27
View File
@@ -978,45 +978,102 @@ func TestNotificationTemplates_Golden(t *testing.T) {
UserName: "Bobby",
UserEmail: "bobby@coder.com",
UserUsername: "bobby",
Labels: map[string]string{
"template_name": "bobby-first-template",
"template_display_name": "Bobby First Template",
},
Labels: map[string]string{},
// We need to use floats as `json.Unmarshal` unmarshal numbers in `map[string]any` to floats.
Data: map[string]any{
"failed_builds": 4.0,
"total_builds": 55.0,
"report_frequency": "week",
"template_versions": []map[string]any{
"templates": []map[string]any{
{
"template_version_name": "bobby-template-version-1",
"failed_count": 3.0,
"failed_builds": []map[string]any{
"name": "bobby-first-template",
"display_name": "Bobby First Template",
"failed_builds": 4.0,
"total_builds": 55.0,
"versions": []map[string]any{
{
"workspace_owner_username": "mtojek",
"workspace_name": "workspace-1",
"build_number": 1234.0,
"template_version_name": "bobby-template-version-1",
"failed_count": 3.0,
"failed_builds": []map[string]any{
{
"workspace_owner_username": "mtojek",
"workspace_name": "workspace-1",
"workspace_id": "24f5bd8f-1566-4374-9734-c3efa0454dc7",
"build_number": 1234.0,
},
{
"workspace_owner_username": "johndoe",
"workspace_name": "my-workspace-3",
"workspace_id": "372a194b-dcde-43f1-b7cf-8a2f3d3114a0",
"build_number": 5678.0,
},
{
"workspace_owner_username": "jack",
"workspace_name": "workwork",
"workspace_id": "1386d294-19c1-4351-89e2-6cae1afb9bfe",
"build_number": 774.0,
},
},
},
{
"workspace_owner_username": "johndoe",
"workspace_name": "my-workspace-3",
"build_number": 5678.0,
},
{
"workspace_owner_username": "jack",
"workspace_name": "workwork",
"build_number": 774.0,
"template_version_name": "bobby-template-version-2",
"failed_count": 1.0,
"failed_builds": []map[string]any{
{
"workspace_owner_username": "ben",
"workspace_name": "cool-workspace",
"workspace_id": "86fd99b1-1b6e-4b7e-b58e-0aee6e35c159",
"build_number": 8888.0,
},
},
},
},
},
{
"template_version_name": "bobby-template-version-2",
"failed_count": 1.0,
"failed_builds": []map[string]any{
"name": "bobby-second-template",
"display_name": "Bobby Second Template",
"failed_builds": 5.0,
"total_builds": 50.0,
"versions": []map[string]any{
{
"workspace_owner_username": "ben",
"workspace_name": "cool-workspace",
"build_number": 8888.0,
"template_version_name": "bobby-template-version-1",
"failed_count": 3.0,
"failed_builds": []map[string]any{
{
"workspace_owner_username": "daniellemaywood",
"workspace_name": "workspace-9",
"workspace_id": "cd469690-b6eb-4123-b759-980be7a7b278",
"build_number": 9234.0,
},
{
"workspace_owner_username": "johndoe",
"workspace_name": "my-workspace-7",
"workspace_id": "c447d472-0800-4529-a836-788754d5e27d",
"build_number": 8678.0,
},
{
"workspace_owner_username": "jack",
"workspace_name": "workworkwork",
"workspace_id": "919db6df-48f0-4dc1-b357-9036a2c40f86",
"build_number": 374.0,
},
},
},
{
"template_version_name": "bobby-template-version-2",
"failed_count": 2.0,
"failed_builds": []map[string]any{
{
"workspace_owner_username": "ben",
"workspace_name": "more-cool-workspace",
"workspace_id": "c8fb0652-9290-4bf2-a711-71b910243ac2",
"build_number": 8878.0,
},
{
"workspace_owner_username": "ben",
"workspace_name": "less-cool-workspace",
"workspace_id": "703d718d-2234-4990-9a02-5b1df6cf462a",
"build_number": 8848.0,
},
},
},
},
},
coderd/notifications/reports/generator.go
+96 -64
View File
@@ -18,6 +18,7 @@ import (
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/notifications"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/coder/v2/codersdk"
)
@@ -102,6 +103,11 @@ const (
failedWorkspaceBuildsReportFrequencyLabel = "week"
)
type adminReport struct {
stats database.GetWorkspaceBuildStatsByTemplatesRow
failedBuilds []database.GetFailedWorkspaceBuildsByTemplateIDRow
}
func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) error {
now := clk.Now()
since := now.Add(-failedWorkspaceBuildsReportFrequency)
@@ -136,6 +142,8 @@ func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db dat
return xerrors.Errorf("unable to fetch failed workspace builds: %w", err)
}
reports := make(map[uuid.UUID][]adminReport)
for _, stats := range templateStatsRows {
select {
case <-ctx.Done():
@@ -165,33 +173,40 @@ func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db dat
logger.Error(ctx, "unable to fetch failed workspace builds", slog.F("template_id", stats.TemplateID), slog.Error(err))
continue
}
reportData := buildDataForReportFailedWorkspaceBuilds(stats, failedBuilds)
// Send reports to template admins
templateDisplayName := stats.TemplateDisplayName
if templateDisplayName == "" {
templateDisplayName = stats.TemplateName
}
for _, templateAdmin := range templateAdmins {
select {
case <-ctx.Done():
logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
break
default:
}
adminReports := reports[templateAdmin.ID]
adminReports = append(adminReports, adminReport{
failedBuilds: failedBuilds,
stats: stats,
})
if _, err := enqueuer.EnqueueWithData(ctx, templateAdmin.ID, notifications.TemplateWorkspaceBuildsFailedReport,
map[string]string{
"template_name": stats.TemplateName,
"template_display_name": templateDisplayName,
},
reportData,
"report_generator",
stats.TemplateID, stats.TemplateOrganizationID,
); err != nil {
logger.Warn(ctx, "failed to send a report with failed workspace builds", slog.Error(err))
}
reports[templateAdmin.ID] = adminReports
}
}
for templateAdmin, reports := range reports {
select {
case <-ctx.Done():
logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
break
default:
}
reportData := buildDataForReportFailedWorkspaceBuilds(reports)
targets := []uuid.UUID{}
for _, report := range reports {
targets = append(targets, report.stats.TemplateID, report.stats.TemplateOrganizationID)
}
if _, err := enqueuer.EnqueueWithData(ctx, templateAdmin, notifications.TemplateWorkspaceBuildsFailedReport,
map[string]string{},
reportData,
"report_generator",
slice.Unique(targets)...,
); err != nil {
logger.Warn(ctx, "failed to send a report with failed workspace builds", slog.Error(err))
}
}
@@ -213,54 +228,71 @@ func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db dat
const workspaceBuildsLimitPerTemplateVersion = 10
func buildDataForReportFailedWorkspaceBuilds(stats database.GetWorkspaceBuildStatsByTemplatesRow, failedBuilds []database.GetFailedWorkspaceBuildsByTemplateIDRow) map[string]any {
// Build notification model for template versions and failed workspace builds.
//
// Failed builds are sorted by template version ascending, workspace build number descending.
// Review builds, group them by template versions, and assign to builds to template versions.
// The map requires `[]map[string]any{}` to be compatible with data passed to `NotificationEnqueuer`.
templateVersions := []map[string]any{}
for _, failedBuild := range failedBuilds {
c := len(templateVersions)
func buildDataForReportFailedWorkspaceBuilds(reports []adminReport) map[string]any {
templates := []map[string]any{}
if c == 0 || templateVersions[c-1]["template_version_name"] != failedBuild.TemplateVersionName {
templateVersions = append(templateVersions, map[string]any{
"template_version_name": failedBuild.TemplateVersionName,
"failed_count": 1,
"failed_builds": []map[string]any{
{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"build_number": failedBuild.WorkspaceBuildNumber,
for _, report := range reports {
// Build notification model for template versions and failed workspace builds.
//
// Failed builds are sorted by template version ascending, workspace build number descending.
// Review builds, group them by template versions, and assign to builds to template versions.
// The map requires `[]map[string]any{}` to be compatible with data passed to `NotificationEnqueuer`.
templateVersions := []map[string]any{}
for _, failedBuild := range report.failedBuilds {
c := len(templateVersions)
if c == 0 || templateVersions[c-1]["template_version_name"] != failedBuild.TemplateVersionName {
templateVersions = append(templateVersions, map[string]any{
"template_version_name": failedBuild.TemplateVersionName,
"failed_count": 1,
"failed_builds": []map[string]any{
{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"workspace_id": failedBuild.WorkspaceID,
"build_number": failedBuild.WorkspaceBuildNumber,
},
},
},
})
continue
})
continue
}
tv := templateVersions[c-1]
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
tv["failed_count"] = tv["failed_count"].(int) + 1
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
builds := tv["failed_builds"].([]map[string]any)
if len(builds) < workspaceBuildsLimitPerTemplateVersion {
// return N last builds to prevent long email reports
builds = append(builds, map[string]any{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"workspace_id": failedBuild.WorkspaceID,
"build_number": failedBuild.WorkspaceBuildNumber,
})
tv["failed_builds"] = builds
}
templateVersions[c-1] = tv
}
tv := templateVersions[c-1]
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
tv["failed_count"] = tv["failed_count"].(int) + 1
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
builds := tv["failed_builds"].([]map[string]any)
if len(builds) < workspaceBuildsLimitPerTemplateVersion {
// return N last builds to prevent long email reports
builds = append(builds, map[string]any{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"build_number": failedBuild.WorkspaceBuildNumber,
})
tv["failed_builds"] = builds
templateDisplayName := report.stats.TemplateDisplayName
if templateDisplayName == "" {
templateDisplayName = report.stats.TemplateName
}
templateVersions[c-1] = tv
templates = append(templates, map[string]any{
"failed_builds": report.stats.FailedBuilds,
"total_builds": report.stats.TotalBuilds,
"versions": templateVersions,
"name": report.stats.TemplateName,
"display_name": templateDisplayName,
})
}
return map[string]any{
"failed_builds": stats.FailedBuilds,
"total_builds": stats.TotalBuilds,
"report_frequency": failedWorkspaceBuildsReportFrequencyLabel,
"template_versions": templateVersions,
"report_frequency": failedWorkspaceBuildsReportFrequencyLabel,
"templates": templates,
}
}
coderd/notifications/reports/generator_internal_test.go
+122 -80
View File
@@ -3,6 +3,7 @@ package reports
import (
"context"
"database/sql"
"sort"
"testing"
"time"
@@ -118,17 +119,13 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
t.Run("FailedBuilds_SecondRun_Report_ThirdRunTooEarly_NoReport_FourthRun_Report", func(t *testing.T) {
t.Parallel()
verifyNotification := func(t *testing.T, recipient database.User, notif *notificationstest.FakeNotification, tmpl database.Template, failedBuilds, totalBuilds int64, templateVersions []map[string]interface{}) {
verifyNotification := func(t *testing.T, recipientID uuid.UUID, notif *notificationstest.FakeNotification, templates []map[string]any) {
t.Helper()
require.Equal(t, recipient.ID, notif.UserID)
require.Equal(t, recipientID, notif.UserID)
require.Equal(t, notifications.TemplateWorkspaceBuildsFailedReport, notif.TemplateID)
require.Equal(t, tmpl.Name, notif.Labels["template_name"])
require.Equal(t, tmpl.DisplayName, notif.Labels["template_display_name"])
require.Equal(t, failedBuilds, notif.Data["failed_builds"])
require.Equal(t, totalBuilds, notif.Data["total_builds"])
require.Equal(t, "week", notif.Data["report_frequency"])
require.Equal(t, templateVersions, notif.Data["template_versions"])
require.Equal(t, templates, notif.Data["templates"])
}
// Setup
@@ -212,43 +209,65 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
require.NoError(t, err)
sent := notifEnq.Sent()
require.Len(t, sent, 4) // 2 templates, 2 template admins
for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
verifyNotification(t, templateAdmin, sent[i], t1, 3, 4, []map[string]interface{}{
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(7), "workspace_name": w3.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(1), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
},
"failed_count": 2,
"template_version_name": t1v1.Name,
},
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(3), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
},
"failed_count": 1,
"template_version_name": t1v2.Name,
},
})
}
require.Len(t, sent, 2) // 2 templates, 2 template admins
for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
verifyNotification(t, templateAdmin, sent[i+2], t2, 3, 5, []map[string]interface{}{
templateAdmins := []uuid.UUID{templateAdmin1.ID, templateAdmin2.ID}
// Ensure consistent order for tests
sort.Slice(templateAdmins, func(i, j int) bool {
return templateAdmins[i].String() < templateAdmins[j].String()
})
sort.Slice(sent, func(i, j int) bool {
return sent[i].UserID.String() < sent[j].UserID.String()
})
for i, templateAdmin := range templateAdmins {
verifyNotification(t, templateAdmin, sent[i], []map[string]any{
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(8), "workspace_name": w4.Name, "workspace_owner_username": user2.Username},
"name": t1.Name,
"display_name": t1.DisplayName,
"failed_builds": int64(3),
"total_builds": int64(4),
"versions": []map[string]any{
{
"failed_builds": []map[string]any{
{"build_number": int32(7), "workspace_name": w3.Name, "workspace_id": w3.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(1), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
},
"failed_count": 2,
"template_version_name": t1v1.Name,
},
{
"failed_builds": []map[string]any{
{"build_number": int32(3), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
},
"failed_count": 1,
"template_version_name": t1v2.Name,
},
},
"failed_count": 1,
"template_version_name": t2v1.Name,
},
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(6), "workspace_name": w2.Name, "workspace_owner_username": user2.Username},
{"build_number": int32(5), "workspace_name": w2.Name, "workspace_owner_username": user2.Username},
"name": t2.Name,
"display_name": t2.DisplayName,
"failed_builds": int64(3),
"total_builds": int64(5),
"versions": []map[string]any{
{
"failed_builds": []map[string]any{
{"build_number": int32(8), "workspace_name": w4.Name, "workspace_id": w4.ID, "workspace_owner_username": user2.Username},
},
"failed_count": 1,
"template_version_name": t2v1.Name,
},
{
"failed_builds": []map[string]any{
{"build_number": int32(6), "workspace_name": w2.Name, "workspace_id": w2.ID, "workspace_owner_username": user2.Username},
{"build_number": int32(5), "workspace_name": w2.Name, "workspace_id": w2.ID, "workspace_owner_username": user2.Username},
},
"failed_count": 2,
"template_version_name": t2v2.Name,
},
},
"failed_count": 2,
"template_version_name": t2v2.Name,
},
})
}
@@ -279,14 +298,33 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
// Then: we should see the failed job in the report
sent = notifEnq.Sent()
require.Len(t, sent, 2) // a new failed job should be reported
for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
verifyNotification(t, templateAdmin, sent[i], t1, 1, 1, []map[string]interface{}{
templateAdmins = []uuid.UUID{templateAdmin1.ID, templateAdmin2.ID}
// Ensure consistent order for tests
sort.Slice(templateAdmins, func(i, j int) bool {
return templateAdmins[i].String() < templateAdmins[j].String()
})
sort.Slice(sent, func(i, j int) bool {
return sent[i].UserID.String() < sent[j].UserID.String()
})
for i, templateAdmin := range templateAdmins {
verifyNotification(t, templateAdmin, sent[i], []map[string]any{
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(77), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
"name": t1.Name,
"display_name": t1.DisplayName,
"failed_builds": int64(1),
"total_builds": int64(1),
"versions": []map[string]any{
{
"failed_builds": []map[string]any{
{"build_number": int32(77), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
},
"failed_count": 1,
"template_version_name": t1v2.Name,
},
},
"failed_count": 1,
"template_version_name": t1v2.Name,
},
})
}
@@ -295,17 +333,13 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
t.Run("TooManyFailedBuilds_SecondRun_Report", func(t *testing.T) {
t.Parallel()
verifyNotification := func(t *testing.T, recipient database.User, notif *notificationstest.FakeNotification, tmpl database.Template, failedBuilds, totalBuilds int64, templateVersions []map[string]interface{}) {
verifyNotification := func(t *testing.T, recipient database.User, notif *notificationstest.FakeNotification, templates []map[string]any) {
t.Helper()
require.Equal(t, recipient.ID, notif.UserID)
require.Equal(t, notifications.TemplateWorkspaceBuildsFailedReport, notif.TemplateID)
require.Equal(t, tmpl.Name, notif.Labels["template_name"])
require.Equal(t, tmpl.DisplayName, notif.Labels["template_display_name"])
require.Equal(t, failedBuilds, notif.Data["failed_builds"])
require.Equal(t, totalBuilds, notif.Data["total_builds"])
require.Equal(t, "week", notif.Data["report_frequency"])
require.Equal(t, templateVersions, notif.Data["template_versions"])
require.Equal(t, templates, notif.Data["templates"])
}
// Setup
@@ -369,38 +403,46 @@ func TestReportFailedWorkspaceBuilds(t *testing.T) {
sent := notifEnq.Sent()
require.Len(t, sent, 1) // 1 template, 1 template admin
verifyNotification(t, templateAdmin1, sent[0], t1, 46, 47, []map[string]interface{}{
verifyNotification(t, templateAdmin1, sent[0], []map[string]any{
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(23), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(22), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(21), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(20), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(19), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(18), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(17), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(16), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(15), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(14), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
"name": t1.Name,
"display_name": t1.DisplayName,
"failed_builds": int64(46),
"total_builds": int64(47),
"versions": []map[string]any{
{
"failed_builds": []map[string]any{
{"build_number": int32(23), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(22), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(21), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(20), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(19), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(18), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(17), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(16), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(15), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(14), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
},
"failed_count": 23,
"template_version_name": t1v1.Name,
},
{
"failed_builds": []map[string]any{
{"build_number": int32(123), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(122), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(121), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(120), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(119), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(118), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(117), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(116), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(115), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
{"build_number": int32(114), "workspace_name": w1.Name, "workspace_id": w1.ID, "workspace_owner_username": user1.Username},
},
"failed_count": 23,
"template_version_name": t1v2.Name,
},
},
"failed_count": 23,
"template_version_name": t1v1.Name,
},
{
"failed_builds": []map[string]interface{}{
{"build_number": int32(123), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(122), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(121), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(120), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(119), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(118), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(117), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(116), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(115), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
{"build_number": int32(114), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
},
"failed_count": 23,
"template_version_name": t1v2.Name,
},
})
})
coderd/notifications/testdata/rendered-templates/smtp/TemplateWorkspaceBuildsFailedReport.html.golden
Vendored
+98 -35
View File
@@ -1,6 +1,6 @@
From: system@coder.com
To: bobby@coder.com
Subject: Workspace builds failed for template "Bobby First Template"
Subject: Failed workspace builds report
Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
Date: Fri, 11 Oct 2024 09:03:06 +0000
Content-Type: multipart/alternative; boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
@@ -12,29 +12,51 @@ Content-Type: text/plain; charset=UTF-8
Hi Bobby,
Template Bobby First Template has failed to build 4/55 times over the last =
week.
The following templates have had build failures over the last week:
Bobby First Template failed to build 4/55 times
Bobby Second Template failed to build 5/50 times
Report:
Bobby First Template
bobby-template-version-1 failed 3 times:
mtojek / workspace-1 / #1234 (http://test.com/@mtojek/workspace-1/builds/12=
34)
johndoe / my-workspace-3 / #5678 (http://test.com/@johndoe/my-workspace-3/b=
uilds/5678)
jack / workwork / #774 (http://test.com/@jack/workwork/builds/774)
mtojek / workspace-1 / #1234 (http://test.com/@mtojek/workspace-1/build=
s/1234)
johndoe / my-workspace-3 / #5678 (http://test.com/@johndoe/my-workspace=
-3/builds/5678)
jack / workwork / #774 (http://test.com/@jack/workwork/builds/774)
bobby-template-version-2 failed 1 time:
ben / cool-workspace / #8888 (http://test.com/@ben/cool-workspace/build=
s/8888)
Bobby Second Template
bobby-template-version-1 failed 3 times:
daniellemaywood / workspace-9 / #9234 (http://test.com/@daniellemaywood=
/workspace-9/builds/9234)
johndoe / my-workspace-7 / #8678 (http://test.com/@johndoe/my-workspace=
-7/builds/8678)
jack / workworkwork / #374 (http://test.com/@jack/workworkwork/builds/3=
74)
bobby-template-version-2 failed 2 times:
ben / more-cool-workspace / #8878 (http://test.com/@ben/more-cool-works=
pace/builds/8878)
ben / less-cool-workspace / #8848 (http://test.com/@ben/less-cool-works=
pace/builds/8848)
ben / cool-workspace / #8888 (http://test.com/@ben/cool-workspace/builds/88=
88)
We recommend reviewing these issues to ensure future builds are successful.
View workspaces: http://test.com/workspaces?filter=3Dtemplate%3Abobby-first=
-template
View workspaces: http://test.com/workspaces?filter=3Did%3A24f5bd8f-1566-437=
4-9734-c3efa0454dc7+id%3A372a194b-dcde-43f1-b7cf-8a2f3d3114a0+id%3A1386d294=
-19c1-4351-89e2-6cae1afb9bfe+id%3A86fd99b1-1b6e-4b7e-b58e-0aee6e35c159+id%3=
Acd469690-b6eb-4123-b759-980be7a7b278+id%3Ac447d472-0800-4529-a836-788754d5=
e27d+id%3A919db6df-48f0-4dc1-b357-9036a2c40f86+id%3Ac8fb0652-9290-4bf2-a711=
-71b910243ac2+id%3A703d718d-2234-4990-9a02-5b1df6cf462a
--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
Content-Transfer-Encoding: quoted-printable
@@ -46,8 +68,7 @@ Content-Type: text/html; charset=UTF-8
<meta charset=3D"UTF-8" />
<meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
=3D1.0" />
<title>Workspace builds failed for template "Bobby First Template"</tit=
le>
<title>Failed workspace builds report</title>
</head>
<body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
@@ -62,35 +83,73 @@ er Logo" style=3D"height: 40px;" />
</div>
<h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
argin: 8px 0 32px; line-height: 1.5;">
Workspace builds failed for template "Bobby First Template"
Failed workspace builds report
</h1>
<div style=3D"line-height: 1.5;">
<p>Hi Bobby,</p>
<p>Template <strong>Bobby First Template</strong> has failed to bui=
ld <sup>4</sup>&frasl;<sub>55</sub> times over the last week.</p>
<p>The following templates have had build failures over the last we=
ek:</p>
<ul>
<li><p><strong>Bobby First Template</strong> failed to build <sup>4</sup>&f=
rasl;<sub>55</sub> times</p></li>
<li><p><strong>Bobby Second Template</strong> failed to build <sup>5</sup>&=
frasl;<sub>50</sub> times</p></li>
</ul>
<p><strong>Report:</strong></p>
<p><strong>bobby-template-version-1</strong> failed 3 times:</p>
<p><strong>Bobby First Template</strong></p>
<ul>
<li><a href=3D"http://test.com/@mtojek/workspace-1/builds/1234">mtojek / wo=
rkspace-1 / #1234</a><br>
</li>
<li><a href=3D"http://test.com/@johndoe/my-workspace-3/builds/5678">johndoe=
/ my-workspace-3 / #5678</a><br>
</li>
<li><a href=3D"http://test.com/@jack/workwork/builds/774">jack / workwork /=
#774</a><br>
</li>
</ul>
<li><p><strong>bobby-template-version-1</strong> failed 3 times:</p>
<p><strong>bobby-template-version-2</strong> failed 1 time:</p>
<ul>
<li><p><a href=3D"http://test.com/@mtojek/workspace-1/builds/1234">mtojek /=
workspace-1 / #1234</a></p></li>
<li><p><a href=3D"http://test.com/@johndoe/my-workspace-3/builds/5678">john=
doe / my-workspace-3 / #5678</a></p></li>
<li><p><a href=3D"http://test.com/@jack/workwork/builds/774">jack / workwor=
k / #774</a></p></li>
</ul></li>
<li><p><strong>bobby-template-version-2</strong> failed 1 time:</p>
<ul>
<li><a href=3D"http://test.com/@ben/cool-workspace/builds/8888">ben / cool-=
workspace / #8888</a><br>
</li>
</ul></li>
</ul>
<p><strong>Bobby Second Template</strong></p>
<ul>
<li><p><strong>bobby-template-version-1</strong> failed 3 times:</p>
<ul>
<li><p><a href=3D"http://test.com/@daniellemaywood/workspace-9/builds/9234"=
>daniellemaywood / workspace-9 / #9234</a></p></li>
<li><p><a href=3D"http://test.com/@johndoe/my-workspace-7/builds/8678">john=
doe / my-workspace-7 / #8678</a></p></li>
<li><p><a href=3D"http://test.com/@jack/workworkwork/builds/374">jack / wor=
kworkwork / #374</a></p></li>
</ul></li>
<li><p><strong>bobby-template-version-2</strong> failed 2 times:</p>
<ul>
<li><p><a href=3D"http://test.com/@ben/more-cool-workspace/builds/8878">ben=
/ more-cool-workspace / #8878</a></p></li>
<li><p><a href=3D"http://test.com/@ben/less-cool-workspace/builds/8848">ben=
/ less-cool-workspace / #8848</a></p></li>
</ul></li>
</ul>
<p>We recommend reviewing these issues to ensure future builds are successf=
@@ -98,10 +157,14 @@ ul.</p>
</div>
<div style=3D"text-align: center; margin-top: 32px;">
=20
<a href=3D"http://test.com/workspaces?filter=3Dtemplate%3Abobby-fir=
st-template" style=3D"display: inline-block; padding: 13px 24px; background=
-color: #020617; color: #f8fafc; text-decoration: none; border-radius: 8px;=
margin: 0 4px;">
<a href=3D"http://test.com/workspaces?filter=3Did%3A24f5bd8f-1566-4=
374-9734-c3efa0454dc7+id%3A372a194b-dcde-43f1-b7cf-8a2f3d3114a0+id%3A1386d2=
94-19c1-4351-89e2-6cae1afb9bfe+id%3A86fd99b1-1b6e-4b7e-b58e-0aee6e35c159+id=
%3Acd469690-b6eb-4123-b759-980be7a7b278+id%3Ac447d472-0800-4529-a836-788754=
d5e27d+id%3A919db6df-48f0-4dc1-b357-9036a2c40f86+id%3Ac8fb0652-9290-4bf2-a7=
11-71b910243ac2+id%3A703d718d-2234-4990-9a02-5b1df6cf462a" style=3D"display=
: inline-block; padding: 13px 24px; background-color: #020617; color: #f8fa=
fc; text-decoration: none; border-radius: 8px; margin: 0 4px;">
View workspaces
</a>
=20
coderd/notifications/testdata/rendered-templates/webhook/TemplateWorkspaceBuildsFailedReport.json.golden
Vendored
+93 -36
View File
@@ -3,7 +3,7 @@
"msg_id": "00000000-0000-0000-0000-000000000000",
"payload": {
"_version": "1.2",
"notification_name": "Report: Workspace Builds Failed For Template",
"notification_name": "Report: Workspace Builds Failed",
"notification_template_id": "00000000-0000-0000-0000-000000000000",
"user_id": "00000000-0000-0000-0000-000000000000",
"user_email": "bobby@coder.com",
@@ -12,56 +12,113 @@
"actions": [
{
"label": "View workspaces",
"url": "http://test.com/workspaces?filter=template%3Abobby-first-template"
"url": "http://test.com/workspaces?filter=id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000+id%3A00000000-0000-0000-0000-000000000000"
}
],
"labels": {
"template_display_name": "Bobby First Template",
"template_name": "bobby-first-template"
},
"labels": {},
"data": {
"failed_builds": 4,
"report_frequency": "week",
"template_versions": [
"templates": [
{
"failed_builds": [
"display_name": "Bobby First Template",
"failed_builds": 4,
"name": "bobby-first-template",
"total_builds": 55,
"versions": [
{
"build_number": 1234,
"workspace_name": "workspace-1",
"workspace_owner_username": "mtojek"
"failed_builds": [
{
"build_number": 1234,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "workspace-1",
"workspace_owner_username": "mtojek"
},
{
"build_number": 5678,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "my-workspace-3",
"workspace_owner_username": "johndoe"
},
{
"build_number": 774,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "workwork",
"workspace_owner_username": "jack"
}
],
"failed_count": 3,
"template_version_name": "bobby-template-version-1"
},
{
"build_number": 5678,
"workspace_name": "my-workspace-3",
"workspace_owner_username": "johndoe"
},
{
"build_number": 774,
"workspace_name": "workwork",
"workspace_owner_username": "jack"
"failed_builds": [
{
"build_number": 8888,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "cool-workspace",
"workspace_owner_username": "ben"
}
],
"failed_count": 1,
"template_version_name": "bobby-template-version-2"
}
],
"failed_count": 3,
"template_version_name": "bobby-template-version-1"
]
},
{
"failed_builds": [
"display_name": "Bobby Second Template",
"failed_builds": 5,
"name": "bobby-second-template",
"total_builds": 50,
"versions": [
{
"build_number": 8888,
"workspace_name": "cool-workspace",
"workspace_owner_username": "ben"
"failed_builds": [
{
"build_number": 9234,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "workspace-9",
"workspace_owner_username": "daniellemaywood"
},
{
"build_number": 8678,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "my-workspace-7",
"workspace_owner_username": "johndoe"
},
{
"build_number": 374,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "workworkwork",
"workspace_owner_username": "jack"
}
],
"failed_count": 3,
"template_version_name": "bobby-template-version-1"
},
{
"failed_builds": [
{
"build_number": 8878,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "more-cool-workspace",
"workspace_owner_username": "ben"
},
{
"build_number": 8848,
"workspace_id": "00000000-0000-0000-0000-000000000000",
"workspace_name": "less-cool-workspace",
"workspace_owner_username": "ben"
}
],
"failed_count": 2,
"template_version_name": "bobby-template-version-2"
}
],
"failed_count": 1,
"template_version_name": "bobby-template-version-2"
]
}
],
"total_builds": 55
]
},
"targets": null
},
"title": "Workspace builds failed for template \"Bobby First Template\"",
"title_markdown": "Workspace builds failed for template \"Bobby First Template\"",
"body": "Template Bobby First Template has failed to build 4/55 times over the last week.\n\nReport:\n\nbobby-template-version-1 failed 3 times:\n\nmtojek / workspace-1 / #1234 (http://test.com/@mtojek/workspace-1/builds/1234)\njohndoe / my-workspace-3 / #5678 (http://test.com/@johndoe/my-workspace-3/builds/5678)\njack / workwork / #774 (http://test.com/@jack/workwork/builds/774)\n\nbobby-template-version-2 failed 1 time:\n\nben / cool-workspace / #8888 (http://test.com/@ben/cool-workspace/builds/8888)\n\nWe recommend reviewing these issues to ensure future builds are successful.",
"body_markdown": "Template **Bobby First Template** has failed to build 4/55 times over the last week.\n\n**Report:**\n\n**bobby-template-version-1** failed 3 times:\n\n* [mtojek / workspace-1 / #1234](http://test.com/@mtojek/workspace-1/builds/1234)\n* [johndoe / my-workspace-3 / #5678](http://test.com/@johndoe/my-workspace-3/builds/5678)\n* [jack / workwork / #774](http://test.com/@jack/workwork/builds/774)\n\n**bobby-template-version-2** failed 1 time:\n\n* [ben / cool-workspace / #8888](http://test.com/@ben/cool-workspace/builds/8888)\n\nWe recommend reviewing these issues to ensure future builds are successful."
"title": "Failed workspace builds report",
"title_markdown": "Failed workspace builds report",
"body": "The following templates have had build failures over the last week:\n\nBobby First Template failed to build 4/55 times\nBobby Second Template failed to build 5/50 times\n\nReport:\n\nBobby First Template\n\nbobby-template-version-1 failed 3 times:\n mtojek / workspace-1 / #1234 (http://test.com/@mtojek/workspace-1/builds/1234)\n johndoe / my-workspace-3 / #5678 (http://test.com/@johndoe/my-workspace-3/builds/5678)\n jack / workwork / #774 (http://test.com/@jack/workwork/builds/774)\nbobby-template-version-2 failed 1 time:\n ben / cool-workspace / #8888 (http://test.com/@ben/cool-workspace/builds/8888)\n\n\nBobby Second Template\n\nbobby-template-version-1 failed 3 times:\n daniellemaywood / workspace-9 / #9234 (http://test.com/@daniellemaywood/workspace-9/builds/9234)\n johndoe / my-workspace-7 / #8678 (http://test.com/@johndoe/my-workspace-7/builds/8678)\n jack / workworkwork / #374 (http://test.com/@jack/workworkwork/builds/374)\nbobby-template-version-2 failed 2 times:\n ben / more-cool-workspace / #8878 (http://test.com/@ben/more-cool-workspace/builds/8878)\n ben / less-cool-workspace / #8848 (http://test.com/@ben/less-cool-workspace/builds/8848)\n\n\nWe recommend reviewing these issues to ensure future builds are successful.",
"body_markdown": "The following templates have had build failures over the last week:\n\n- **Bobby First Template** failed to build 4/55 times\n\n- **Bobby Second Template** failed to build 5/50 times\n\n\n**Report:**\n\n**Bobby First Template**\n\n- **bobby-template-version-1** failed 3 times:\n\n - [mtojek / workspace-1 / #1234](http://test.com/@mtojek/workspace-1/builds/1234)\n\n - [johndoe / my-workspace-3 / #5678](http://test.com/@johndoe/my-workspace-3/builds/5678)\n\n - [jack / workwork / #774](http://test.com/@jack/workwork/builds/774)\n\n\n- **bobby-template-version-2** failed 1 time:\n\n - [ben / cool-workspace / #8888](http://test.com/@ben/cool-workspace/builds/8888)\n\n\n\n**Bobby Second Template**\n\n- **bobby-template-version-1** failed 3 times:\n\n - [daniellemaywood / workspace-9 / #9234](http://test.com/@daniellemaywood/workspace-9/builds/9234)\n\n - [johndoe / my-workspace-7 / #8678](http://test.com/@johndoe/my-workspace-7/builds/8678)\n\n - [jack / workworkwork / #374](http://test.com/@jack/workworkwork/builds/374)\n\n\n- **bobby-template-version-2** failed 2 times:\n\n - [ben / more-cool-workspace / #8878](http://test.com/@ben/more-cool-workspace/builds/8878)\n\n - [ben / less-cool-workspace / #8848](http://test.com/@ben/less-cool-workspace/builds/8848)\n\n\n\n\nWe recommend reviewing these issues to ensure future builds are successful."
}