diff --git a/api/types/cluster_alert.go b/api/types/cluster_alert.go index d5540b7ac07..24bee8f5a20 100644 --- a/api/types/cluster_alert.go +++ b/api/types/cluster_alert.go @@ -115,7 +115,6 @@ func (c *ClusterAlert) setDefaults() { // CheckAndSetDefaults verfies required fields. func (c *ClusterAlert) CheckAndSetDefaults() error { c.setDefaults() - if c.Version != V1 { return trace.BadParameter("unsupported cluster alert version: %s", c.Version) } @@ -128,14 +127,8 @@ func (c *ClusterAlert) CheckAndSetDefaults() error { return trace.BadParameter("alert name must be specified") } - if c.Spec.Message == "" { - return trace.BadParameter("alert message must be specified") - } - - for _, c := range c.Spec.Message { - if unicode.IsControl(c) { - return trace.BadParameter("control characters not supported in alerts") - } + if err := c.CheckMessage(); err != nil { + return trace.Wrap(err) } for key, val := range c.Metadata.Labels { @@ -149,6 +142,19 @@ func (c *ClusterAlert) CheckAndSetDefaults() error { return nil } +func (c *ClusterAlert) CheckMessage() error { + if c.Spec.Message == "" { + return trace.BadParameter("alert message must be specified") + } + + for _, c := range c.Spec.Message { + if unicode.IsControl(c) { + return trace.BadParameter("control characters not supported in alerts") + } + } + return nil +} + // Match checks if the given cluster alert matches this query. func (r *GetClusterAlertsRequest) Match(alert ClusterAlert) bool { if alert.Spec.Severity < r.Severity { diff --git a/lib/auth/auth.go b/lib/auth/auth.go index 3bc61d85b64..31dc2b5e92c 100644 --- a/lib/auth/auth.go +++ b/lib/auth/auth.go @@ -36,6 +36,7 @@ import ( insecurerand "math/rand" "net" "net/url" + "os" "strings" "sync" "time" @@ -51,6 +52,7 @@ import ( "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "golang.org/x/crypto/ssh" + "golang.org/x/mod/semver" "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/client" @@ -82,6 +84,8 @@ import ( "github.com/gravitational/teleport/lib/tlsca" "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/teleport/lib/utils/interval" + vc "github.com/gravitational/teleport/lib/versioncontrol" + "github.com/gravitational/teleport/lib/versioncontrol/github" ) const ( @@ -208,7 +212,7 @@ func NewServer(cfg *InitConfig, opts ...ServerOption) (*Server, error) { SessionTrackerService: cfg.SessionTrackerService, Enforcer: cfg.Enforcer, ConnectionsDiagnostic: cfg.ConnectionsDiagnostic, - Status: cfg.Status, + StatusInternal: cfg.Status, } closeCtx, cancelFunc := context.WithCancel(context.TODO()) @@ -260,7 +264,7 @@ type Services struct { services.SessionTrackerService services.Enforcer services.ConnectionsDiagnostic - services.Status + services.StatusInternal types.Events events.IAuditLog } @@ -459,6 +463,24 @@ func (a *Server) runPeriodicOperations() { defer ticker.Stop() defer heartbeatCheckTicker.Stop() defer promTicker.Stop() + + firstReleaseCheck := utils.FullJitter(time.Hour * 6) + + // this environment variable is "unstable" since it will be deprecated + // by an upcoming tctl command. currently exists for testing purposes only. + if os.Getenv("TELEPORT_UNSTABLE_VC_SYNC_ON_START") == "yes" { + firstReleaseCheck = utils.HalfJitter(time.Second * 10) + } + + // note the use of FullJitter for the releases check interval. this lets us ensure + // that frequent restarts don't prevent checks from happening despite the infrequent + // effective check rate. + releaseCheck := interval.New(interval.Config{ + Duration: time.Hour * 24, + FirstDuration: firstReleaseCheck, + Jitter: utils.NewFullJitter(), + }) + defer releaseCheck.Stop() for { select { case <-a.closeCtx.Done(): @@ -486,10 +508,93 @@ func (a *Server) runPeriodicOperations() { heartbeatsMissedByAuth.Set(float64(missedKeepAliveCount)) case <-promTicker.C: a.updateVersionMetrics() + case <-releaseCheck.Next(): + a.checkForReleases(ctx) } } } +// checkForReleases loads latest github releases and generates an alert if +// an appropriate upgrade is available. +func (a *Server) checkForReleases(ctx context.Context) { + const alertID = "upgrade-suggestion" + log.Debug("Checking for new teleport releases via github api.") + + // NOTE: essentially everything in this function is going to be + // scrapped/replaced once the inventory and version-control systems + // are a bit further along. + + var loadFailed bool + current := vc.Normalize(teleport.Version) + + latest, err := github.LatestStable() + if err != nil { + log.Warnf("Failed to load github releases: %v (this will not impact teleport functionality)", err) + loadFailed = true + latest = current + } + + // use visitor to find the oldest version among connected instances + // TODO(fspmarshall): replace this check as soon as we have a backend inventory repr. using + // connected instances is a poor approximation and may lead to missed notifications if auth + // server is up to date, but instances not connected to this auth need update. + var instanceVisitor vc.Visitor + a.inventory.Iter(func(handle inventory.UpstreamHandle) { + instanceVisitor.Visit(vc.Normalize(handle.Hello().Version)) + }) + + msg := makeUpgradeSuggestionMsg(current, latest, instanceVisitor.Oldest()) + + if msg != "" { + alert, err := types.NewClusterAlert( + alertID, + msg, + types.WithAlertLabel(types.AlertOnLogin, "yes"), + types.WithAlertLabel(types.AlertPermitAll, "yes"), + ) + if err != nil { + log.Warnf("Failed to build %s alert: %v (this is a bug)", alertID, err) + return + } + if err := a.UpsertClusterAlert(ctx, alert); err != nil { + log.Warnf("Failed to set %s alert: %v", alertID, err) + return + } + } else if !loadFailed { + log.Debugf("Cluster appears up to date, clearing %s alert.", alertID) + err := a.DeleteClusterAlert(ctx, alertID) + if err != nil && !trace.IsNotFound(err) { + log.Warnf("Failed to delete %s alert: %v", alertID, err) + } + } +} + +// makeUpgradeSuggestionMsg generates an upgrade suggestion alert msg if one is +// needed (returns "" if everything looks up to date). +func makeUpgradeSuggestionMsg(current, latest, oldestInstance string) string { + // check if this teleport instance wants upgrade + if semver.Compare(latest, current) == 1 { + // specialize the message a bit to distinguish between a major and minor upgrade. + var msg string + if semver.Major(latest) != semver.Major(current) { + log.Debugf("Generating alert msg for new major version. current=%s, latest=%s", current, latest) + msg = fmt.Sprintf("The latest major version of Teleport is %s. Please consider upgrading your Cluster.", semver.Major(latest)) + } else { + log.Debugf("Generating alert msg for new minor or patch release. current=%s, latest=%s", current, latest) + msg = fmt.Sprintf("Teleport %s is now available, please consider upgrading your cluster.", latest) + } + return msg + } + + if oldestInstance != "" && semver.Compare(latest, oldestInstance) == 1 { + log.Debugf("Generating alert msg for older peripheral instance(s). latest=%s, oldestInstance=%s", latest, oldestInstance) + // at least one connected instance is older than this auth server. + return "Some Agents within this Cluster are running an older version of Teleport. Please consider upgrading them." + } + + return "" +} + // updateVersionMetrics leverages the cache to report all versions of teleport servers connected to the // cluster via prometheus metrics func (a *Server) updateVersionMetrics() { diff --git a/lib/auth/init.go b/lib/auth/init.go index 0da581de1c5..281da4ea0a6 100644 --- a/lib/auth/init.go +++ b/lib/auth/init.go @@ -132,7 +132,7 @@ type InitConfig struct { Databases services.Databases // Status is a service that manages cluster status info. - Status services.Status + Status services.StatusInternal // Roles is a set of roles to create Roles []types.Role diff --git a/lib/client/api.go b/lib/client/api.go index db2e1f857d9..fbdc4d0c8e0 100644 --- a/lib/client/api.go +++ b/lib/client/api.go @@ -2441,6 +2441,24 @@ func (tc *TeleportClient) ListNodesWithFilters(ctx context.Context) ([]types.Ser return servers, nil } +// GetClusterAlerts returns a list of matching alerts from the current cluster. +func (tc *TeleportClient) GetClusterAlerts(ctx context.Context, req types.GetClusterAlertsRequest) ([]types.ClusterAlert, error) { + ctx, span := tc.Tracer.Start(ctx, + "teleportClient/GetClusterAlerts", + oteltrace.WithSpanKind(oteltrace.SpanKindClient), + ) + defer span.End() + + proxyClient, err := tc.ConnectToProxy(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + defer proxyClient.Close() + + alerts, err := proxyClient.GetClusterAlerts(ctx, req) + return alerts, trace.Wrap(err) +} + // ListNodesWithFiltersAllClusters returns a map of all nodes in all clusters connected to this proxy. func (tc *TeleportClient) ListNodesWithFiltersAllClusters(ctx context.Context) (map[string][]types.Server, error) { proxyClient, err := tc.ConnectToProxy(ctx) diff --git a/lib/client/client.go b/lib/client/client.go index 2640f213b12..6b2fa35c11a 100644 --- a/lib/client/client.go +++ b/lib/client/client.go @@ -721,6 +721,24 @@ func (proxy *ProxyClient) FindNodesByFilters(ctx context.Context, req proto.List return servers, trace.Wrap(err) } +func (proxy *ProxyClient) GetClusterAlerts(ctx context.Context, req types.GetClusterAlertsRequest) ([]types.ClusterAlert, error) { + ctx, span := proxy.Tracer.Start( + ctx, + "proxyClient/GetClusterAlerts", + oteltrace.WithSpanKind(oteltrace.SpanKindClient), + ) + defer span.End() + + site, err := proxy.CurrentClusterAccessPoint(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + defer site.Close() + + alerts, err := site.GetClusterAlerts(ctx, req) + return alerts, trace.Wrap(err) +} + // FindNodesByFiltersForCluster returns list of the nodes in a specified cluster which have filters matched. func (proxy *ProxyClient) FindNodesByFiltersForCluster(ctx context.Context, req proto.ListResourcesRequest, cluster string) ([]types.Server, error) { ctx, span := proxy.Tracer.Start( diff --git a/lib/services/local/status.go b/lib/services/local/status.go index 3692344e90d..1bf821518a5 100644 --- a/lib/services/local/status.go +++ b/lib/services/local/status.go @@ -138,4 +138,12 @@ func (s *StatusService) UpsertClusterAlert(ctx context.Context, alert types.Clus return trace.Wrap(err) } +func (s *StatusService) DeleteClusterAlert(ctx context.Context, alertID string) error { + err := s.Backend.Delete(ctx, backend.Key(clusterAlertPrefix, alertID)) + if trace.IsNotFound(err) { + return trace.NotFound("cluster alert %q not found", alertID) + } + return trace.Wrap(err) +} + const clusterAlertPrefix = "cluster-alerts" diff --git a/lib/services/status.go b/lib/services/status.go index 9d85111ad77..5a8521b9489 100644 --- a/lib/services/status.go +++ b/lib/services/status.go @@ -30,3 +30,11 @@ type Status interface { // UpsertClusterAlert creates the specified alert, overwriting any preexising alert with the same ID. UpsertClusterAlert(ctx context.Context, alert types.ClusterAlert) error } + +// StatusInternal extends Status with auth-internal methods. +type StatusInternal interface { + Status + + // DeleteClusterAlert deletes the cluster alert with the specified ID. + DeleteClusterAlert(ctx context.Context, alertID string) error +} diff --git a/lib/utils/retry.go b/lib/utils/retry.go index 2a1bd6d0a45..feb2d612cd4 100644 --- a/lib/utils/retry.go +++ b/lib/utils/retry.go @@ -38,6 +38,11 @@ var HalfJitter = NewHalfJitter() // repeated calls. var SeventhJitter = NewSeventhJitter() +// FullJitter is a global jitter instance used for one-off jitters. +// Prefer instantiating a new jitter instance for operations that require +// repeated calls +var FullJitter = NewFullJitter() + // Jitter is a function which applies random jitter to a // duration. Used to randomize backoff values. Must be // safe for concurrent usage. diff --git a/lib/versioncontrol/github/github.go b/lib/versioncontrol/github/github.go index 93e78ec2613..ccfa6f22a33 100644 --- a/lib/versioncontrol/github/github.go +++ b/lib/versioncontrol/github/github.go @@ -23,6 +23,8 @@ import ( "net/http" "github.com/gravitational/teleport" + vc "github.com/gravitational/teleport/lib/versioncontrol" + "github.com/gravitational/trace" "github.com/sirupsen/logrus" @@ -37,6 +39,31 @@ var log = logrus.WithFields(logrus.Fields{ trace.Component: teleport.ComponentVersionControl, }) +// LatestStable gets the most recent "stable" (not pre-release) version. +// NOTE: this may return a version *older* than the current teleport version +// if the current teleport version is a newer prerelease. +func LatestStable() (string, error) { + return latestStable(Iterator{}, vc.Normalize(teleport.Version)) +} + +// latestStable is the business logic of LatestStable, broken out for testing purposes. +func latestStable(iter Iterator, current string) (string, error) { + if !semver.IsValid(current) { + return "", trace.BadParameter("cannot get latest stable, invalid semver: %q", current) + } + // we only care about newer releaseas, so set halting point to 'current' + iter.halt = current + + // set up visitor that will default to 'current' if nothing newer is observed. + var visitor vc.Visitor + for iter.Next() { + for _, r := range iter.Page() { + visitor.Visit(r.Version) + } + } + return visitor.Latest(), trace.Wrap(iter.Error()) +} + // defaultHaltingPoint represents the default cutoff for the iterator. this value // is expanded/truncated to `.` so e.g. `v1` would halt iteration on the // first `v1.0.X` release observed, and `v1.2.3` would halt iteration on the first diff --git a/lib/versioncontrol/github/github_test.go b/lib/versioncontrol/github/github_test.go index 603aeae057f..47b4ebc7ee5 100644 --- a/lib/versioncontrol/github/github_test.go +++ b/lib/versioncontrol/github/github_test.go @@ -210,3 +210,67 @@ func TestCachedReleases(t *testing.T) { } } + +func TestLatestStable(t *testing.T) { + tts := []struct { + pages []string + expect string + desc string + }{ + { + pages: []string{ + page1, + page2, + page3, + }, + expect: "v10.0.2", + desc: "all cached pages", + }, + { + pages: []string{ + page2, + page3, + }, + expect: "v9.3.7", + desc: "earlier pages subset", + }, + { + pages: []string{ + page1, + }, + expect: "v10.0.2", + desc: "page 1 only", + }, + { + pages: []string{ + page2, + }, + expect: "v9.3.7", + desc: "page 2 only", + }, + { + pages: []string{ + page3, + }, + expect: "v9.2.4", + desc: "page 3 only", + }, + } + + for _, tt := range tts { + var iter Iterator + iter.size = cachedPageSize + iter.getPage = func(n, size int) ([]release, error) { + if n > len(tt.pages) { + return nil, nil + } + page := tt.pages[n-1] + return parsePage([]byte(page)) + } + + latest, err := latestStable(iter, "v1.2.3") + require.NoError(t, err) + + require.Equal(t, tt.expect, latest) + } +} diff --git a/lib/versioncontrol/versioncontrol.go b/lib/versioncontrol/versioncontrol.go index c52260ebb36..bc3cfd65963 100644 --- a/lib/versioncontrol/versioncontrol.go +++ b/lib/versioncontrol/versioncontrol.go @@ -15,3 +15,69 @@ limitations under the License. */ package versioncontrol + +import ( + "fmt" + + "golang.org/x/mod/semver" +) + +// Normalize attaches the expected `v` prefix to a version string if the supplied +// version is currently invalid, and attaching the prefix makes it valid. Useful for normalizing +// the teleport.Version variable. +// NOTE: this isn't equivalent to "canonicalization" which makes equivalent version strings +// comparable via `==`. version strings returned by this function should still only be compared +// via `semver.Compare`. +func Normalize(v string) string { + if semver.IsValid(v) { + return v + } + + if n := fmt.Sprintf("v%s", v); semver.IsValid(n) { + return n + } + + return v +} + +// Visitor is a helper for aggregating information about observed versions. Useful for +// getting latest/oldest version observed during iteration/pagination. Zero value omits +// prereleases. +type Visitor struct { + PermitPrerelease bool + latest string + oldest string +} + +// Visit processes the supplied version string. If ok is false, the string was +// ignored due to being invalid, or because if was a prerelease if the visitor +// is configured to ignore those. +func (v *Visitor) Visit(s string) (ok bool) { + if !semver.IsValid(s) { + return false + } + + if !v.PermitPrerelease && semver.Prerelease(s) != "" { + return false + } + + if v.latest == "" || semver.Compare(v.latest, s) == -1 { + v.latest = s + } + + if v.oldest == "" || semver.Compare(v.oldest, s) == 1 { + v.oldest = s + } + + return true +} + +// Latest gets the most recent version string from among those observed. +func (v *Visitor) Latest() string { + return v.latest +} + +// Oldest gets the oldest version string from among those observed. +func (v *Visitor) Oldest() string { + return v.oldest +} diff --git a/lib/versioncontrol/versioncontrol_test.go b/lib/versioncontrol/versioncontrol_test.go new file mode 100644 index 00000000000..a124f597de6 --- /dev/null +++ b/lib/versioncontrol/versioncontrol_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package versioncontrol + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestVisitor(t *testing.T) { + tts := []struct { + versions []string + latest string + oldest string + permitPrerelease bool + desc string + }{ + { + versions: []string{ + "v1.2.3", + "v2.3.4-alpha.1", + }, + latest: "v1.2.3", + oldest: "v1.2.3", + desc: "one stable release", + }, + { + versions: []string{ + "v1.2.3", + "v2.3.4", + "v2.2.2", + "v3.5.7", + "invalid", + "v0.0.1-alpha.2", + }, + latest: "v3.5.7", + oldest: "v1.2.3", + desc: "mixed releases", + }, + { + versions: []string{ + "invalid", + "12356", + "127.0.0.1:8080", + }, + desc: "all invalid", + }, + { + versions: []string{ + "v3.4.5-alpha.1", + "v3.4.4", + "v0.1.2-alpha.2", + "v0.1.11", + }, + latest: "v3.4.5-alpha.1", + oldest: "v0.1.2-alpha.2", + permitPrerelease: true, + desc: "prerelease on", + }, + { + versions: []string{ + "v3.4.5-alpha.1", + "v3.4.4", + "v0.1.2-alpha.2", + "v0.1.11", + }, + latest: "v3.4.4", + oldest: "v0.1.11", + permitPrerelease: false, + desc: "prerelease off", + }, + { + versions: []string{ + "v3.4.5-alpha.1", + "v3.4.4", + "v0.1.12-alpha.2", + "v0.1.2", + }, + latest: "v3.4.5-alpha.1", + oldest: "v0.1.2", + permitPrerelease: true, + desc: "prerelease on (mixed)", + }, + } + + for _, tt := range tts { + visitor := Visitor{ + PermitPrerelease: tt.permitPrerelease, + } + + for _, v := range tt.versions { + visitor.Visit(v) + } + + require.Equal(t, tt.latest, visitor.Latest(), tt.desc) + require.Equal(t, tt.oldest, visitor.Oldest(), tt.desc) + } +} diff --git a/tool/tctl/common/alert_command.go b/tool/tctl/common/alert_command.go index 54ca41da983..edba4f0735f 100644 --- a/tool/tctl/common/alert_command.go +++ b/tool/tctl/common/alert_command.go @@ -25,6 +25,7 @@ import ( "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/asciitable" "github.com/gravitational/teleport/lib/auth" + libclient "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/service" "github.com/gravitational/kingpin" @@ -39,6 +40,8 @@ type AlertCommand struct { message string + labels string + verbose bool alertList *kingpin.CmdClause @@ -52,9 +55,11 @@ func (c *AlertCommand) Initialize(app *kingpin.Application, config *service.Conf c.alertList = alert.Command("list", "List cluster alerts").Alias("ls") c.alertList.Flag("verbose", "Show detailed alert info").Short('v').BoolVar(&c.verbose) + c.alertList.Flag("labels", labelHelp).StringVar(&c.labels) c.alertCreate = alert.Command("create", "Create cluster alerts").Hidden() c.alertCreate.Arg("message", "Alert body message").Required().StringVar(&c.message) + c.alertCreate.Flag("labels", labelHelp).StringVar(&c.labels) } // TryRun takes the CLI command as an argument (like "alerts ls") and executes it. @@ -71,8 +76,13 @@ func (c *AlertCommand) TryRun(ctx context.Context, cmd string, client auth.Clien } func (c *AlertCommand) List(ctx context.Context, client auth.ClientI) error { + labels, err := libclient.ParseLabelSpec(c.labels) + if err != nil { + return trace.Wrap(err) + } + alerts, err := client.GetClusterAlerts(ctx, types.GetClusterAlertsRequest{ - // TODO(fspmarshall): support query parameters + Labels: labels, }) if err != nil { return trace.Wrap(err) @@ -115,9 +125,17 @@ func (c *AlertCommand) List(ctx context.Context, client auth.ClientI) error { } func (c *AlertCommand) Create(ctx context.Context, client auth.ClientI) error { + labels, err := libclient.ParseLabelSpec(c.labels) + if err != nil { + return trace.Wrap(err) + } + alert, err := types.NewClusterAlert(uuid.New().String(), c.message) if err != nil { return trace.Wrap(err) } + + alert.Metadata.Labels = labels + return trace.Wrap(client.UpsertClusterAlert(ctx, alert)) } diff --git a/tool/tsh/tsh.go b/tool/tsh/tsh.go index 854ec524ed4..42cfa9480ae 100644 --- a/tool/tsh/tsh.go +++ b/tool/tsh/tsh.go @@ -1558,10 +1558,34 @@ func onLogin(cf *CLIConf) error { if err != nil { return trace.Wrap(err) } + for _, warning := range resp.LicenseWarnings { fmt.Fprintf(os.Stderr, "%s\n\n", warning) } + // get any "on login" alerts + alerts, err := tc.GetClusterAlerts(cf.Context, types.GetClusterAlertsRequest{ + Labels: map[string]string{ + types.AlertOnLogin: "yes", + }, + }) + if err != nil && !trace.IsNotImplemented(err) { + return trace.Wrap(err) + } + + types.SortClusterAlerts(alerts) + + for _, alert := range alerts { + if err := alert.CheckMessage(); err != nil { + log.Warnf("Skipping invalid alert %q: %v", alert.Metadata.Name, err) + } + fmt.Fprintf(os.Stderr, "%s\n\n", alert.Spec.Message) + } + // NOTE: we currently print all alerts that are marked as `on-login`, because we + // don't use the alert API very heavily. If we start to make more use of it, we + // could probably add a separate `tsh alerts ls` command, and truncate the list + // with a message like "run 'tsh alerts ls' to view N additional alerts". + return nil }