mirror of
https://github.com/gravitational/teleport.git
synced 2026-10-25 00:19:07 +08:00
Trent Clarke
97c18fa1a9
Fixes #7606, where a node doesn't notice when the tunnel port changes. Imagine you have a cluster with a node connected in via a tunnel through a proxy `proxy.example.com` on port `3024` Now change the proxy config so that `tunnel_public_address` is `proxy.example.com:4024`. You either restart the proxy, or reload the proxy config with a `SIGHUP`. ...and then the node a) loses its connection to auth (because the tunnel is gone), and b) _doesn't reconnect_, because even though the proxy address hasn't changed, the node has cached the old tunnel_public_address and keeps trying to connect to that. You can always manually restart the node to have it reconnect, but that would be a pain if you have thousands of nodes. In order to not have to manually restart all nodes, this change implements a check for a connection failures to the auth server, and re-starts the node if there are multiple connection failures in a given period of time. The check as-implemented piggybacks on the node's "common.rotate" service, which can already restart the node in certain circumstances, and uses the success of the periodic rotation sync as a proxy for the health of the node's connection to the auth server. See-Also: #7606
94 lines
3.0 KiB
Go
94 lines
3.0 KiB
Go
// Copyright 2021 Gravitational, Inc
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package utils
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jonboulle/clockwork"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestTimedCounterReturnsZeroOnConstruction(t *testing.T) {
|
|
uut := NewTimedCounter(clockwork.NewFakeClock(), time.Second)
|
|
require.Zero(t, uut.Count())
|
|
}
|
|
|
|
func TestTimedCounterIncrement(t *testing.T) {
|
|
clock := clockwork.NewFakeClock()
|
|
uut := NewTimedCounter(clock, time.Second)
|
|
require.Equal(t, uut.Increment(), 1)
|
|
}
|
|
|
|
func TestTimedCounterExpiresEvents(t *testing.T) {
|
|
// Given a counter with a 10-second cutoff, primed with events at 1 second
|
|
// intervals
|
|
clock := clockwork.NewFakeClock()
|
|
uut := NewTimedCounter(clock, 10*time.Second)
|
|
for i := 1; i <= 5; i++ {
|
|
require.Equal(t, uut.Increment(), i)
|
|
require.Equal(t, uut.Count(), i)
|
|
clock.Advance(1 * time.Second)
|
|
}
|
|
|
|
// When I advance the clock to a time slightly earlier than the 10s
|
|
// cutoff for the first event...
|
|
clock.Advance(4900 * time.Millisecond) // 9.9s "after" the first event
|
|
|
|
// Expect that the all the events are still counted
|
|
require.Equal(t, uut.Count(), 5)
|
|
|
|
// When I advance the clock to just *after* the first event's cutoff...
|
|
clock.Advance(200 * time.Millisecond) // 10.1s "after" the first event
|
|
|
|
// Expect that the first event has expired and is no longer considered
|
|
// by the counter
|
|
require.Equal(t, uut.Count(), 4)
|
|
|
|
// When I advance the clock past another 3 events' cutoff times
|
|
clock.Advance(3 * time.Second)
|
|
|
|
// Expect that there is only one event left to count
|
|
require.Equal(t, uut.Count(), 1)
|
|
|
|
// When I advance the clock well past the final event's cutoff
|
|
clock.Advance(100 * time.Second)
|
|
|
|
// Expect that there are no events left to count
|
|
require.Equal(t, uut.Count(), 0)
|
|
}
|
|
|
|
func TestTimedCounterIncrementExpiresValues(t *testing.T) {
|
|
// Given a counter with a 10-second cutoff, primed with 5 events at 1-
|
|
// second intervals
|
|
clock := clockwork.NewFakeClock()
|
|
|
|
uut := NewTimedCounter(clock, 10*time.Second)
|
|
for i := 1; i <= 5; i++ {
|
|
require.Equal(t, uut.Increment(), i)
|
|
require.Equal(t, uut.Count(), i)
|
|
clock.Advance(1 * time.Second)
|
|
}
|
|
|
|
// When I advance the clock to 12.5s after the first event, by which time
|
|
// events at 0s, 1s and 2s should have expired.
|
|
clock.Advance(7500 * time.Millisecond)
|
|
|
|
// Expect that incrementing the count handles expiring three events and
|
|
// adding a new one (net result: 3 "live" events remaining)
|
|
require.Equal(t, 3, uut.Increment())
|
|
}
|