From 66f1535c1ed39134cfde1e2c0f60a60c00a179ca Mon Sep 17 00:00:00 2001 From: Brian Joerger Date: Tue, 11 Apr 2023 10:27:26 -0700 Subject: [PATCH] * Add --mlock flag with auto, off, best_effort, and strict options. (#24236) * Default headless to --mlock=best_effort to reduce errors to a debug log. * Add error for non-linux operating systems using headless. * Add a better mlock error message and add corresponding troubleshooting docs. --- docs/cspell.json | 1 + .../pages/access-controls/guides/headless.mdx | 35 +++++++++- docs/pages/reference/cli.mdx | 1 + lib/utils/mlock/mlock_linux.go | 1 - lib/utils/mlock/mlock_unsupported.go | 9 ++- tool/tsh/tsh.go | 64 +++++++++++++++++-- 6 files changed, 102 insertions(+), 9 deletions(-) diff --git a/docs/cspell.json b/docs/cspell.json index d22268c2f06..7b205e72891 100644 --- a/docs/cspell.json +++ b/docs/cspell.json @@ -458,6 +458,7 @@ "microk", "minikube", "minikube's", + "mlock", "mongodbatlas", "mongosh", "mpghq", diff --git a/docs/pages/access-controls/guides/headless.mdx b/docs/pages/access-controls/guides/headless.mdx index 4a634d2738e..1916b2e8b54 100644 --- a/docs/pages/access-controls/guides/headless.mdx +++ b/docs/pages/access-controls/guides/headless.mdx @@ -150,4 +150,37 @@ $ tsh ssh --headless --proxy=proxy.example.com --user=alice server01 # # tsh headless approve --user=alice --proxy=proxy.example.com 864cccd9-2425-46d9-a9f2-636387e66ebf # # User approves through link -``` \ No newline at end of file +``` + +## Troubleshooting + +### "WARN: Failed to lock system memory for headless login: ..." + +When using Headless WebAuthn, `tsh` does not write private key and certificate data +to disk(`~/.tsh`). Instead, `tsh` holds these secrets in memory for the duration of +the request. Additionally, it will try to lock the process memory to further protect +the secrets from being stolen by other users on a shared machine. + +Below are some of the specific warning messages you may run into and how to fix them: + +#### "operation not permitted" OR "cannot allocate memory" + +In order to lock the process memory, your OS user must have permission to lock +the amount of memory needed. Use `ulimit -l` to check your OS user's current limit. +The exact amount of memory needed may vary from system to system, so we recommend +updating your ulimit to unlimited, with either `ulimit -l unlimited` or by adding +the line ` hard memlock unlimited` to your `/etc/security/limits.conf`. + +#### "memory locking is not supported on non-linux operating systems" + +The `mlockall` syscall is only supported on Linux operating systems. This means +that on other operating systems, the memory lock attempt will always fail and +output the warning. We recommend only using Headless WebAuthn on Linux machines +for the best level of security on shared machines. + +#### Disable mlock + +If the above solutions are not feasible in your environment, you can also disable +the memory locking requirement by setting the `--mlock` flag or `TELEPORT_MLOCK_MODE` +environment variable to `off` or `best_effort`. This is not recommended in production +environments on shared systems where a memory swap attack is possible. \ No newline at end of file diff --git a/docs/pages/reference/cli.mdx b/docs/pages/reference/cli.mdx index 59373ebed91..d0db73c8ff1 100644 --- a/docs/pages/reference/cli.mdx +++ b/docs/pages/reference/cli.mdx @@ -148,6 +148,7 @@ information about the cluster. | `-d, --debug` | none | none | Verbose logging to stdout | | `-J, --jumphost` | none | A jump host | SSH jumphost | | `--headless` | none | none | Use Headless WebAuthn for authentication | +| `--mlock` | `auto` | `auto`, `off`, `best_effort`, `strict` | Lock process memory to protect client secrets stored in memory from being swapped to disk. | ### tsh help diff --git a/lib/utils/mlock/mlock_linux.go b/lib/utils/mlock/mlock_linux.go index ecdd6bd40c1..7eb9ac2acf5 100644 --- a/lib/utils/mlock/mlock_linux.go +++ b/lib/utils/mlock/mlock_linux.go @@ -21,7 +21,6 @@ import ( ) // LockMemory locks the process memory to prevent secrets from being exposed in a swap. -// This is a noop on unsupported systems (non-linux). func LockMemory() error { return unix.Mlockall(unix.MCL_CURRENT | unix.MCL_FUTURE) } diff --git a/lib/utils/mlock/mlock_unsupported.go b/lib/utils/mlock/mlock_unsupported.go index 092bd83152a..ade11539b56 100644 --- a/lib/utils/mlock/mlock_unsupported.go +++ b/lib/utils/mlock/mlock_unsupported.go @@ -18,8 +18,13 @@ limitations under the License. package mlock +import ( + "github.com/gravitational/trace" +) + +var unsupportedOSError = trace.Errorf("memory locking is not supported on non-linux operating systems") + // LockMemory locks the process memory to prevent secrets from being exposed in a swap. -// This is a noop on unsupported systems (non-linux). func LockMemory() error { - return nil + return unsupportedOSError } diff --git a/tool/tsh/tsh.go b/tool/tsh/tsh.go index a49d41f0ac5..30716bc5522 100644 --- a/tool/tsh/tsh.go +++ b/tool/tsh/tsh.go @@ -441,6 +441,10 @@ type CLIConf struct { // Headless uses headless login for the client session. Headless bool + // MlockMode determines whether the process memory will be locked, and whether errors will be enforced. + // Allowed values include false, strict, and best_effort. + MlockMode string + // HeadlessAuthenticationID is the ID of a headless authentication. HeadlessAuthenticationID string } @@ -527,6 +531,7 @@ const ( useLocalSSHAgentEnvVar = "TELEPORT_USE_LOCAL_SSH_AGENT" globalTshConfigEnvVar = "TELEPORT_GLOBAL_TSH_CONFIG" mfaModeEnvVar = "TELEPORT_MFA_MODE" + mlockModeEnvVar = "TELEPORT_MLOCK_MODE" debugEnvVar = teleport.VerboseLogsEnvVar // "TELEPORT_DEBUG" identityFileEnvVar = "TELEPORT_IDENTITY_FILE" gcloudSecretEnvVar = "TELEPORT_GCLOUD_SECRET" @@ -638,6 +643,10 @@ func Run(ctx context.Context, args []string, opts ...cliOption) error { Envar(mfaModeEnvVar). EnumVar(&cf.MFAMode, modes...) app.Flag("headless", "Use headless login. Shorthand for --auth=headless.").Envar(headlessEnvVar).BoolVar(&cf.Headless) + app.Flag("mlock", fmt.Sprintf("Determines whether process memory will be locked and whether failure to do so will be accepted (%v).", strings.Join(mlockModes, ", "))). + Default(mlockModeAuto). + Envar(mlockModeEnvVar). + StringVar(&cf.MlockMode) app.HelpFlag.Short('h') ver := app.Command("version", "Print the tsh client and Proxy server versions for the current context.") @@ -3342,11 +3351,8 @@ func makeClientForProxy(cf *CLIConf, proxy string, useProfileLogin bool) (*clien cf.AuthConnector = constants.HeadlessConnector } - if cf.AuthConnector == constants.HeadlessConnector { - // Lock the process memory to prevent rsa keys and certificates from being exposed in a swap. - if err := mlock.LockMemory(); err != nil { - return nil, trace.Wrap(err, "failed to lock system memory for headless login") - } + if err := tryLockMemory(cf); err != nil { + return nil, trace.Wrap(err) } c.ClientStore, err = initClientStore(cf, proxy) @@ -4749,3 +4755,51 @@ func onHeadlessApprove(cf *CLIConf) error { }) return trace.Wrap(err) } + +var mlockModes = []string{mlockModeNo, mlockModeAuto, mlockModeBestEffort, mlockModeStrict} + +const ( + // mlockModeNo disables locking process memory. + mlockModeNo = "off" + // mlockModeAuto automatically chooses whether memory locking will be attempted and/or enforced. + mlockModeAuto = "auto" + // mlockBestEfforts enables locking process memory, but errors will be ignored and logged. + mlockModeBestEffort = "best_effort" + // mlockModeStrict enables locking process memory and enforces it succeeds without errors. + mlockModeStrict = "strict" + + // mlockFailureMessage is a user readable message for mlock errors and debug logs. + mlockFailureMessage = "Failed to lock process memory for headless login. " + + "Memory locking is used to prevent secrets in memory from being swapped to disk. " + + "Please ensure that memory locking is available on your system and your user has " + + "locking privileges. This means using a Linux operating system and increasing your " + + `user's memory locking limit to unlimited if needed. Alternatively, set --mlock=off ` + + "or TELEPORT_MLOCK_MODE=off to disable it. This is not recommended in production " + + "environments on shared systems where a memory swap attack is possible.\n" + + "https://goteleport.com/docs/access-controls/guides/headless/#troubleshooting" +) + +// Lock the process memory to prevent rsa keys and certificates in memory from being exposed in a swap. +func tryLockMemory(cf *CLIConf) error { + if cf.MlockMode == mlockModeAuto { + if cf.AuthConnector == constants.HeadlessConnector { + // default to best effort for headless login. + cf.MlockMode = mlockModeBestEffort + } + } + + switch cf.MlockMode { + case mlockModeNo, mlockModeAuto, "": + // noop + return nil + case mlockModeStrict: + err := mlock.LockMemory() + return trace.Wrap(err, mlockFailureMessage) + case mlockModeBestEffort: + err := mlock.LockMemory() + log.WithError(err).Warning(mlockFailureMessage) + return nil + default: + return trace.BadParameter("unexpected value for --mlock, expected one of (%v)", strings.Join(mlockModes, ", ")) + } +}