From 40caec6048153752e520f2716ffbbd7cf63709f3 Mon Sep 17 00:00:00 2001 From: Ev Kontsevoy Date: Thu, 12 Jan 2017 16:04:00 -0800 Subject: [PATCH] Converted boltbk to the new format BoltDB backend is now compatible with how all backends should initialize. Also all BoltDB-specific code/constants have been consolidated inside of `backend.boltbk` package. --- constants.go | 7 +-- lib/backend/boltbk/boltbk.go | 81 +++++++++++++++------------- lib/backend/boltbk/boltcfg.go | 38 ------------- lib/backend/boltbk/doc.go | 5 ++ lib/backend/{fs => dir}/doc.go | 6 +-- lib/backend/{fs => dir}/impl.go | 2 +- lib/backend/{fs => dir}/impl_test.go | 2 +- lib/config/configuration.go | 55 ++++--------------- lib/defaults/defaults.go | 6 --- lib/service/cfg.go | 42 --------------- lib/service/service.go | 30 ++++++----- lib/web/api.go | 1 - 12 files changed, 85 insertions(+), 190 deletions(-) delete mode 100644 lib/backend/boltbk/boltcfg.go create mode 100644 lib/backend/boltbk/doc.go rename lib/backend/{fs => dir}/doc.go (84%) rename lib/backend/{fs => dir}/impl.go (99%) rename lib/backend/{fs => dir}/impl_test.go (99%) diff --git a/constants.go b/constants.go index f50807001fa..10df91fccdb 100644 --- a/constants.go +++ b/constants.go @@ -8,9 +8,6 @@ import ( const ForeverTTL time.Duration = 0 const ( - // BoltBackendType is a BoltDB backend - BoltBackendType = "bolt" - // ETCDBackendType is etcd backend ETCDBackendType = "etcd" @@ -57,4 +54,8 @@ const ( // ConnectorOIDC means connector type OIDC ConnectorOIDC = "oidc" + + // DataDirParameterName is the name of the data dir configuration parameter passed + // to all backends during initialization + DataDirParameterName = "data_dir" ) diff --git a/lib/backend/boltbk/boltbk.go b/lib/backend/boltbk/boltbk.go index 3e81316de2b..a903f70f3f2 100644 --- a/lib/backend/boltbk/boltbk.go +++ b/lib/backend/boltbk/boltbk.go @@ -14,23 +14,37 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package boltbk implements BoltDB backed backend for standalone instances -// and test mode, you should use Etcd in production package boltbk import ( "encoding/json" - "os" + "fmt" "path/filepath" "sort" "sync" "time" "github.com/boltdb/bolt" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/backend" + "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/trace" "github.com/mailgun/timetools" ) +const ( + // keysBoltFile is the BoltDB database file, usually stored in data_dir + keysBoltFile = "keys.db" + + // openTimeout determines for how long BoltDB will wait before giving up + // opening the locked DB file + openTimeout = 5 * time.Second + + // openFileMode flag is passed to db.Open() + openFileMode = 0600 +) + // BoltBackend is a boltdb-based backend used in tests and standalone mode type BoltBackend struct { sync.Mutex @@ -40,51 +54,46 @@ type BoltBackend struct { locks map[string]time.Time } -// Option sets functional options for the backend -type Option func(b *BoltBackend) error - -// Clock sets clock for the backend, used in tests -func Clock(clock timetools.TimeProvider) Option { - return func(b *BoltBackend) error { - b.clock = clock - return nil - } +// GetName() is a part of the backend API and returns the name of this backend +// as shown in 'storage/type' section of Teleport YAML config +func GetName() string { + return "bolt" } -// New returns a new isntance of bolt backend -func New(path string, opts ...Option) (*BoltBackend, error) { - path, err := filepath.Abs(path) - if err != nil { - return nil, trace.Wrap(err, "failed to convert path") +// New initializes and returns a fully created BoltDB backend. It's +// a properly implemented Backend.NewFunc, part of a backend API +func New(params backend.Params) (backend.Backend, error) { + // look at 'path' parameter, if it's missing use 'data_dir' (default): + path := params.GetString("path") + if len(path) == 0 { + path = params.GetString(teleport.DataDirParameterName) } - dir := filepath.Dir(path) - s, err := os.Stat(dir) + // still nothing? return an error: + if path == "" { + return nil, trace.BadParameter("Bolt backend: 'path' is not set") + } + if !utils.IsDir(path) { + return nil, trace.BadParameter("%v is not a valid directory", path) + } + path, err := filepath.Abs(filepath.Join(path, keysBoltFile)) if err != nil { return nil, trace.Wrap(err) } - if !s.IsDir() { - return nil, trace.BadParameter("path '%v' should be a valid directory", dir) - } - b := &BoltBackend{ - locks: make(map[string]time.Time), - } - for _, option := range opts { - if err := option(b); err != nil { - return nil, trace.Wrap(err) - } - } - if b.clock == nil { - b.clock = &timetools.RealTime{} - } - db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 5 * time.Second}) + + fmt.Printf("\n\n\nFULL PATH: %s\n\n", path) + + db, err := bolt.Open(path, openFileMode, &bolt.Options{Timeout: openTimeout}) if err != nil { if err == bolt.ErrTimeout { return nil, trace.Errorf("Local storage is locked. Another instance is running? (%v)", path) } return nil, trace.Wrap(err) } - b.db = db - return b, nil + return &BoltBackend{ + locks: make(map[string]time.Time), + clock: &timetools.RealTime{}, + db: db, + }, nil } // Close closes the backend resources diff --git a/lib/backend/boltbk/boltcfg.go b/lib/backend/boltbk/boltcfg.go deleted file mode 100644 index cf334f95eea..00000000000 --- a/lib/backend/boltbk/boltcfg.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2015 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package boltbk - -import ( - "encoding/json" - - "github.com/gravitational/teleport/lib/backend" - "github.com/gravitational/trace" -) - -// cfg represents JSON config for bolt backlend -type cfg struct { - Path string `json:"path"` -} - -// FromJSON creates a new bolt backend from a JSON string -func FromJSON(paramsJSON string) (backend.Backend, error) { - c := cfg{} - err := json.Unmarshal([]byte(paramsJSON), &c) - if err != nil { - return nil, trace.Wrap(err) - } - return New(c.Path) -} diff --git a/lib/backend/boltbk/doc.go b/lib/backend/boltbk/doc.go new file mode 100644 index 00000000000..8dfe14057e8 --- /dev/null +++ b/lib/backend/boltbk/doc.go @@ -0,0 +1,5 @@ +// Package boltbk implements BoltDB backed backend for standalone instances +// This is a legacy backend which only exists for backward compatibility purposes +// +// Production Teleport clusters should be using either etcd or DynamoDB backends. +package boltbk diff --git a/lib/backend/fs/doc.go b/lib/backend/dir/doc.go similarity index 84% rename from lib/backend/fs/doc.go rename to lib/backend/dir/doc.go index 13e97270929..a18e909a2e4 100644 --- a/lib/backend/fs/doc.go +++ b/lib/backend/dir/doc.go @@ -19,9 +19,9 @@ compliant and support 'date modified' attribute on files. */ -// Package 'fs' implements the "filesystem backend". It uses a regular -// filesystem (directories with files) to store Teleport auth server state. +// Package 'dir' implements the "directory backend". It uses a regular +// filesystem directories/files to store Teleport auth server state. // // Limitations: // - key names cannot start with '.' (dot) -package fs +package dir diff --git a/lib/backend/fs/impl.go b/lib/backend/dir/impl.go similarity index 99% rename from lib/backend/fs/impl.go rename to lib/backend/dir/impl.go index 942729c0c1f..ed23a23ca48 100644 --- a/lib/backend/fs/impl.go +++ b/lib/backend/dir/impl.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fs +package dir import ( "io" diff --git a/lib/backend/fs/impl_test.go b/lib/backend/dir/impl_test.go similarity index 99% rename from lib/backend/fs/impl_test.go rename to lib/backend/dir/impl_test.go index dc2e9e1ff56..a2ccfb2814f 100644 --- a/lib/backend/fs/impl_test.go +++ b/lib/backend/dir/impl_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fs +package dir import ( "sync/atomic" diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 4e4709e0b35..1dcbbcff148 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -32,10 +32,8 @@ import ( "golang.org/x/crypto/ssh" - "github.com/gravitational/teleport" - "github.com/gravitational/teleport/lib/backend/dynamo" - "github.com/gravitational/teleport/lib/backend/etcdbk" - "github.com/gravitational/teleport/lib/backend/fs" + "github.com/gravitational/teleport/lib/backend" + "github.com/gravitational/teleport/lib/backend/boltbk" "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/limiter" @@ -155,50 +153,17 @@ func ApplyFileConfig(fc *FileConfig, cfg *service.Config) error { if fc.Global.DataDir != "" { cfg.DataDir = fc.Global.DataDir } + // use bolt by default: if fc.Storage.Type == "" { - fc.Storage.Type = teleport.BoltBackendType + fc.Storage.Type = boltbk.GetName() + } + if fc.Storage.Params == nil { + fc.Storage.Params = make(backend.Params) } - // configure storage (TODO ev: this needs to go. storage plugins should be - // parsing 'storage' sections themselves) - switch fc.Storage.Type { - // dir backend - case fs.GetName(): - cfg.Auth.KeysBackend.Type = fs.GetName() - cfg.Auth.KeysBackend.BackendConf = &fc.Storage - - // bolt backend (default): - case teleport.BoltBackendType: - cfg.ConfigureBolt() - - // etcd backend (default): - case teleport.ETCDBackendType: - if err = cfg.ConfigureETCD(etcdbk.Config{ - Nodes: fc.Storage.Peers, - Key: fc.Storage.Prefix, - TLSKeyFile: fc.Storage.TLSKeyFile, - TLSCertFile: fc.Storage.TLSCertFile, - TLSCAFile: fc.Storage.TLSCAFile, - }); err != nil { - return trace.Wrap(err) - } - - // optionally-built-in DynamoDB back-end: - case dynamo.BackendType: - cfg.ConfigureBolt() - // dynamo only stores keys, not events/sessions, everything else - // is configured to use bolt: - a := &cfg.Auth - a.KeysBackend.Type = dynamo.BackendType - a.KeysBackend.BackendConf = &fc.Storage - if err != nil { - return trace.Wrap(err) - } - case "": - break // not set - default: - return trace.BadParameter("unsupported storage type: '%v'", fc.Storage.Type) - } + // forward storage config to 'auth backend' config (same thing) + cfg.Auth.KeysBackend.BackendConf = &fc.Storage + cfg.Auth.KeysBackend.Type = fc.Storage.Type // apply logger settings switch fc.Logger.Output { diff --git a/lib/defaults/defaults.go b/lib/defaults/defaults.go index 1ffea6d57e3..c27edb2ef07 100644 --- a/lib/defaults/defaults.go +++ b/lib/defaults/defaults.go @@ -55,12 +55,6 @@ const ( // Name of events bolt database file stored in DataDir EventsBoltFile = "events.db" - // Name of keys bolt database file stored in DataDir - KeysBoltFile = "keys.db" - - // Name of records bolt database file stored in DataDir - RecordsBoltFile = "records.db" - // By default SSH server (and SSH proxy) will bind to this IP BindIP = "0.0.0.0" diff --git a/lib/service/cfg.go b/lib/service/cfg.go index 5d3c2b90359..8f3e4664d1e 100644 --- a/lib/service/cfg.go +++ b/lib/service/cfg.go @@ -17,25 +17,20 @@ limitations under the License. package service import ( - "encoding/json" "fmt" "io" "net" "os" - "path/filepath" "strings" - "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/backend" - "github.com/gravitational/teleport/lib/backend/etcdbk" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/limiter" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/utils" log "github.com/Sirupsen/logrus" - "github.com/gravitational/trace" "gopkg.in/yaml.v2" ) @@ -127,27 +122,6 @@ func (cfg *Config) ApplyToken(token string) bool { return false } -// ConfigureBolt configures Bolt back-ends with a data dir. -func (cfg *Config) ConfigureBolt() { - a := &cfg.Auth - if a.KeysBackend.Type == teleport.BoltBackendType { - a.KeysBackend.Params = boltParams(cfg.DataDir, defaults.KeysBoltFile) - } -} - -// ConfigureETCD configures ETCD backend (still uses BoltDB for some cases) -func (cfg *Config) ConfigureETCD(etcdCfg etcdbk.Config) error { - a := &cfg.Auth - - params, err := etcdParams(etcdCfg) - if err != nil { - return trace.Wrap(err) - } - a.KeysBackend.Type = teleport.ETCDBackendType - a.KeysBackend.Params = params - return nil -} - // RoleConfig is a config for particular Teleport role func (cfg *Config) RoleConfig() RoleConfig { return RoleConfig{ @@ -274,7 +248,6 @@ func ApplyDefaults(cfg *Config) { cfg.Auth.Enabled = true cfg.Auth.SSHAddr = *defaults.AuthListenAddr() cfg.Auth.KeysBackend.Type = defaults.BackendType - cfg.Auth.KeysBackend.Params = boltParams(defaults.DataDir, defaults.KeysBoltFile) cfg.Auth.U2F.Enabled = false cfg.Auth.U2F.AppID = fmt.Sprintf("https://%s:%d", strings.ToLower(hostname), defaults.HTTPListenPort) cfg.Auth.U2F.Facets = []string{cfg.Auth.U2F.AppID} @@ -298,18 +271,3 @@ func ApplyDefaults(cfg *Config) { cfg.DataDir = defaults.DataDir cfg.Console = os.Stdout } - -// Generates a string accepted by the BoltDB driver, like this: -// `{"path": "/var/lib/teleport/records.db"}` -func boltParams(storagePath, dbFile string) string { - return fmt.Sprintf(`{"path": "%s"}`, filepath.Join(storagePath, dbFile)) -} - -// etcdParams generates a string accepted by the ETCD driver, like this: -func etcdParams(cfg etcdbk.Config) (string, error) { - out, err := json.Marshal(cfg) - if err != nil { // don't know what to do seriously - return "", trace.Wrap(err) - } - return string(out), nil -} diff --git a/lib/service/service.go b/lib/service/service.go index adb7f120d38..0acd8d6e31c 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -35,9 +35,9 @@ import ( "github.com/gravitational/teleport/lib/auth/native" "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/backend/boltbk" + "github.com/gravitational/teleport/lib/backend/dir" "github.com/gravitational/teleport/lib/backend/dynamo" "github.com/gravitational/teleport/lib/backend/etcdbk" - "github.com/gravitational/teleport/lib/backend/fs" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/limiter" @@ -795,23 +795,25 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { } // initAuthStorage initializes the storage backend for the auth. service -func (process *TeleportProcess) initAuthStorage() (backend.Backend, error) { - cfg := &process.Config.Auth - var bk backend.Backend - var err error +func (process *TeleportProcess) initAuthStorage() (bk backend.Backend, err error) { + bkConf := &process.Config.Auth.KeysBackend + // pass 'data_dir' setting to a backend: + bkConf.BackendConf.Params["data_dir"] = process.Config.DataDir - switch cfg.KeysBackend.Type { + switch bkConf.Type { + // legacy bolt backend: + case boltbk.GetName(): + bk, err = boltbk.New(bkConf.BackendConf.Params) // filesystem backend: - case fs.GetName(): - bk, err = fs.New(cfg.KeysBackend.BackendConf.Params) - case teleport.ETCDBackendType: - bk, err = etcdbk.FromJSON(cfg.KeysBackend.Params) - case teleport.BoltBackendType: - bk, err = boltbk.FromJSON(cfg.KeysBackend.Params) + case dir.GetName(): + bk, err = dir.New(bkConf.BackendConf.Params) + // DynamoDB bakcend: case dynamo.BackendType: - bk, err = dynamo.New(cfg.KeysBackend.BackendConf) + bk, err = dynamo.New(bkConf.BackendConf) + case teleport.ETCDBackendType: + bk, err = etcdbk.FromJSON(bkConf.Params) default: - err = trace.Errorf("unsupported backend type: %v", cfg.KeysBackend.Type) + err = trace.Errorf("unsupported backend type: %v", bkConf.Type) } if err != nil { return nil, trace.Wrap(err) diff --git a/lib/web/api.go b/lib/web/api.go index 6448097a8b9..9885b8151c2 100644 --- a/lib/web/api.go +++ b/lib/web/api.go @@ -1462,7 +1462,6 @@ func (h *Handler) AuthenticateRequest(w http.ResponseWriter, r *http.Request, ch }) cookie, err := r.Cookie("session") if err != nil || (cookie != nil && cookie.Value == "") { - logger.Infof(missingCookieMsg) if err != nil { logger.Warn(err) }