opensource/sealos
3
0
Fork 0
mirror of https://github.com/labring/sealos.git synced 2026-08-29 01:39:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support registries HA (#2096)

Browse Source 
* feat: support multiple registries

Signed-off-by: fengxsong <fengxsong@outlook.com>

* ci: remove unused functions

Signed-off-by: fengxsong <fengxsong@outlook.com>

* docs: add proposal doc

Signed-off-by: fengxsong <fengxsong@outlook.com>

* fix: adjust helper functions

Signed-off-by: fengxsong <fengxsong@outlook.com>

* refactor: bootstrap module

Signed-off-by: fengxsong <fengxsong@outlook.com>

* refactor dep to applier

Signed-off-by: fengxsong <fengxsong@outlook.com>

* refactor: split registry mirroring from mountrootfs

Signed-off-by: fengxsong <fengxsong@outlook.com>

* fix: format code

Signed-off-by: fengxsong <fengxsong@outlook.com>

* docs: add limitations section in proposal doc

Signed-off-by: fengxsong <fengxsong@outlook.com>

Signed-off-by: fengxsong <fengxsong@outlook.com>
This commit is contained in:
fengxsong
2022-11-23 12:33:52 +08:00
committed by GitHub
parent b732f9be03
commit eff2fbfaa1
21 changed files with 775 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apply/processor/create.go
+30 -12
View File
@@ -19,25 +19,23 @@ import (
"errors"
"fmt"
"github.com/labring/sealos/pkg/utils/strings"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/rand"
"github.com/labring/sealos/pkg/utils/yaml"
"golang.org/x/sync/errgroup"
"k8s.io/apimachinery/pkg/util/sets"
"github.com/labring/sealos/pkg/bootstrap"
"github.com/labring/sealos/pkg/checker"
"github.com/labring/sealos/pkg/clusterfile"
"github.com/labring/sealos/pkg/config"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/filesystem"
"github.com/labring/sealos/pkg/guest"
"github.com/labring/sealos/pkg/image"
"github.com/labring/sealos/pkg/image/types"
"github.com/labring/sealos/pkg/runtime"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/rand"
"github.com/labring/sealos/pkg/utils/yaml"
)
type CreateProcessor struct {
@@ -62,6 +60,7 @@ func (c *CreateProcessor) Execute(cluster *v2.Cluster) error {
return nil
}
func (c *CreateProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error) {
var todoList []func(cluster *v2.Cluster) error
todoList = append(todoList,
@@ -70,6 +69,8 @@ func (c *CreateProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, erro
c.PreProcess,
c.RunConfig,
c.MountRootfs,
c.MirrorRegistry,
c.Bootstrap,
// c.GetPhasePluginFunc(plugin.PhasePreInit),
c.Init,
c.Join,
@@ -79,15 +80,16 @@ func (c *CreateProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, erro
)
return todoList, nil
}
func (c *CreateProcessor) Check(cluster *v2.Cluster) error {
logger.Info("Executing pipeline Check in CreateProcessor.")
err := checker.RunCheckList([]checker.Interface{checker.NewHostChecker()}, cluster, checker.PhasePre)
if err != nil {
return err
}
return nil
}
func (c *CreateProcessor) CheckImageType(cluster *v2.Cluster) error {
ociList, err := c.ImageManager.Inspect(cluster.Spec.Image...)
if err != nil {
@@ -106,6 +108,7 @@ func (c *CreateProcessor) CheckImageType(cluster *v2.Cluster) error {
}
return nil
}
func (c *CreateProcessor) PreProcess(cluster *v2.Cluster) error {
logger.Info("Executing pipeline PreProcess in CreateProcessor.")
err := c.RegistryManager.Pull(types.DefaultPlatform(), types.PullPolicyMissing, cluster.Spec.Image...)
@@ -157,14 +160,29 @@ func (c *CreateProcessor) RunConfig(cluster *v2.Cluster) error {
func (c *CreateProcessor) MountRootfs(cluster *v2.Cluster) error {
logger.Info("Executing pipeline MountRootfs in CreateProcessor.")
hosts := append(cluster.GetMasterIPAndPortList(), cluster.GetNodeIPAndPortList()...)
if strings.NotInIPList(cluster.GetRegistryIPAndPort(), hosts) {
hosts = append(hosts, cluster.GetRegistryIPAndPort())
}
fs, err := filesystem.NewRootfsMounter(cluster.Status.Mounts)
if err != nil {
return err
}
return fs.MountRootfs(cluster, hosts, true, cluster.HasAppImage())
return fs.MountRootfs(cluster, hosts)
}
func (c *CreateProcessor) MirrorRegistry(cluster *v2.Cluster) error {
logger.Info("Executing pipeline MirrorRegistry in CreateProcessor.")
return MirrorRegistry(cluster, cluster.Status.Mounts)
}
func (c *CreateProcessor) Bootstrap(cluster *v2.Cluster) error {
logger.Info("Executing pipeline Bootstrap in CreateProcessor")
hosts := append(cluster.GetMasterIPAndPortList(), cluster.GetNodeIPAndPortList()...)
bs := bootstrap.New(cluster)
if err := bs.Preflight(hosts...); err != nil {
return err
}
if err := bs.Init(hosts...); err != nil {
return err
}
return bs.ApplyAddons(hosts...)
}
func (c *CreateProcessor) Init(cluster *v2.Cluster) error {
pkg/apply/processor/install.go
+6 -6
View File
@@ -20,8 +20,6 @@ import (
"fmt"
"strings"
stringsutil "github.com/labring/sealos/pkg/utils/strings"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/utils/confirm"
"github.com/labring/sealos/pkg/utils/logger"
@@ -93,6 +91,7 @@ func (c *InstallProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, err
c.PreProcess,
c.RunConfig,
c.MountRootfs,
c.MirrorRegistry,
// i.GetPhasePluginFunc(plugin.PhasePreGuest),
c.RunGuest,
c.PostProcess,
@@ -197,15 +196,16 @@ func (c *InstallProcessor) MountRootfs(cluster *v2.Cluster) error {
return nil
}
hosts := append(cluster.GetMasterIPAndPortList(), cluster.GetNodeIPAndPortList()...)
if stringsutil.NotInIPList(cluster.GetRegistryIPAndPort(), hosts) {
hosts = append(hosts, cluster.GetRegistryIPAndPort())
}
fs, err := filesystem.NewRootfsMounter(c.NewMounts)
if err != nil {
return err
}
return fs.MountRootfs(cluster, hosts)
}
return fs.MountRootfs(cluster, hosts, false, true)
func (c *InstallProcessor) MirrorRegistry(cluster *v2.Cluster) error {
logger.Info("Executing pipeline MirrorRegistry in InstallProcessor.")
return MirrorRegistry(cluster, c.NewMounts)
}
func (c *InstallProcessor) RunGuest(cluster *v2.Cluster) error {
pkg/apply/processor/interface.go
+10
View File
@@ -15,12 +15,15 @@
package processor
import (
"context"
"path"
"github.com/pkg/errors"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/filesystem/registry"
"github.com/labring/sealos/pkg/image/types"
"github.com/labring/sealos/pkg/ssh"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/confirm"
"github.com/labring/sealos/pkg/utils/file"
@@ -128,3 +131,10 @@ func ConfirmDeleteNodes() error {
}
return nil
}
func MirrorRegistry(cluster *v2.Cluster, mounts []v2.MountImage) error {
registries := cluster.GetRegistryIPAndPortList()
sshClient := ssh.NewSSHClient(&cluster.Spec.SSH, true)
mirror := registry.New(constants.NewData(cluster.GetName()).RootFSPath(), sshClient, mounts)
return mirror.MirrorTo(context.Background(), registries...)
}
pkg/apply/processor/scale.go
+26 -7
View File
@@ -18,21 +18,21 @@ import (
"context"
"fmt"
"github.com/labring/sealos/pkg/constants"
fileutil "github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/yaml"
"golang.org/x/sync/errgroup"
"github.com/labring/sealos/pkg/bootstrap"
"github.com/labring/sealos/pkg/checker"
"github.com/labring/sealos/pkg/clusterfile"
"github.com/labring/sealos/pkg/config"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/filesystem"
"github.com/labring/sealos/pkg/image"
"github.com/labring/sealos/pkg/image/types"
"github.com/labring/sealos/pkg/runtime"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
fileutil "github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/yaml"
)
type ScaleProcessor struct {
@@ -72,6 +72,7 @@ func (c *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error
c.PreProcessImage,
c.RunConfig,
c.MountRootfs,
c.Bootstrap,
//s.GetPhasePluginFunc(plugin.PhasePreJoin),
c.Join,
//s.GetPhasePluginFunc(plugin.PhasePostJoin),
@@ -183,7 +184,7 @@ func (c *ScaleProcessor) PreProcess(cluster *v2.Cluster) error {
}
runTime, err := runtime.NewDefaultRuntime(cluster, c.ClusterFile.GetKubeadmConfig())
if err != nil {
return fmt.Errorf("failed to init runtime, %v", err)
return fmt.Errorf("failed to init runtime: %v", err)
}
c.Runtime = runTime
@@ -233,7 +234,25 @@ func (c *ScaleProcessor) MountRootfs(cluster *v2.Cluster) error {
return err
}
return fs.MountRootfs(cluster, hosts, true, false)
return fs.MountRootfs(cluster, hosts)
}
func (c *ScaleProcessor) MirrorRegistry(cluster *v2.Cluster) error {
logger.Info("Executing pipeline MirrorRegistry in ScaleProcessor.")
return MirrorRegistry(cluster, cluster.Status.Mounts)
}
func (c *ScaleProcessor) Bootstrap(cluster *v2.Cluster) error {
logger.Info("Executing pipeline Bootstrap in ScaleProcessor")
hosts := append(c.MastersToJoin, c.NodesToJoin...)
bs := bootstrap.New(cluster)
if err := bs.Preflight(hosts...); err != nil {
return err
}
if err := bs.Init(hosts...); err != nil {
return err
}
return bs.ApplyAddons(hosts...)
}
func NewScaleProcessor(clusterFile clusterfile.Interface, images v2.ImageList, masterToJoin, masterToDelete, nodeToJoin, nodeToDelete []string) (Interface, error) {
pkg/bootstrap/applier.go
+109
View File
@@ -0,0 +1,109 @@
// Copyright © 2022 sealos.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"fmt"
"path"
"sync"
"k8s.io/apimachinery/pkg/util/sets"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/passwd"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/logger"
)
type Applier interface {
Name() string
Filter(Context, string) bool
Apply(Context, string) error
Undo(Context, string) error
}
type registryApplier struct {
}
func (*registryApplier) Name() string { return "registry addon applier" }
func (*registryApplier) Filter(ctx Context, host string) bool {
registries := sets.NewString(ctx.GetCluster().GetRegistryIPAndPortList()...)
return registries.Has(host)
}
func (a *registryApplier) Apply(ctx Context, host string) error {
rc := GetRegistryInfo(ctx.GetExecer(), ctx.GetData().RootFSPath(), ctx.GetCluster().GetRegistryIPAndPort())
lnCmd := fmt.Sprintf(constants.DefaultLnFmt, ctx.GetData().RootFSRegistryPath(), rc.Data)
logger.Debug("make soft link: %s", lnCmd)
if err := ctx.GetExecer().CmdAsync(host, lnCmd); err != nil {
return fmt.Errorf("failed to make link: %v", err)
}
htpasswdPath, err := a.configLocalHtpasswd(ctx.GetData().EtcPath(), rc)
if err != nil {
return err
}
if len(htpasswdPath) > 0 {
if err = ctx.GetExecer().Copy(host, htpasswdPath, path.Join(ctx.GetData().RootFSEtcPath(), "registry_htpasswd")); err != nil {
return err
}
}
return ctx.GetExecer().CmdAsync(host, ctx.GetShellWrapper()(host, ctx.GetBash().InitRegistryBash()))
}
func (a *registryApplier) configLocalHtpasswd(cfgBasedir string, rc *v2.RegistryConfig) (string, error) {
if rc.Username == "" || rc.Password == "" {
logger.Warn("registry username or password is empty")
return "", nil
}
mk := newHtpasswdMaker(cfgBasedir)
return mk.Make(rc.Username, rc.Password)
}
type htpasswdMaker struct {
path string
err error
once sync.Once
}
func (m *htpasswdMaker) Make(u, p string) (string, error) {
m.once.Do(func() {
if u == "" || p == "" {
return
}
pwd := passwd.Htpasswd(u, p)
if err := file.WriteFile(m.path, []byte(pwd)); err != nil {
m.err = fmt.Errorf("failed to make htpasswd: %v", err)
}
})
return m.path, m.err
}
var htpasswdMakers = map[string]*htpasswdMaker{}
func newHtpasswdMaker(root string) *htpasswdMaker {
fp := path.Join(root, "registry_htpasswd")
if v, ok := htpasswdMakers[fp]; ok {
return v
}
m := &htpasswdMaker{path: fp}
htpasswdMakers[fp] = m
return m
}
func (*registryApplier) Undo(ctx Context, host string) error {
return ctx.GetExecer().CmdAsync(host, ctx.GetShellWrapper()(host, ctx.GetBash().CleanRegistryBash()))
}
pkg/bootstrap/bootstrap.go
+181
View File
@@ -0,0 +1,181 @@
// Copyright © 2022 sealos.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"context"
"fmt"
"golang.org/x/sync/errgroup"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/logger"
)
type Phase string
const (
Preflight Phase = "preflight"
Init Phase = "init"
Addon Phase = "addon"
)
type Interface interface {
Preflight(hosts ...string) error
Init(hosts ...string) error
RegisterApplier(Phase, ...Applier) error
ApplyAddons(hosts ...string) error
Reset(hosts ...string) error
}
type realBootstrap struct {
ctx Context
checks []Applier
initializers []Applier
addons []Applier
}
type shellWrapper func(string, string) string
func New(cluster *v2.Cluster) Interface {
ctx := NewContextFrom(cluster)
bs := &realBootstrap{
ctx: ctx,
checks: make([]Applier, 0),
initializers: make([]Applier, 0),
addons: make([]Applier, 0),
}
// register builtin appliers
_ = bs.RegisterApplier(Preflight, &defaultChecker{})
_ = bs.RegisterApplier(Init, &defaultInitializer{})
_ = bs.RegisterApplier(Addon, &registryApplier{})
return bs
}
func (bs *realBootstrap) Preflight(hosts ...string) error {
return bs.apply(bs.checks, hosts...)
}
func (bs *realBootstrap) Init(hosts ...string) error {
return bs.apply(bs.initializers, hosts...)
}
func (bs *realBootstrap) ApplyAddons(hosts ...string) error {
return bs.apply(bs.addons, hosts...)
}
func (bs *realBootstrap) apply(appliers []Applier, hosts ...string) error {
return runParallel(hosts, func(host string) error {
for i := range appliers {
applier := appliers[i]
if !applier.Filter(bs.ctx, host) {
return nil
}
logger.Debug("apply %s on host %s", applier.Name(), host)
if err := applier.Apply(bs.ctx, host); err != nil {
return err
}
}
return nil
})
}
func (bs *realBootstrap) RegisterApplier(phase Phase, appliers ...Applier) error {
switch phase {
case Preflight:
bs.checks = append(bs.checks, appliers...)
case Init:
bs.initializers = append(bs.initializers, appliers...)
case Addon:
bs.addons = append(bs.addons, appliers...)
default:
return fmt.Errorf("unknown phase %s", phase)
}
return nil
}
func (bs *realBootstrap) Reset(hosts ...string) error {
appliers := make([]Applier, 0)
// only undo addons OR?
appliers = append(appliers, bs.addons...)
return runParallel(hosts, func(host string) error {
for i := range appliers {
applier := appliers[i]
if !applier.Filter(bs.ctx, host) {
return nil
}
logger.Debug("undo %s on host %s", applier.Name(), host)
if err := applier.Undo(bs.ctx, host); err != nil {
return err
}
}
return nil
})
}
func runParallel(hosts []string, fn func(string) error) error {
eg, _ := errgroup.WithContext(context.Background())
for i := range hosts {
host := hosts[i]
eg.Go(func() error {
return fn(host)
})
}
return eg.Wait()
}
type defaultChecker struct {
is *ImageShim
}
func (c *defaultChecker) Name() string {
return "default checker"
}
func (c *defaultChecker) Filter(_ Context, _ string) bool {
return true
}
func (c *defaultChecker) Apply(ctx Context, host string) error {
if c.is == nil {
c.is = NewImageShimHelper(ctx.GetExecer(), ctx.GetCluster().GetRegistryIP())
}
shimCmd := c.is.ApplyCMD(ctx.GetData().RootFSPath())
cmds := []string{ctx.GetShellWrapper()(host, ctx.GetBash().CheckBash()), shimCmd}
return ctx.GetExecer().CmdAsync(host, cmds...)
}
func (c *defaultChecker) Undo(_ Context, _ string) error {
return nil
}
type defaultInitializer struct{}
func (initializer *defaultInitializer) Name() string {
return "default initializer"
}
func (initializer *defaultInitializer) Filter(_ Context, _ string) bool {
return true
}
func (initializer *defaultInitializer) Apply(ctx Context, host string) error {
cmds := []string{ctx.GetShellWrapper()(host, ctx.GetBash().InitBash())}
return ctx.GetExecer().CmdAsync(host, cmds...)
}
func (initializer *defaultInitializer) Undo(_ Context, _ string) error {
return nil
}
pkg/bootstrap/context.go
+73
View File
@@ -0,0 +1,73 @@
// Copyright © 2022 sealos.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/env"
"github.com/labring/sealos/pkg/ssh"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
)
type Context interface {
GetBash() constants.Bash
GetCluster() *v2.Cluster
GetData() constants.Data
GetExecer() ssh.Interface
GetShellWrapper() shellWrapper
}
type realContext struct {
bash constants.Bash
cluster *v2.Cluster
data constants.Data
execer ssh.Interface
shellWrapper shellWrapper
}
func (ctx realContext) GetBash() constants.Bash {
return ctx.bash
}
func (ctx realContext) GetCluster() *v2.Cluster {
return ctx.cluster
}
func (ctx realContext) GetData() constants.Data {
return ctx.data
}
func (ctx realContext) GetExecer() ssh.Interface {
return ctx.execer
}
func (ctx realContext) GetShellWrapper() shellWrapper {
return ctx.shellWrapper
}
func NewContextFrom(cluster *v2.Cluster) Context {
execer := ssh.NewSSHClient(&cluster.Spec.SSH, true)
envProcessor := env.NewEnvProcessor(cluster, cluster.Status.Mounts)
return &realContext{
cluster: cluster,
execer: execer,
bash: constants.NewBash(cluster.GetName(), cluster.GetImageLabels()),
data: constants.NewData(cluster.GetName()),
shellWrapper: func(host, s string) string {
return envProcessor.WrapperShell(host, s)
},
}
}
pkg/bootstrap/helpers.go
+76
View File
@@ -0,0 +1,76 @@
// Copyright © 2021 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"fmt"
"path"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/ssh"
"github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/iputils"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/yaml"
)
func GetRegistryInfo(sshInterface ssh.Interface, rootfs, defaultRegistry string) *v1beta1.RegistryConfig {
const registryCustomConfig = "registry.yml"
var DefaultConfig = &v1beta1.RegistryConfig{
IP: iputils.GetHostIP(defaultRegistry),
Domain: constants.DefaultRegistryDomain,
Port: "5000",
Username: constants.DefaultRegistryUsername,
Password: constants.DefaultRegistryPassword,
Data: constants.DefaultRegistryData,
}
etcPath := path.Join(rootfs, constants.EtcDirName, registryCustomConfig)
out, _ := sshInterface.Cmd(defaultRegistry, fmt.Sprintf("cat %s", etcPath))
logger.Debug("image shim data info: %s", string(out))
registryConfig, err := yaml.UnmarshalData(out)
if err != nil {
logger.Warn("read registry config path error: %+v", err)
logger.Info("use default registry config")
return DefaultConfig
}
domain, _, _ := unstructured.NestedString(registryConfig, "domain")
port, _, _ := unstructured.NestedString(registryConfig, "port")
username, _, _ := unstructured.NestedString(registryConfig, "username")
password, _, _ := unstructured.NestedString(registryConfig, "password")
data, _, _ := unstructured.NestedString(registryConfig, "data")
ip, _, _ := unstructured.NestedString(registryConfig, "ip")
if ip == "" {
ip = defaultRegistry
}
if domain == "" {
domain = DefaultConfig.Domain
}
if port == "" {
port = DefaultConfig.Port
}
rConfig := &v1beta1.RegistryConfig{
IP: ip,
Domain: domain,
Port: port,
Username: username,
Password: password,
Data: data,
}
logger.Debug("show registry info, IP: %s, Domain: %s, Data: %s", rConfig.IP, rConfig.Domain, rConfig.Data)
return rConfig
}
pkg/runtime/image_shim.go → pkg/bootstrap/image_shim.go
+10 -7
View File
@@ -14,33 +14,32 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package runtime
package bootstrap
import (
"fmt"
"path"
"github.com/labring/sealos/pkg/ssh"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/ssh"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/yaml"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
var defaultRootDirectory = "/var/lib/image-cri-shim"
type ImageShim struct {
SSHInterface ssh.Interface
IP string
sshInterface ssh.Interface
host string
}
// GetInfo default dir is /var/lib/image-cri-shim
func (is *ImageShim) GetInfo(rootfs string) string {
const imageCustomConfig = "image-cri-shim.yaml"
etcPath := path.Join(rootfs, constants.EtcDirName, imageCustomConfig)
data, _ := is.SSHInterface.Cmd(is.IP, fmt.Sprintf("cat %s", etcPath))
data, _ := is.sshInterface.Cmd(is.host, fmt.Sprintf("cat %s", etcPath))
logger.Debug("image shim data info: %s", string(data))
shimConfig, err := yaml.UnmarshalData(data)
if err != nil {
@@ -60,3 +59,7 @@ func (is *ImageShim) ApplyCMD(rootfs string) string {
func (is *ImageShim) DeleteCMD(rootfs string) string {
return fmt.Sprintf("rm -rf %s", is.GetInfo(rootfs))
}
func NewImageShimHelper(execer ssh.Interface, host string) *ImageShim {
return &ImageShim{execer, host}
}
pkg/constants/data.go
+8 -1
View File
@@ -16,7 +16,10 @@ limitations under the License.
package constants
import "path/filepath"
import (
"io/fs"
"path/filepath"
)
var (
DefaultClusterRootfsDir = "/var/lib/sealos"
@@ -52,6 +55,10 @@ func GetAppWorkDir(clusterName, applicationName string) string {
return filepath.Join(DataPath(), clusterName, "applications", applicationName, "workdir")
}
func IsRegistryDir(entry fs.DirEntry) bool {
return entry.IsDir() && entry.Name() == RegistryDirName
}
type Data interface {
Homedir() string
RootFSPath() string
pkg/filesystem/registry/registry.go
+58
View File
@@ -0,0 +1,58 @@
/*
Copyright 2022 fengxsong@outlook.com
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"context"
"golang.org/x/sync/errgroup"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/ssh"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
)
type Interface interface {
MirrorTo(context.Context, ...string) error
}
type scp struct {
root string
ssh ssh.Interface
mounts []v2.MountImage
}
func (s *scp) MirrorTo(ctx context.Context, hosts ...string) error {
if ctx == nil {
ctx = context.Background()
}
eg, _ := errgroup.WithContext(ctx)
for i := range s.mounts {
m := s.mounts[i]
for j := range hosts {
host := hosts[j]
eg.Go(func() error {
return ssh.CopyDir(s.ssh, host, m.MountPoint, s.root, constants.IsRegistryDir)
})
}
}
return eg.Wait()
}
func New(root string, ssh ssh.Interface, mounts []v2.MountImage) Interface {
return &scp{root, ssh, mounts}
}
pkg/filesystem/rootfs/rootfs.go
+1 -1
View File
@@ -20,7 +20,7 @@ import v2 "github.com/labring/sealos/pkg/types/v1beta1"
type Interface interface {
// MountRootfs :send cloud rootfs to all hosts.
MountRootfs(cluster *v2.Cluster, hosts []string, initFlag, appFlag bool) error
MountRootfs(cluster *v2.Cluster, hosts []string) error
// UnMountRootfs :umount rootfs on all hosts.
UnMountRootfs(cluster *v2.Cluster, hosts []string) error
}
pkg/filesystem/rootfs/rootfs_default.go
+34 -89
View File
@@ -19,23 +19,19 @@ package rootfs
import (
"context"
"fmt"
"os"
"io/fs"
"path"
"path/filepath"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/ssh"
"github.com/labring/sealos/pkg/utils/exec"
"github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/iputils"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/env"
"github.com/labring/sealos/pkg/runtime"
"github.com/labring/sealos/pkg/ssh"
v2 "github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/exec"
"github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/logger"
)
type defaultRootfs struct {
@@ -45,8 +41,8 @@ type defaultRootfs struct {
mounts []v2.MountImage
}
func (f *defaultRootfs) MountRootfs(cluster *v2.Cluster, hosts []string, initFlag, appFlag bool) error {
return f.mountRootfs(cluster, hosts, initFlag, appFlag)
func (f *defaultRootfs) MountRootfs(cluster *v2.Cluster, hosts []string) error {
return f.mountRootfs(cluster, hosts)
}
func (f *defaultRootfs) UnMountRootfs(cluster *v2.Cluster, hosts []string) error {
@@ -61,9 +57,10 @@ func (f *defaultRootfs) getSSH(cluster *v2.Cluster) ssh.Interface {
return ssh.NewSSHClient(&cluster.Spec.SSH, true)
}
func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string, initFlag, appFlag bool) error {
func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error {
target := constants.NewData(f.getClusterName(cluster)).RootFSPath()
eg, _ := errgroup.WithContext(context.Background())
ctx := context.Background()
eg, _ := errgroup.WithContext(ctx)
envProcessor := env.NewEnvProcessor(cluster, f.mounts)
for _, mount := range f.mounts {
src := mount
@@ -92,55 +89,29 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string, initFl
if err := eg.Wait(); err != nil {
return err
}
check := constants.NewBash(f.getClusterName(cluster), cluster.GetImageLabels())
sshClient := f.getSSH(cluster)
shim := runtime.ImageShim{
SSHInterface: sshClient,
IP: cluster.GetMaster0IPAndPort(),
}
cper := &copier{target, sshClient}
for _, IP := range ipList {
ip := IP
notRegistryDirFilter := func(entry fs.DirEntry) bool { return !constants.IsRegistryDir(entry) }
for idx := range ipList {
ip := ipList[idx]
eg.Go(func() error {
fileEg, _ := errgroup.WithContext(context.Background())
for _, mount := range f.mounts {
mountInfo := mount
fileEg.Go(func() error {
if mountInfo.Type == v2.RootfsImage {
logger.Debug("send rootfs mount images ,ip: %s , init flag: %v, app flag: %v,image name: %s, image type: %s", ip, initFlag, appFlag, mountInfo.ImageName, mountInfo.Type)
err := cper.CopyFiles(ip, mountInfo.MountPoint, target, iputils.GetHostIP(ip) == cluster.GetRegistryIP())
egg, _ := errgroup.WithContext(ctx)
for idj := range f.mounts {
mount := f.mounts[idj]
egg.Go(func() error {
switch mount.Type {
case v2.RootfsImage, v2.PatchImage:
logger.Debug("send mount image, ip: %s, image name: %s, image type: %s", ip, mount.ImageName, mount.Type)
err := ssh.CopyDir(sshClient, ip, mount.MountPoint, target, notRegistryDirFilter)
if err != nil {
return fmt.Errorf("copy container %s rootfs failed %v", mountInfo.Name, err)
return fmt.Errorf("failed to copy %s %s: %v", mount.Type, mount.Name, err)
}
}
return nil
})
}
if err := fileEg.Wait(); err != nil {
return err
}
for _, mountInfo := range f.mounts {
if mountInfo.Type == v2.PatchImage {
logger.Debug("send addons mount images ,ip: %s , init flag: %v, app flag: %v,image name: %s, image type: %s", ip, initFlag, appFlag, mountInfo.ImageName, mountInfo.Type)
err := cper.CopyFiles(ip, mountInfo.MountPoint, target, iputils.GetHostIP(ip) == cluster.GetRegistryIP())
if err != nil {
return fmt.Errorf("copy container %s rootfs failed %v", mountInfo.Name, err)
}
}
}
if initFlag {
checkBash := check.CheckBash()
if checkBash == "" {
return nil
}
if err := f.getSSH(cluster).CmdAsync(ip, envProcessor.WrapperShell(ip, check.CheckBash()), shim.ApplyCMD(target)); err != nil {
return err
}
if err := f.getSSH(cluster).CmdAsync(ip, envProcessor.WrapperShell(ip, check.InitBash())); err != nil {
return err
}
}
return nil
return egg.Wait()
})
}
err := eg.Wait()
@@ -148,16 +119,16 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string, initFl
return err
}
endEg, _ := errgroup.WithContext(context.Background())
for _, mount := range f.mounts {
ip := cluster.GetMaster0IPAndPort()
mountInfo := mount
endEg, _ := errgroup.WithContext(ctx)
master0 := cluster.GetMaster0IPAndPort()
for idx := range f.mounts {
mountInfo := f.mounts[idx]
endEg.Go(func() error {
if appFlag && mountInfo.Type == v2.AppImage {
logger.Debug("send app mount images ,ip: %s , init flag: %v, app flag: %v,image name: %s, image type: %s", ip, initFlag, appFlag, mountInfo.ImageName, mountInfo.Type)
err = cper.CopyFiles(ip, mountInfo.MountPoint, constants.GetAppWorkDir(cluster.Name, mountInfo.Name), iputils.GetHostIP(ip) == cluster.GetRegistryIP())
if mountInfo.Type == v2.AppImage {
logger.Debug("send app mount images, ip: %s, image name: %s, image type: %s", master0, mountInfo.ImageName, mountInfo.Type)
err = ssh.CopyDir(sshClient, master0, mountInfo.MountPoint, constants.GetAppWorkDir(cluster.Name, mountInfo.Name), notRegistryDirFilter)
if err != nil {
return fmt.Errorf("copy container %s app rootfs failed %v", mountInfo.Name, err)
return fmt.Errorf("failed to copy %s %s: %v", mountInfo.Type, mountInfo.Name, err)
}
}
return nil
@@ -202,32 +173,6 @@ func renderENV(mountDir string, ipList []string, p env.Interface) error {
return nil
}
type copier struct {
root string
ssh ssh.Interface
}
func (c *copier) CopyFiles(ip, src, target string, isRegistry bool) error {
logger.Debug("copyFiles isRegistry: %v, ip: %v, src: %v, target: %v", isRegistry, ip, src, target)
files, err := os.ReadDir(src)
if err != nil {
return fmt.Errorf("failed to read dir entries %s", err)
}
for _, f := range files {
if f.Name() == constants.RegistryDirName {
if !isRegistry {
continue
}
target = c.root
}
err = c.ssh.Copy(ip, filepath.Join(src, f.Name()), filepath.Join(target, f.Name()))
if err != nil {
return fmt.Errorf("failed to copy sub files %v", err)
}
}
return nil
}
func NewDefaultRootfs(mounts []v2.MountImage) (Interface, error) {
return &defaultRootfs{mounts: mounts}, nil
}
pkg/runtime/registry.go
-102
View File
@@ -16,113 +16,11 @@ package runtime
import (
"fmt"
"path"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/passwd"
"github.com/labring/sealos/pkg/ssh"
"github.com/labring/sealos/pkg/types/v1beta1"
"github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/iputils"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/yaml"
)
func GetRegistryInfo(sshInterface ssh.Interface, rootfs, defaultRegistry string) *v1beta1.RegistryConfig {
const registryCustomConfig = "registry.yml"
var DefaultConfig = &v1beta1.RegistryConfig{
IP: iputils.GetHostIP(defaultRegistry),
Domain: constants.DefaultRegistryDomain,
Port: "5000",
Username: constants.DefaultRegistryUsername,
Password: constants.DefaultRegistryPassword,
Data: constants.DefaultRegistryData,
}
etcPath := path.Join(rootfs, constants.EtcDirName, registryCustomConfig)
out, _ := sshInterface.Cmd(defaultRegistry, fmt.Sprintf("cat %s", etcPath))
logger.Debug("image shim data info: %s", string(out))
registryConfig, err := yaml.UnmarshalData(out)
if err != nil {
logger.Warn("read registry config path error: %+v", err)
logger.Info("use default registry config")
return DefaultConfig
}
domain, _, _ := unstructured.NestedString(registryConfig, "domain")
port, _, _ := unstructured.NestedString(registryConfig, "port")
username, _, _ := unstructured.NestedString(registryConfig, "username")
password, _, _ := unstructured.NestedString(registryConfig, "password")
data, _, _ := unstructured.NestedString(registryConfig, "data")
ip, _, _ := unstructured.NestedString(registryConfig, "ip")
if ip == "" {
ip = defaultRegistry
}
if domain == "" {
domain = DefaultConfig.Domain
}
if port == "" {
port = DefaultConfig.Port
}
rConfig := &v1beta1.RegistryConfig{
IP: ip,
Domain: domain,
Port: port,
Username: username,
Password: password,
Data: data,
}
logger.Debug("show registry info, IP: %s, Domain: %s, Data: %s", rConfig.IP, rConfig.Domain, rConfig.Data)
return rConfig
}
func (k *KubeadmRuntime) htpasswd(registry *v1beta1.RegistryConfig) error {
htpasswdEtcPath := path.Join(k.getContentData().EtcPath(), "registry_htpasswd")
if registry.Username == "" && registry.Password == "" {
logger.Warn("registry username and password is empty")
return nil
}
logger.Debug("get htpasswd data: ip %s, username %s, password %s", registry.IP, registry.Username, registry.Password)
data := passwd.Htpasswd(registry.Username, registry.Password)
logger.Debug("write htpasswd etc file: %s,data: %s", htpasswdEtcPath, data)
if err := file.WriteFile(htpasswdEtcPath, []byte(data)); err != nil {
return errors.Wrap(err, "write registry htpasswd error")
}
htpasswdDetEtcPath := path.Join(k.getContentData().RootFSEtcPath(), "registry_htpasswd")
return k.sshCopy(registry.IP, htpasswdEtcPath, htpasswdDetEtcPath)
}
func (k *KubeadmRuntime) ApplyRegistry() error {
logger.Info("start to apply registry")
registry := k.getRegistry()
lnCmd := fmt.Sprintf(constants.DefaultLnFmt, k.getContentData().RootFSRegistryPath(), registry.Data)
logger.Debug("apply registry ln cmd: %s", lnCmd)
err := k.sshCmdAsync(registry.IP, lnCmd)
if err != nil {
return fmt.Errorf("copy registry data failed %v", err)
}
err = k.htpasswd(registry)
if err != nil {
return fmt.Errorf("generator registry htpasswd failed %v", err)
}
err = k.execInitRegistry(registry.IP)
if err != nil {
return fmt.Errorf("exec registry.sh failed %v", err)
}
return nil
}
func (k *KubeadmRuntime) DeleteRegistry() error {
ip := k.getRegistryIPAndPort()
logger.Info("delete registry in %s...", ip)
if err := k.execCleanRegistry(ip); err != nil {
return fmt.Errorf("exec clean-registry.sh failed %v", err)
}
return nil
}
func (k *KubeadmRuntime) registryAuth(ip string) error {
logger.Info("registry auth in node %s", ip)
registry := k.getRegistry()
pkg/runtime/reset.go
+6 -7
View File
@@ -18,10 +18,11 @@ import (
"context"
"fmt"
"golang.org/x/sync/errgroup"
"github.com/labring/sealos/pkg/bootstrap"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/utils/logger"
"golang.org/x/sync/errgroup"
)
const (
@@ -36,7 +37,8 @@ rm -rf %s
func (k *KubeadmRuntime) reset() error {
//Why delete registry is first?
//Because the registry use some cri command, eg 'nerdctl'
err := k.DeleteRegistry()
bs := bootstrap.New(k.Cluster)
err := bs.Reset(k.Cluster.GetRegistryIPAndPortList()...)
k.resetNodes(k.getNodeIPAndPortList())
k.resetMasters(k.getMasterIPAndPortList())
return err
@@ -71,10 +73,7 @@ func (k *KubeadmRuntime) resetMasters(nodes []string) {
func (k *KubeadmRuntime) resetNode(node string) error {
logger.Info("start to reset node: %s", node)
resetCmd := fmt.Sprintf(remoteCleanMasterOrNode, vlogToStr(k.vlog), k.getEtcdDataDir())
shim := &ImageShim{
SSHInterface: k.getSSHInterface(),
IP: k.getMaster0IPAndPort(),
}
shim := bootstrap.NewImageShimHelper(k.getSSHInterface(), k.getMaster0IPAndPort())
deleteShimCmd := shim.DeleteCMD(k.getContentData().RootFSPath())
if err := k.sshCmdAsync(node, resetCmd); err != nil {
logger.Error("failed to clean node, exec command %s failed, %v", resetCmd, err)
pkg/runtime/runtime.go
-1
View File
@@ -46,7 +46,6 @@ func (k *KubeadmRuntime) Init() error {
k.ConfigInitKubeadmToMaster0,
k.UpdateCert,
k.CopyStaticFilesToMasters,
k.ApplyRegistry,
k.InitMaster0,
}
pkg/runtime/runtime_getter.go
+3 -7
View File
@@ -23,6 +23,7 @@ import (
"golang.org/x/sync/errgroup"
"github.com/labring/sealos/pkg/bootstrap"
"github.com/labring/sealos/pkg/constants"
"github.com/labring/sealos/pkg/env"
"github.com/labring/sealos/pkg/remote"
@@ -34,7 +35,7 @@ import (
func (k *KubeadmRuntime) getRegistry() *v1beta1.RegistryConfig {
k.registryOnce.Do(func() {
k.Registry = GetRegistryInfo(k.getSSHInterface(), k.getContentData().RootFSPath(), k.getRegistryIPAndPort())
k.Registry = bootstrap.GetRegistryInfo(k.getSSHInterface(), k.getContentData().RootFSPath(), k.getRegistryIPAndPort())
})
return k.Registry
}
@@ -165,12 +166,7 @@ func (k *KubeadmRuntime) execHostsDelete(ip, domain string) error {
func (k *KubeadmRuntime) execClean(ip string) error {
return k.getSSHInterface().CmdAsync(ip, k.getENVInterface().WrapperShell(ip, k.getScriptsBash().CleanBash()))
}
func (k *KubeadmRuntime) execInitRegistry(ip string) error {
return k.getSSHInterface().CmdAsync(ip, k.getENVInterface().WrapperShell(ip, k.getScriptsBash().InitRegistryBash()))
}
func (k *KubeadmRuntime) execCleanRegistry(ip string) error {
return k.getSSHInterface().CmdAsync(ip, k.getENVInterface().WrapperShell(ip, k.getScriptsBash().CleanRegistryBash()))
}
func (k *KubeadmRuntime) execAuth(ip string) error {
return k.getSSHInterface().CmdAsync(ip, k.getENVInterface().WrapperShell(ip, k.getScriptsBash().AuthBash()))
}
pkg/ssh/utils.go
+40
View File
@@ -0,0 +1,40 @@
/*
Copyright 2022 fengxsong@outlook.com
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ssh
import (
"fmt"
"io/fs"
"os"
"path/filepath"
)
func CopyDir(sshClient Interface, host, src, dest string, filter func(fs.DirEntry) bool) error {
entries, err := os.ReadDir(src)
if err != nil {
return fmt.Errorf("failed to read dir entries %s", err)
}
for _, f := range entries {
if filter(f) {
err = sshClient.Copy(host, filepath.Join(src, f.Name()), filepath.Join(dest, f.Name()))
if err != nil {
return fmt.Errorf("failed to copy entry %v", err)
}
}
}
return nil
}
pkg/types/v1beta1/cluster_args.go
+27 -9
View File
@@ -26,19 +26,23 @@ import (
func (c *Cluster) GetSSH() SSH {
return c.Spec.SSH
}
func (c *Cluster) SetSSH(ssh SSH) {
c.Spec.SSH = ssh
}
func (c *Cluster) GetHosts() []Host {
return c.Spec.Hosts
}
func (c *Cluster) SetHosts(hosts []Host) {
c.Spec.Hosts = hosts
}
func (c *Cluster) GetMasterIPList() []string {
return iputils.GetHostIPs(c.GetIPSByRole(MASTER))
return iputils.GetHostIPs(c.GetMasterIPAndPortList())
}
func (c *Cluster) GetMasterIPAndPortList() []string {
return c.GetIPSByRole(MASTER)
}
@@ -47,14 +51,30 @@ func (c *Cluster) GetNodeIPList() []string {
return iputils.GetHostIPs(c.GetIPSByRole(NODE))
}
func (c *Cluster) GetRegistryIP() string {
return c.GetMaster0IP()
}
func (c *Cluster) GetNodeIPAndPortList() []string {
return c.GetIPSByRole(NODE)
}
func (c *Cluster) GetRegistryIP() string {
return iputils.GetHostIP(c.GetRegistryIPAndPort())
}
func (c *Cluster) GetRegistryIPAndPort() string {
return c.GetRegistryIPAndPortList()[0]
}
func (c *Cluster) GetRegistryIPList() []string {
return iputils.GetHostIPs(c.GetRegistryIPAndPortList())
}
func (c *Cluster) GetRegistryIPAndPortList() []string {
ret := c.GetIPSByRole(REGISTRY)
if len(ret) == 0 {
ret = []string{c.GetMaster0IPAndPort()}
}
return ret
}
func (c *Cluster) GetMaster0IP() string {
if len(c.Spec.Hosts) == 0 {
return ""
@@ -75,10 +95,6 @@ func (c *Cluster) GetMaster0IPAndPort() string {
return c.Spec.Hosts[0].IPS[0]
}
func (c *Cluster) GetRegistryIPAndPort() string {
return c.GetMaster0IPAndPort()
}
func (c *Cluster) GetMaster0IPAPIServer() string {
master0 := c.GetMaster0IP()
return fmt.Sprintf("https://%s:6443", master0)
@@ -129,6 +145,7 @@ func (c *Cluster) FindImage(targetImage string) *MountImage {
}
return image
}
func (c *Cluster) SetMountImage(targetMount *MountImage) {
tgMount := targetMount.DeepCopy()
if c.Status.Mounts != nil {
@@ -188,6 +205,7 @@ func (c *Cluster) GetAppImage(defaultImageName, defaultMount string) *MountImage
}
return image
}
func (c *Cluster) HasAppImage() bool {
if c.Status.Mounts != nil {
for _, img := range c.Status.Mounts {
pkg/types/v1beta1/constants.go
+3 -2
View File
@@ -21,8 +21,9 @@ const (
)
var (
MASTER = "master"
NODE = "node"
MASTER = "master"
NODE = "node"
REGISTRY = "registry"
)
type Provider string
proposal/High-availability-for-Container-Registry.md
+74
View File
@@ -0,0 +1,74 @@
# High availability for Container Registry
## Summary
Currently the only single point in the cluster is the builtin container registry. Once if the first master node or even only the registry fails with unexpected error, the rest of the nodes in the cluster will fail to pull images. The proposal is to make sure high availability for registry by initializing multiple registries in the cluster, and also distribute offline artifacts to all nodes that with registry role.
## Design Details
1. Added a role named `registry`, and currently only supports apparently specifying a set of host in the `Clusterfile`, the first master node will be selected as the local registry by default if none of the nodes are selected.
a) If no roles are specified, `sealos` will behave as before, aka, this PR does not change all the default behavior of sealos
b) If we choose some nodes to run as registry, `bootstrap` process will run registry in those hosts before cluster initialization, `run` process will also distribute `registry` dir to those hosts.
2. Refactored the `filesystem` module and move the system initialization operation to `bootstrap` module, now the `filesystem` module can focus on mounting and unmounting the application image.
The `bootstrap` interface is designed as follows
```go
type Interface interface {
// Preflight run preflight checks or something need to be performed before init
Preflight(hosts ...string) error
// Init do actual system initialization, cri
Init(hosts ...string) error
// Register applier to each Phase
RegisterApplier(Phase, ...Applier) error
// ApplyAddons run task to apply addons
ApplyAddons(hosts ...string) error
// Reset undo bootstrap process
Reset(hosts ...string) error
}
```
`Applier` is a interface that can be registered as hook at different phases
```go
type Applier interface {
Name() string
Filter(Context, string) bool
Apply(Context, string) error
Undo(Context, string) error
}
```
the builtin implementations are:
- `defaultChecker` which do preflight checks
- `defaultInitializer` which do actual initializations
- `registryApplier` which install/uninstall container registry.
The `Context` interface can be used as arg for function `Applier.Filter`/`Applier.Apply`/`Applier.Undo`.
```go
type Context interface {
GetBash() constants.Bash
GetCluster() *v2.Cluster
GetData() constants.Data
GetExecer() ssh.Interface
GetShellWrapper() shellWrapper
}
```
## Limitations
- `registry` node must be one of the nodes in the cluster, can't use external registry yet.
- since **currently only supports apparently specifying a set of host in the `Clusterfile`**, registry doesn't support scaling after cluster initialized.
## Features not yet implemented
- registry role flag in `sealos run`/`sealos add`/`sealos remove` command
- high availability, how to ensure HA? use `lvscare` like the way`kube-apiserver` does, OR create a externalName type service?
## Implementation History
- 2022-11-17 First [Proposal and PR](https://github.com/labring/sealos/pull/2096)