refactor(image-cri-shim): remove registry.d support (#6089)

This commit is contained in:
cuisongliu
2025-10-15 12:33:02 +08:00
committed by GitHub
parent 739972f3a8
commit 178c4be92c
6 changed files with 68 additions and 331 deletions
+4 -16
View File
@@ -127,15 +127,9 @@ func watchAuthConfig(ctx context.Context, path string, imgShim shim.Shim, interv
lastHash := ""
if data, err := os.ReadFile(path); err == nil {
if cfg, err := types.UnmarshalData(data); err == nil {
cfg.RegistryDir = types.NormalizeRegistryDir(cfg.RegistryDir)
digest, err := types.RegistryDirDigest(cfg.RegistryDir)
if err != nil {
logger.Warn("failed to fingerprint registry.d %s: %v", cfg.RegistryDir, err)
} else {
sum := sha256.Sum256(append(data, digest...))
lastHash = hex.EncodeToString(sum[:])
}
if _, err := types.UnmarshalData(data); err == nil {
sum := sha256.Sum256(data)
lastHash = hex.EncodeToString(sum[:])
} else {
logger.Warn("failed to parse shim config %s: %v", path, err)
}
@@ -161,13 +155,7 @@ func watchAuthConfig(ctx context.Context, path string, imgShim shim.Shim, interv
logger.Warn("failed to parse shim config %s: %v", path, err)
continue
}
cfg.RegistryDir = types.NormalizeRegistryDir(cfg.RegistryDir)
digest, err := types.RegistryDirDigest(cfg.RegistryDir)
if err != nil {
logger.Warn("failed to fingerprint registry.d %s: %v", cfg.RegistryDir, err)
continue
}
sum := sha256.Sum256(append(data, digest...))
sum := sha256.Sum256(data)
hash := hex.EncodeToString(sum[:])
if hash == lastHash {
continue
+30 -28
View File
@@ -18,7 +18,6 @@ package cmd
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
@@ -51,28 +50,18 @@ func (f *fakeShim) UpdateAuth(auth *types.ShimAuthConfig) {
func TestWatchAuthConfigReloads(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "shim-config.yaml")
registryDir := filepath.Join(dir, "registry.d")
if err := os.MkdirAll(registryDir, 0o755); err != nil {
t.Fatalf("failed to create registry.d directory: %v", err)
}
mirrorPath := filepath.Join(registryDir, "mirror.yaml")
if err := os.WriteFile(mirrorPath, []byte("address: \"https://mirror.example.com\"\nauth: \"user:pass\"\n"), 0o644); err != nil {
t.Fatalf("failed to write mirror registry config: %v", err)
}
if err := os.WriteFile(filepath.Join(registryDir, "public.yaml"), []byte("address: \"https://public.example.com\"\n"), 0o644); err != nil {
t.Fatalf("failed to write public registry config: %v", err)
}
initialConfig := []byte(fmt.Sprintf(`shim: "/tmp/test.sock"
initialConfig := []byte(`shim: "/tmp/test.sock"
cri: "/var/run/containerd/containerd.sock"
address: "https://example.com"
force: true
auth: "offline:initial"
reloadInterval: 10ms
registry.d: %q
`, registryDir))
registries:
- address: "https://mirror.example.com"
auth: "user:pass"
- address: "https://public.example.com"
`)
if err := os.WriteFile(cfgPath, initialConfig, 0o644); err != nil {
t.Fatalf("failed to write initial config: %v", err)
@@ -89,14 +78,17 @@ registry.d: %q
time.Sleep(20 * time.Millisecond)
updatedConfig := []byte(fmt.Sprintf(`shim: "/tmp/test.sock"
updatedConfig := []byte(`shim: "/tmp/test.sock"
cri: "/var/run/containerd/containerd.sock"
address: "https://example.com"
force: true
auth: "offline:updated"
reloadInterval: 10ms
registry.d: %q
`, registryDir))
registries:
- address: "https://mirror.example.com"
auth: "user:pass"
- address: "https://public.example.com"
`)
if err := os.WriteFile(cfgPath, updatedConfig, 0o644); err != nil {
t.Fatalf("failed to write updated config: %v", err)
@@ -122,24 +114,34 @@ registry.d: %q
t.Fatalf("expected registry credentials for mirror.example.com, got %#v", auth.CRIConfigs)
}
if mirror.Password != "pass" {
t.Fatalf("expected initial password from registry.d, got %q", mirror.Password)
t.Fatalf("expected initial mirror password, got %q", mirror.Password)
}
if !auth.SkipLoginRegistries["public.example.com"] {
t.Fatalf("expected public.example.com to skip login, got %#v", auth.SkipLoginRegistries)
}
if cfg, ok := auth.CRIConfigs["public.example.com"]; !ok || cfg.Username != "" || cfg.Password != "" {
t.Fatalf("expected public.example.com to use anonymous credentials, got %#v", cfg)
}
if err := os.WriteFile(mirrorPath, []byte("address: \"https://mirror.example.com\"\nauth: \"user:changed\"\n"), 0o644); err != nil {
t.Fatalf("failed to write updated mirror registry config: %v", err)
// update mirror credentials via config file
mirrorUpdated := []byte(`shim: "/tmp/test.sock"
cri: "/var/run/containerd/containerd.sock"
address: "https://example.com"
force: true
auth: "offline:updated"
reloadInterval: 10ms
registries:
- address: "https://mirror.example.com"
auth: "user:changed"
- address: "https://public.example.com"
`)
if err := os.WriteFile(cfgPath, mirrorUpdated, 0o644); err != nil {
t.Fatalf("failed to write mirror update: %v", err)
}
select {
case auth = <-shim.updates:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for registry.d auth update")
t.Fatal("timed out waiting for mirror auth update")
}
mirror, ok = auth.CRIConfigs["mirror.example.com"]
@@ -147,7 +149,7 @@ registry.d: %q
t.Fatalf("expected registry credentials for mirror.example.com after update, got %#v", auth.CRIConfigs)
}
if mirror.Password != "changed" {
t.Fatalf("expected updated password from registry.d, got %q", mirror.Password)
t.Fatalf("expected updated mirror password, got %q", mirror.Password)
}
cancel()
@@ -17,15 +17,8 @@ limitations under the License.
package types
import (
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"time"
@@ -45,7 +38,6 @@ const (
// SealosShimSock is the CRI socket the shim listens on.
SealosShimSock = "/var/run/image-cri-shim.sock"
DefaultImageCRIShimConfig = "/etc/image-cri-shim.yaml"
DefaultRegistryDir = "/etc/image-cri-shim.d"
DefaultReloadInterval = 15 * time.Second
)
@@ -63,7 +55,6 @@ type Config struct {
Timeout metav1.Duration `json:"timeout"`
ReloadInterval metav1.Duration `json:"reloadInterval"`
Auth string `json:"auth"`
RegistryDir string `json:"registry.d"`
Registries []Registry `json:"registries" yaml:"registries,omitempty"`
}
@@ -79,15 +70,6 @@ func registryMatchDomain(reg Registry) string {
}
func (c *Config) PreProcess() (*ShimAuthConfig, error) {
c.RegistryDir = NormalizeRegistryDir(c.RegistryDir)
if err := syncRegistriesToDir(c.RegistryDir, c.Registries); err != nil {
return nil, err
}
registries, err := loadRegistriesFromDir(c.RegistryDir)
if err != nil {
return nil, err
}
if c.ImageShimSocket == "" {
c.ImageShimSocket = SealosShimSock
}
@@ -95,7 +77,6 @@ func (c *Config) PreProcess() (*ShimAuthConfig, error) {
logger.Info("cri-socket: %s", c.RuntimeSocket)
logger.Info("hub-address: %s", c.Address)
logger.Info("auth: %s", c.Auth)
logger.Info("registry.d: %s", c.RegistryDir)
rawURL, err := url.Parse(c.Address)
if err != nil {
logger.Warn("url parse error: %+v", err)
@@ -133,7 +114,7 @@ func (c *Config) PreProcess() (*ShimAuthConfig, error) {
// cri registry auth
criAuth := make(map[string]types2.AuthConfig)
skipLogin := make(map[string]bool)
for _, registry := range registries {
for _, registry := range c.Registries {
if registry.Address == "" {
continue
}
@@ -204,153 +185,3 @@ func UnmarshalData(metadata []byte) (*Config, error) {
}
return cfg, nil
}
// NormalizeRegistryDir ensures the registry directory falls back to the default path when empty.
func NormalizeRegistryDir(dir string) string {
if strings.TrimSpace(dir) == "" {
return DefaultRegistryDir
}
return dir
}
func syncRegistriesToDir(dir string, registries []Registry) error {
if len(registries) == 0 {
return nil
}
dir = NormalizeRegistryDir(dir)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("ensure registry.d %s: %w", dir, err)
}
for _, reg := range registries {
address := strings.TrimSpace(reg.Address)
if address == "" {
continue
}
host := registryFileName(address)
if host == "" {
logger.Warn("skip registry entry %q: unable to determine filename", address)
continue
}
data, err := yaml.Marshal(reg)
if err != nil {
return fmt.Errorf("marshal registry %s: %w", host, err)
}
path := filepath.Join(dir, fmt.Sprintf("%s.yaml", host))
if existing, err := os.ReadFile(path); err == nil {
if bytes.Equal(bytes.TrimSpace(existing), bytes.TrimSpace(data)) {
continue
}
} else if !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("read registry config %s: %w", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return fmt.Errorf("write registry config %s: %w", path, err)
}
logger.Debug("synced registry config to %s", path)
}
return nil
}
func registryFileName(address string) string {
if address == "" {
return ""
}
raw := strings.TrimSpace(address)
if raw == "" {
return ""
}
if u, err := url.Parse(raw); err == nil {
host := u.Host
if host != "" {
return host
}
// Some registry addresses might be provided without scheme, causing Parse to treat them as paths.
}
trimmed := strings.TrimPrefix(raw, "http://")
trimmed = strings.TrimPrefix(trimmed, "https://")
if idx := strings.Index(trimmed, "/"); idx != -1 {
trimmed = trimmed[:idx]
}
return strings.TrimSpace(trimmed)
}
func loadRegistriesFromDir(dir string) ([]Registry, error) {
files, err := listRegistryConfigFiles(dir)
if err != nil {
return nil, err
}
registries := make([]Registry, 0, len(files))
for _, file := range files {
data, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("read registry config %s: %w", file, err)
}
var reg Registry
if err := yaml.Unmarshal(data, &reg); err != nil {
return nil, fmt.Errorf("parse registry config %s: %w", file, err)
}
if strings.TrimSpace(reg.Address) == "" {
logger.Warn("skip registry config %s: address is empty", file)
continue
}
registries = append(registries, reg)
}
return registries, nil
}
func listRegistryConfigFiles(dir string) ([]string, error) {
dir = NormalizeRegistryDir(dir)
entries, err := os.ReadDir(dir)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("list registry.d %s: %w", dir, err)
}
files := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(entry.Name()))
if ext != ".yaml" && ext != ".yml" {
continue
}
files = append(files, filepath.Join(dir, entry.Name()))
}
sort.Strings(files)
return files, nil
}
// RegistryDirDigest returns a deterministic digest for the registry.d directory contents.
func RegistryDirDigest(dir string) ([]byte, error) {
files, err := listRegistryConfigFiles(dir)
if err != nil {
return nil, err
}
if len(files) == 0 {
return nil, nil
}
hasher := sha256.New()
for _, file := range files {
data, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("read registry config %s: %w", file, err)
}
_, _ = hasher.Write([]byte(filepath.Base(file)))
_, _ = hasher.Write(data)
}
return hasher.Sum(nil), nil
}
@@ -16,53 +16,16 @@ limitations under the License.
package types
import (
"os"
"path/filepath"
"testing"
"sigs.k8s.io/yaml"
)
import "testing"
func TestUnmarshal(t *testing.T) {
cfg, err := Unmarshal("testdata/image-cri-shim.yaml")
if err != nil {
t.Error(err)
return
}
auth, err := cfg.PreProcess()
if err != nil {
t.Error(err)
return
}
if _, ok := auth.CRIConfigs["192.168.64.1:5000"]; !ok {
t.Fatalf("expected registry credentials loaded from registry.d, got %#v", auth.CRIConfigs)
}
}
func TestRegistriesLoadedFromDir(t *testing.T) {
dir := t.TempDir()
cfg := &Config{
ImageShimSocket: "/var/run/image-cri-shim.sock",
RuntimeSocket: "/run/containerd/containerd.sock",
Address: "http://sealos.hub:5000",
Force: true,
Auth: "offline:user",
RegistryDir: dir,
t.Fatalf("unmarshal failed: %v", err)
}
entries := map[string]Registry{
"192.168.64.1:5000.yaml": {Address: "http://192.168.64.1:5000", Auth: "admin:passw0rd"},
"public.example.com.yaml": {Address: "https://public.example.com"},
}
for name, reg := range entries {
data, err := yaml.Marshal(reg)
if err != nil {
t.Fatalf("failed to marshal registry %s: %v", name, err)
}
if err := os.WriteFile(filepath.Join(dir, name), data, 0o644); err != nil {
t.Fatalf("failed to write registry file %s: %v", name, err)
}
if len(cfg.Registries) == 0 {
t.Fatalf("expected registries from config, got %#v", cfg.Registries)
}
auth, err := cfg.PreProcess()
@@ -70,62 +33,48 @@ func TestRegistriesLoadedFromDir(t *testing.T) {
t.Fatalf("preprocess failed: %v", err)
}
mirror, ok := auth.CRIConfigs["mirror.example.com"]
if !ok {
t.Fatalf("expected registry credentials for mirror.example.com, got %#v", auth.CRIConfigs)
}
if mirror.Username != "admin" || mirror.Password != "passw0rd" {
t.Fatalf("unexpected credentials for mirror.example.com, got %#v", mirror)
}
if !auth.SkipLoginRegistries["public.example.com"] {
t.Fatalf("expected public.example.com to skip login, got %#v", auth.SkipLoginRegistries)
}
if authCfg, ok := auth.CRIConfigs["192.168.64.1:5000"]; !ok || authCfg.Username != "admin" || authCfg.Password != "passw0rd" {
t.Fatalf("expected credentials for 192.168.64.1:5000, got %#v", auth.CRIConfigs["192.168.64.1:5000"])
}
}
func TestRegistriesFieldSyncedToDir(t *testing.T) {
dir := t.TempDir()
registries := []Registry{
{Address: "https://hub.192.168.64.4.nip.io", Auth: "admin:syncpass"},
{Address: "http://192.168.64.1:5000"},
}
func TestInlineRegistriesPreProcess(t *testing.T) {
cfg := &Config{
ImageShimSocket: "/var/run/image-cri-shim.sock",
RuntimeSocket: "/run/containerd/containerd.sock",
Address: "https://registry.internal",
Force: true,
Auth: "sync:user",
RegistryDir: dir,
Registries: registries,
Auth: "offline:user",
Registries: []Registry{
{Address: "https://mirror.internal", Auth: "admin:syncpass"},
{Address: "https://public.example.com"},
},
}
auth, err := cfg.PreProcess()
if err != nil {
t.Fatalf("preprocess with inline registries failed: %v", err)
t.Fatalf("preprocess failed: %v", err)
}
expectedFiles := map[string]Registry{
"hub.192.168.64.4.nip.io.yaml": registries[0],
"192.168.64.1:5000.yaml": registries[1],
mirrorDomain := registryMatchDomain(cfg.Registries[0])
mirror, ok := auth.CRIConfigs[mirrorDomain]
if !ok {
t.Fatalf("expected registry credentials for %s, got %#v", mirrorDomain, auth.CRIConfigs)
}
for name, want := range expectedFiles {
path := filepath.Join(dir, name)
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("expected registry file %s to be written: %v", path, err)
}
var got Registry
if err := yaml.Unmarshal(data, &got); err != nil {
t.Fatalf("failed to parse registry file %s: %v", path, err)
}
if got != want {
t.Fatalf("registry file %s content mismatch: got %#v, want %#v", path, got, want)
}
if mirror.Username != "admin" || mirror.Password != "syncpass" {
t.Fatalf("unexpected credentials for %s, got %#v", mirrorDomain, mirror)
}
firstDomain := registryMatchDomain(registries[0])
if authCfg, ok := auth.CRIConfigs[firstDomain]; !ok || authCfg.Username != "admin" || authCfg.Password != "syncpass" {
t.Fatalf("expected credentials for %s, got %#v", firstDomain, auth.CRIConfigs[firstDomain])
}
secondDomain := registryMatchDomain(registries[1])
if !auth.SkipLoginRegistries[secondDomain] {
t.Fatalf("expected %s to skip login, got %#v", secondDomain, auth.SkipLoginRegistries)
publicDomain := registryMatchDomain(cfg.Registries[1])
if !auth.SkipLoginRegistries[publicDomain] {
t.Fatalf("expected %s to skip login, got %#v", publicDomain, auth.SkipLoginRegistries)
}
}
@@ -21,4 +21,7 @@ image: /var/lib/image-cri-shim
timeout: 15m
reloadInterval: 15s
auth: admin:passw0rd
registry.d: testdata/registries
registries:
- address: https://mirror.example.com
auth: admin:passw0rd
- address: https://public.example.com
+1 -37
View File
@@ -17,11 +17,9 @@ package e2e
import (
"errors"
"fmt"
"path/filepath"
"sync"
"time"
"github.com/labring/sealos/pkg/utils/file"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
@@ -32,7 +30,6 @@ import (
"github.com/labring/image-cri-shim/pkg/server"
shimType "github.com/labring/image-cri-shim/pkg/types"
registry2 "github.com/labring/sreg/pkg/registry/crane"
"github.com/onsi/ginkgo/v2"
k8sv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -237,7 +234,6 @@ COPY image-cri-shim cri`
sourceImage = "nginx:latest"
rewrittenImage = "docker.m.daocloud.io/library/nginx:latest"
mirrorAddress = "https://docker.m.daocloud.io"
aliasDomain = "daocloud"
)
shimConfigRaw := utils.GetFileDataLocally(DefaultImageCRIShimConfig)
@@ -246,7 +242,7 @@ COPY image-cri-shim cri`
cfgCopy := *cfg
cfgCopy.ReloadInterval = metav1.Duration{Duration: time.Second}
cfgCopy.RegistryDir = shimType.DefaultRegistryDir
cfgCopy.Registries = append(cfgCopy.Registries, shimType.Registry{Address: mirrorAddress})
payload, err := yaml.Marshal(cfgCopy)
utils.CheckErr(err, "failed to marshal shim config with mirror")
@@ -256,36 +252,10 @@ COPY image-cri-shim cri`
waitForShimLog("reloaded shim auth configuration", restoreSince, 60*time.Second)
}(shimConfigRaw)
registryDir := shimType.NormalizeRegistryDir(cfgCopy.RegistryDir)
normalizedAlias := registry2.NormalizeRegistry(aliasDomain)
registryFile := filepath.Join(registryDir, fmt.Sprintf("%s.yaml", normalizedAlias))
_ = file.WriteFile(registryFile, []byte(fmt.Sprintf("address: %s", mirrorAddress)))
existingRegistry, regReadErr := exec.RunSimpleCmd(fmt.Sprintf("sudo cat %s", registryFile))
defer func() {
if regReadErr != nil {
exec.RunSimpleCmd(fmt.Sprintf("sudo rm -f %s", registryFile))
} else {
writeRegistryConfig(registryFile, existingRegistry)
}
}()
mirrorConfig := shimType.Registry{Address: mirrorAddress}
mirrorPayload, err := yaml.Marshal(mirrorConfig)
utils.CheckErr(err, "failed to marshal mirror registry config")
writeRegistryConfig(registryFile, string(mirrorPayload))
since := time.Now()
writeShimConfig(payload)
waitForShimLog("reloaded shim auth configuration", since, 60*time.Second)
gomega.Eventually(func() error {
_, err := exec.RunSimpleCmd(fmt.Sprintf("sudo test -f %s", registryFile))
return err
}, 30*time.Second, 2*time.Second).Should(gomega.Succeed(), fmt.Sprintf("expected registry config %s to exist", registryFile))
registryData := utils.GetFileDataLocally(registryFile)
gomega.Expect(registryData).To(gomega.ContainSubstring(mirrorAddress))
_, _ = fakeClient.CmdInterface.Exec("crictl", "rmi", sourceImage)
_, _ = fakeClient.CmdInterface.Exec("crictl", "rmi", rewrittenImage)
@@ -309,12 +279,6 @@ func writeShimConfig(data []byte) {
utils.CheckErr(err, "failed to write shim config")
}
func writeRegistryConfig(path string, content string) {
cmd := fmt.Sprintf("cat <<'EOF' | sudo tee %s >/dev/null\n%s\nEOF", path, content)
_, err := exec.RunSimpleCmd(cmd)
utils.CheckErr(err, fmt.Sprintf("failed to write registry config %s", path))
}
func waitForShimLog(fragment string, since time.Time, timeout time.Duration) {
gomega.Eventually(func() string {
cmd := fmt.Sprintf("sudo journalctl -u image-cri-shim --since \"%s\" --no-pager", since.Add(-5*time.Second).Format(shimJournalTimeLayout))