diff --git a/cmd/sealctl/cmd/registry.go b/cmd/sealctl/cmd/registry.go index 8e85d7549..aeeb50839 100644 --- a/cmd/sealctl/cmd/registry.go +++ b/cmd/sealctl/cmd/registry.go @@ -17,45 +17,20 @@ limitations under the License. package cmd import ( - "runtime" - "github.com/spf13/cobra" "github.com/labring/sealos/pkg/registry/cmd" ) -var ( - registryPullRegistryDir string - registryPullArch string - registryPullMaxPullProcs int -) +func init() { + rootCmd.AddCommand(newRegistryImageCmd()) +} func newRegistryImageCmd() *cobra.Command { var registryImageCmd = &cobra.Command{ Use: "registry", Short: "registry images manager", } - - registryImageCmd.AddCommand(cmd.NewRegistryListCmd()) - registryImageCmd.AddCommand(cmd.NewRegistryImageCmd()) - registryImageCmd.AddCommand(cmd.NewRegistryPruneCmd()) - registryImageCmd.AddCommand(newRegistryImageSaveCmd()) + cmd.RegisterRootCommand(registryImageCmd, rootCmd) return registryImageCmd } - -func newRegistryImageSaveCmd() *cobra.Command { - var registryImagePullCmd = &cobra.Command{ - Use: "save", - Short: "save registry images to local registry dir", - } - registryImagePullCmd.PersistentFlags().StringVar(®istryPullArch, "arch", runtime.GOARCH, "pull images arch") - registryImagePullCmd.PersistentFlags().StringVar(®istryPullRegistryDir, "data-dir", "/var/lib/registry", "registry data dir path") - registryImagePullCmd.PersistentFlags().IntVar(®istryPullMaxPullProcs, "max-pull-procs", 5, "maximum number of goroutines for pulling") - registryImagePullCmd.AddCommand(cmd.NewSaveCmd(registryPullRegistryDir, registryPullArch, registryPullMaxPullProcs, "default")) - registryImagePullCmd.AddCommand(cmd.NewSaveCmd(registryPullRegistryDir, registryPullArch, registryPullMaxPullProcs, "raw")) - return registryImagePullCmd -} - -func init() { - rootCmd.AddCommand(newRegistryImageCmd()) -} diff --git a/docs/4.0/docs/examples/sealctl-registry.md b/docs/4.0/docs/examples/sealctl-registry.md new file mode 100644 index 000000000..f574d6c71 --- /dev/null +++ b/docs/4.0/docs/examples/sealctl-registry.md @@ -0,0 +1,227 @@ +# Using sealctl to manage registry images + +## Save images to local registry directory + +### default type + +The default mode scans images in the context directory. The logic of scanning images is the logic of build images, and the directories of charts, manifests, and images scan images. + +``` +registry images manager save to local dir by default type + +Usage: + sealctl registry save default [CONTEXT] [flags] + +Flags: + --arch string pull images arch (default "arm64") + --data-dir string registry data dir path (default "/var/lib/registry") + -h, --help help for default + --max-pull-procs int maximum number of goroutines for pulling (default 5) + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` + + + +#### Usage + +```shell +sealctl registry save default . +``` + +### raw type + +raw type using images as image list. + +``` +registry images manager save to local dir by raw type + +Usage: + sealctl registry save raw [flags] + +Flags: + --arch string pull images arch (default "arm64") + --data-dir string registry data dir path (default "/var/lib/registry") + -h, --help help for raw + --images strings images list + --max-pull-procs int maximum number of goroutines for pulling (default 5) + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` +#### Usage + +```shell +sealctl registry save raw --images nginx +``` + +## Check registry status + +To use this command, you need to `sealctl login` or `sealos login` first. + +``` +registry status + +Usage: + sealctl registry status [flags] + +Examples: +sealctl registry status + +Flags: + -h, --help help for status + --json output in JSON format + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` + +#### Usage +```shell +$ sealctl registry status ++--------------------+---------------------------+----------+----------+---------+ +| Name | URL | UserName | Password | Healthy | ++--------------------+---------------------------+----------+----------+---------+ +| 192.168.64.63:5000 | http://192.168.64.63:5000 | admin | passw0rd | ok | ++--------------------+---------------------------+----------+----------+---------+ +``` + + +## List registry image + +``` +registry image list + +Usage: + sealctl registry images [flags] + +Examples: +Example: + sealctl registry images --filter name=public* + sealctl registry images --filter tag=*1.1* + sealctl registry images --filter tag=*sec + sealctl registry images --filter name=public,tag=v1.1.1 + sealctl registry images --filter tag= + +Flags: + --filter string Filter support 'name' and 'tag' , strategy support prefix (eg key*),suffix(eg *key),equals(eg key),empty(eg ),like(eg *key*) + -h, --help help for images + --json output in JSON format + -n, --name string registry name (default "sealos.hub:5000") + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` + +#### Usage + +```shell +$ sealctl registry images -n 192.168.64.63:5000 --filter name=kube*,tag=* ++--------------------+-------------------------+---------+-------------------------------------------------------------------------+ +| RegistryName | ImageName | Tag | ImageID | ++--------------------+-------------------------+---------+-------------------------------------------------------------------------+ +| 192.168.64.63:5000 | kube-apiserver | v1.19.0 | sha256:522d17d35a8994637d27d1232bebd35cfae8e3e21ab359431403f2b8023e332c | +| 192.168.64.63:5000 | kube-controller-manager | v1.19.0 | sha256:6c11a3d4d06385f7547a5ea0c3f0d5e7b12baa558111d01406ac1f778cb3f00b | +| 192.168.64.63:5000 | kube-proxy | v1.19.0 | sha256:c752ecbd04bc4517168a19323bb60fb45324eee1e480b2b97d3fd6ea0a54f42d | +| 192.168.64.63:5000 | kube-scheduler | v1.19.0 | sha256:529a1566960a5b3024f2c94128e1cbd882ca1804f222ec5de99b25567858ecb9 | ++--------------------+-------------------------+---------+-------------------------------------------------------------------------+ +2023-01-01T12:31:49 info Image count 4 +2023-01-01T12:31:49 info Images Version count 4 +``` + +### Filtering + +Keys for `--filter`: +- `name` +- `tag` + +Expressions: +- `*val`: suffix +- `val*`: prefix +- `value`: equals +- ``: empty +- `*val*`: contains + + + +## Remove registry image + +``` +registry rmi image + +Usage: + sealctl registry rmi [flags] + +Examples: +sealctl registry rmi labring/lvscare:v4.1.3 + +Flags: + -h, --help help for rmi + -n, --name string registry name (default "sealos.hub:5000") + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` + +#### Usage + +```shell +$ sealctl registry rmi 192.168.64.63:5000/kube-apiserver:v1.19.0 -n 192.168.64.63:5000 +``` + +## Prune registry + +This command can only be run on a registry node, which used the binary deploy mode. Does not support nodes using container deploy mode. + +``` +registry `garbage-collect` deletes layers not referenced by any manifests + +Usage: + sealctl registry prune [flags] + +Flags: + -c, --config string registry config path (default "/etc/registry/registry_config.yml") + -u, --delete-untagged delete manifests that are not currently referenced via tag + -d, --dry-run do everything except remove the blobs + -h, --help help for prune + +Global Flags: + --debug enable debug logger + --root string storage root dir (default "/var/lib/containers/storage") + --runroot string storage state dir (default "/run/containers/storage") + --show-path enable show code path + --storage-driver string storage-driver (default "overlay") + --storage-opt strings storage driver option +``` + +#### Usage + +```shell +$ sealctl registry prune +``` diff --git a/fork/github.com/heroku/docker-registry-client/registry/json.go b/fork/github.com/heroku/docker-registry-client/registry/json.go index 85e2106e1..ad95a0f90 100644 --- a/fork/github.com/heroku/docker-registry-client/registry/json.go +++ b/fork/github.com/heroku/docker-registry-client/registry/json.go @@ -3,10 +3,9 @@ package registry import ( "encoding/json" "errors" - "fmt" + http2 "github.com/labring/sealos/pkg/utils/http" "net/http" "regexp" - "strings" ) var ( @@ -17,6 +16,9 @@ var ( // next page URL while updating pointed-to variable with a parsed JSON // value. When there are no more pages it returns `ErrNoMorePages`. func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) { + if _, ok := http2.IsURL(url); !ok { + url = registry.URL + url + } resp, err := registry.Client.Get(url) if err != nil { return "", err @@ -45,10 +47,6 @@ func getNextLink(resp *http.Response) (string, error) { for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] { parts := nextLinkRE.FindStringSubmatch(link) if parts != nil { - // support 2.7+ distribution - if strings.HasPrefix(parts[1], "/") { - return fmt.Sprintf("%s://%s%s", resp.Request.URL.Scheme, resp.Request.URL.Host, parts[1]), nil - } return parts[1], nil } } diff --git a/fork/github.com/heroku/docker-registry-client/registry/manifest.go b/fork/github.com/heroku/docker-registry-client/registry/manifest.go index 8558bbc76..10891be15 100644 --- a/fork/github.com/heroku/docker-registry-client/registry/manifest.go +++ b/fork/github.com/heroku/docker-registry-client/registry/manifest.go @@ -2,6 +2,7 @@ package registry import ( "bytes" + "github.com/containers/image/v5/manifest" "io" "net/http" @@ -49,8 +50,14 @@ func (registry *Registry) ManifestV2(repository, reference string) (*schema2.Des if err != nil { return nil, err } - - req.Header.Set("Accept", schema2.MediaTypeManifest) + headers := map[string][]string{ + "Accept": manifest.DefaultRequestedManifestMIMETypes, + } + for n, h := range headers { + for _, hh := range h { + req.Header.Add(n, hh) + } + } resp, err := registry.Client.Do(req) if err != nil { return nil, err @@ -74,12 +81,21 @@ func (registry *Registry) ManifestDigest(repository, reference string) (digest.D url := registry.url("/v2/%s/manifests/%s", repository, reference) registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference) - resp, err := registry.Client.Head(url) - //req, _ := http.NewRequest("HEAD", url, nil) - //req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json") - //resp, err := registry.Client.Do(req) - if resp != nil { - defer resp.Body.Close() + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return "", err + } + headers := map[string][]string{ + "Accept": manifest.DefaultRequestedManifestMIMETypes, + } + for n, h := range headers { + for _, hh := range h { + req.Header.Add(n, hh) + } + } + resp, err := registry.Client.Do(req) + if err != nil { + return "", err } if err != nil { return "", err diff --git a/fork/github.com/heroku/docker-registry-client/registry/repositories.go b/fork/github.com/heroku/docker-registry-client/registry/repositories.go index 912b40d8e..1be578ae1 100644 --- a/fork/github.com/heroku/docker-registry-client/registry/repositories.go +++ b/fork/github.com/heroku/docker-registry-client/registry/repositories.go @@ -4,7 +4,7 @@ type repositoriesResponse struct { Repositories []string `json:"repositories"` } -func (registry *Registry) Repositories() ([]string, error) { +func (registry *Registry) Repositories(filterFunc func(data []string) []string) ([]string, error) { url := "/v2/_catalog?n=50" repos := make([]string, 0, 10) @@ -12,15 +12,14 @@ func (registry *Registry) Repositories() ([]string, error) { var response repositoriesResponse //var last string for { - url = registry.URL + url registry.Logf("registry.repositories url=%s", url) url, err = registry.getPaginatedJSON(url, &response) switch err { case ErrNoMorePages: - repos = append(repos, response.Repositories...) + repos = append(repos, filterFunc(response.Repositories)...) return repos, nil case nil: - repos = append(repos, response.Repositories...) + repos = append(repos, filterFunc(response.Repositories)...) continue default: return nil, err diff --git a/pkg/registry/cmd/flags.go b/pkg/registry/cmd/flags.go new file mode 100644 index 000000000..126e17794 --- /dev/null +++ b/pkg/registry/cmd/flags.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + "runtime" + + "github.com/pkg/errors" + + "github.com/docker/docker/api/types" + "github.com/spf13/pflag" + + "github.com/labring/sealos/pkg/registry" + "github.com/labring/sealos/pkg/utils/file" +) + +const defaultRegistryName = "sealos.hub:5000" + +type imagesResults struct { + registryName string + filter string + json bool +} + +func (opts *imagesResults) RegisterFlags(fs *pflag.FlagSet) { + fs.SetInterspersed(false) + fs.BoolVar(&opts.json, "json", opts.json, "output in JSON format") + fs.StringVarP(&opts.registryName, "name", "n", defaultRegistryName, "registry name") + fs.StringVar(&opts.filter, "filter", opts.filter, "Filter support 'name' and 'tag' , strategy support prefix (eg key*),suffix(eg *key),equals(eg key),empty(eg ),like(eg *key*)") +} + +type rmiResults struct { + registryName string +} + +func (opts *rmiResults) RegisterFlags(fs *pflag.FlagSet) { + fs.SetInterspersed(false) + fs.StringVarP(&opts.registryName, "name", "n", defaultRegistryName, "registry name") +} + +type registryStatusResults struct { + json bool +} + +func (opts *registryStatusResults) RegisterFlags(fs *pflag.FlagSet) { + fs.SetInterspersed(false) + fs.BoolVar(&opts.json, "json", opts.json, "output in JSON format") +} + +type registrySaveResults struct { + registryPullRegistryDir string + registryPullArch string + registryPullMaxPullProcs int +} + +func (opts *registrySaveResults) RegisterFlags(fs *pflag.FlagSet) { + fs.SetInterspersed(false) + fs.StringVar(&opts.registryPullArch, "arch", runtime.GOARCH, "pull images arch") + fs.StringVar(&opts.registryPullRegistryDir, "data-dir", "/var/lib/registry", "registry data dir path") + fs.IntVar(&opts.registryPullMaxPullProcs, "max-pull-procs", 5, "maximum number of goroutines for pulling") +} + +func (opts *registrySaveResults) CheckAuth() (map[string]types.AuthConfig, error) { + if !file.IsExist(opts.registryPullRegistryDir) { + _ = os.MkdirAll(opts.registryPullRegistryDir, 0755) + } + cfg, err := registry.GetAuthInfo() + if err != nil { + return nil, errors.Wrap(err, "auth info is error") + } + return cfg, nil +} + +type registrySaveRawResults struct { + *registrySaveResults + images []string +} + +func (opts *registrySaveRawResults) RegisterFlags(fs *pflag.FlagSet) { + opts.registrySaveResults.RegisterFlags(fs) + fs.StringSliceVar(&opts.images, "images", []string{}, "images list") +} + +type registrySaveDefaultResults struct { + *registrySaveResults +} + +func (opts *registrySaveDefaultResults) RegisterFlags(fs *pflag.FlagSet) { + opts.registrySaveResults.RegisterFlags(fs) +} diff --git a/pkg/registry/cmd/list.go b/pkg/registry/cmd/list.go index 8ea82d9a8..b4fe3c4a1 100644 --- a/pkg/registry/cmd/list.go +++ b/pkg/registry/cmd/list.go @@ -17,16 +17,17 @@ limitations under the License. package cmd import ( + "fmt" "os" - "github.com/docker/docker/api/types" - "github.com/spf13/cobra" - "github.com/labring/sealos/pkg/registry" "github.com/labring/sealos/pkg/utils/logger" + + "github.com/docker/docker/api/types" + "github.com/spf13/cobra" ) -func NewRegistryListCmd() *cobra.Command { +func newRegistryListImageCmd() *cobra.Command { preValidate := func() map[string]types.AuthConfig { cfg, err := registry.GetAuthInfo() if err != nil { @@ -37,63 +38,32 @@ func NewRegistryListCmd() *cobra.Command { } var auth map[string]types.AuthConfig var is registry.Registry + flagsResults := imagesResults{} var registryImageListCmd = &cobra.Command{ - Use: "list", - Short: "registry list", - Example: "sealctl registry list", + Use: "images", + Short: "registry image list", + Example: fmt.Sprintf(`Example: + %[1]s registry images --filter name=public* + %[1]s registry images --filter tag=*1.1* + %[1]s registry images --filter tag=*sec + %[1]s registry images --filter name=public,tag=v1.1.1 + %[1]s registry images --filter tag=`, rootCmd.CommandPath()), + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - is.ListRegistry() + is.ListImages(flagsResults.registryName, flagsResults.filter, flagsResults.json) return nil }, - PreRun: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { auth = preValidate() is = registry.NewImage(auth) - }, - } - - return registryImageListCmd -} - -func NewRegistryImageCmd() *cobra.Command { - var registryName string - var registryImageCmd = &cobra.Command{ - Use: "image", - Short: "registry images manager", - } - - registryImageCmd.PersistentFlags().StringVarP(®istryName, "name", "n", "sealos.hub:5000", "registry name") - - registryImageCmd.AddCommand(NewRegistryListImageCmd(registryName)) - registryImageCmd.AddCommand(NewRegistryImageRmiCmd(registryName)) - - return registryImageCmd -} - -func NewRegistryListImageCmd(registryName string) *cobra.Command { - preValidate := func() map[string]types.AuthConfig { - cfg, err := registry.GetAuthInfo() - if err != nil { - logger.Error("auth info is error: %+v", err) - os.Exit(1) - } - return cfg - } - var auth map[string]types.AuthConfig - var is registry.Registry - var registryImageListCmd = &cobra.Command{ - Use: "list", - Short: "registry list image", - Example: "sealctl registry image list", - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - is.ListImages(registryName, registryName) + if _, ok := auth[flagsResults.registryName]; !ok { + return fmt.Errorf("not found %s in auth info", flagsResults.registryName) + } return nil }, - PreRun: func(cmd *cobra.Command, args []string) { - auth = preValidate() - is = registry.NewImage(auth) - }, } - + flags := registryImageListCmd.Flags() + flags.SetInterspersed(false) + flagsResults.RegisterFlags(flags) return registryImageListCmd } diff --git a/pkg/registry/cmd/prune.go b/pkg/registry/cmd/prune.go index 5b9d3c3b5..f212ec688 100644 --- a/pkg/registry/cmd/prune.go +++ b/pkg/registry/cmd/prune.go @@ -26,7 +26,7 @@ import ( "github.com/labring/sealos/pkg/utils/exec" ) -func NewRegistryPruneCmd() *cobra.Command { +func newRegistryPruneCmd() *cobra.Command { var configPath string var dryRun, deleteUntagged bool var registryImagePruneCmd = &cobra.Command{ diff --git a/pkg/registry/cmd/registry.go b/pkg/registry/cmd/registry.go new file mode 100644 index 000000000..5d9470279 --- /dev/null +++ b/pkg/registry/cmd/registry.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + + "github.com/containers/buildah/pkg/parse" + "github.com/spf13/cobra" +) + +var ( + rootCmd *cobra.Command +) + +func subCommands() []*cobra.Command { + return []*cobra.Command{ + newRegistryStatusCmd(), + newRegistryListImageCmd(), + newRegistryImageRmiCmd(), + newRegistryPruneCmd(), + newRegistryImageSaveCmd(), + } +} + +func RegisterRootCommand(cmd *cobra.Command, root *cobra.Command) { + rootCmd = root + os.Setenv("TMPDIR", parse.GetTempDir()) + cmd.SilenceUsage = true + cmd.AddCommand(subCommands()...) +} diff --git a/pkg/registry/cmd/rmi.go b/pkg/registry/cmd/rmi.go index b4577d16a..d5b1b3ea3 100644 --- a/pkg/registry/cmd/rmi.go +++ b/pkg/registry/cmd/rmi.go @@ -32,7 +32,7 @@ import ( "github.com/labring/sealos/pkg/utils/logger" ) -func NewRegistryImageRmiCmd(registryName string) *cobra.Command { +func newRegistryImageRmiCmd() *cobra.Command { preValidate := func() map[string]types.AuthConfig { cfg, err := registry.GetAuthInfo() if err != nil { @@ -43,17 +43,18 @@ func NewRegistryImageRmiCmd(registryName string) *cobra.Command { } var auth map[string]types.AuthConfig var is registry.Registry - var registryImageListCmd = &cobra.Command{ + flagsResults := rmiResults{} + var registryImageRMICmd = &cobra.Command{ Use: "rmi", Short: "registry rmi image", - Example: "sealctl registry image rmi labring/lvscare:v4.1.3", + Example: fmt.Sprintf(`%[1]s registry rmi labring/lvscare:v4.1.3`, rootCmd.Root().CommandPath()), Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { imageList := args errorList := field.ErrorList{} for i := range imageList { - imageList[i] = strings.ReplaceAll(imageList[i], fmt.Sprintf("%s/", registryName), "") - err := is.RmiImage(registryName, imageList[i]) + imageList[i] = strings.ReplaceAll(imageList[i], fmt.Sprintf("%s/", flagsResults.registryName), "") + err := is.RmiImage(flagsResults.registryName, imageList[i]) if err != nil { errorList = append(errorList, field.Invalid(field.NewPath("image"), imageList[i], err.Error())) } @@ -63,12 +64,14 @@ func NewRegistryImageRmiCmd(registryName string) *cobra.Command { PreRunE: func(cmd *cobra.Command, args []string) error { auth = preValidate() is = registry.NewImage(auth) - if registryName == "" { + if flagsResults.registryName == "" { return errors.New("registryName not allow empty") } return nil }, } - - return registryImageListCmd + flags := registryImageRMICmd.Flags() + flags.SetInterspersed(false) + flagsResults.RegisterFlags(flags) + return registryImageRMICmd } diff --git a/pkg/registry/cmd/save.go b/pkg/registry/cmd/save.go index 4ff57b4a9..0bf4e0f5a 100644 --- a/pkg/registry/cmd/save.go +++ b/pkg/registry/cmd/save.go @@ -20,98 +20,104 @@ import ( "context" "os" + "github.com/pkg/errors" + "github.com/docker/docker/api/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/labring/sealos/pkg/buildimage" "github.com/labring/sealos/pkg/registry" - "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/logger" ) -func NewSaveCmd(registryPullRegistryDir, registryPullArch string, registryPullMaxPullProcs int, cmdType string) *cobra.Command { - preValidate := func() map[string]types.AuthConfig { - if !file.IsExist(registryPullRegistryDir) { - _ = os.MkdirAll(registryPullRegistryDir, 0755) - } - cfg, err := registry.GetAuthInfo() - if err != nil { - logger.Error("auth info is error: %+v", err) - os.Exit(1) - } - return cfg - } - pullRawCmd := func() *cobra.Command { - var imageFile string - var auth map[string]types.AuthConfig - var registryImagePullRaw = &cobra.Command{ - Use: "raw", - Short: "registry images manager save to local dir by raw type", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - var ( - imageContext = "." - ) - if len(args) != 0 { - imageContext = args[0] - } - images, err := buildimage.List(imageContext) - if err != nil { - logger.Error("get images list is error: %s", err.Error()) - os.Exit(1) - } - - is := registry.NewImageSaver(context.Background(), registryPullMaxPullProcs, auth) - outImages, err := is.SaveImages(images, registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: registryPullArch}) - if err != nil { - logger.Error("pull registry images is error: %s", err.Error()) - os.Exit(1) - } - logger.Info("pull images list save to local : %+v", outImages) - }, - PreRun: func(cmd *cobra.Command, args []string) { - auth = preValidate() - if !file.IsExist(imageFile) { - logger.Error("ImageFile path is not exist") - os.Exit(1) - } - }, - } - - registryImagePullRaw.PersistentFlags().StringVarP(&imageFile, "image-file", "f", "ImageFile", "ImageFile path") - return registryImagePullRaw - } - pullDefaultCmd := func() *cobra.Command { - var images []string - var auth map[string]types.AuthConfig - var registryImagePullDefault = &cobra.Command{ - Use: "default", - Short: "registry images manager pull to local dir by default type", - Run: func(cmd *cobra.Command, args []string) { - is := registry.NewImageSaver(context.Background(), registryPullMaxPullProcs, auth) - outImages, err := is.SaveImages(images, registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: registryPullArch}) - if err != nil { - logger.Error("pull registry images is error: %s", err.Error()) - os.Exit(1) - } - logger.Info("pull images list save to local : %+v", outImages) - }, - PreRun: func(cmd *cobra.Command, args []string) { - auth = preValidate() - }, - } - - registryImagePullDefault.PersistentFlags().StringSliceVar(&images, "images", []string{}, "images list") - return registryImagePullDefault +func newRegistryImageSaveCmd() *cobra.Command { + var registryImagePullCmd = &cobra.Command{ + Use: "save", + Short: "save registry images to local registry dir", } + registryImagePullCmd.AddCommand(saveCmd("default")) + registryImagePullCmd.AddCommand(saveCmd("raw")) + return registryImagePullCmd +} +func saveCmd(cmdType string) *cobra.Command { switch cmdType { case "raw": - return pullRawCmd() + return saveRawCmd() case "default": - return pullDefaultCmd() + return saveDefaultCmd() default: - return pullDefaultCmd() + return saveDefaultCmd() } } +func saveDefaultCmd() *cobra.Command { + var images []string + var auth map[string]types.AuthConfig + flagsResults := registrySaveDefaultResults{ + registrySaveResults: new(registrySaveResults), + } + var registryImagePullDefault = &cobra.Command{ + Use: "default [CONTEXT]", + Short: "registry images manager save to local dir by default type", + Run: func(cmd *cobra.Command, args []string) { + is := registry.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth) + outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch}) + if err != nil { + logger.Error("pull registry images is error: %s", err.Error()) + os.Exit(1) + } + logger.Info("pull images list save to local : %+v", outImages) + }, + Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + var err error + images, err = buildimage.List(args[0]) + if err != nil { + return err + } + auth, err = flagsResults.CheckAuth() + if err != nil { + return err + } + return nil + }, + } + flags := registryImagePullDefault.Flags() + flags.SetInterspersed(false) + flagsResults.RegisterFlags(flags) + return registryImagePullDefault +} + +func saveRawCmd() *cobra.Command { + var auth map[string]types.AuthConfig + flagsResults := registrySaveRawResults{ + registrySaveResults: new(registrySaveResults), + } + var registryImagePullRaw = &cobra.Command{ + Use: "raw", + Short: "registry images manager save to local dir by raw type", + RunE: func(cmd *cobra.Command, args []string) error { + is := registry.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth) + outImages, err := is.SaveImages(flagsResults.images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch}) + if err != nil { + return errors.Wrap(err, "pull registry images is error") + } + logger.Info("pull images list save to local : %+v", outImages) + return nil + }, + PreRunE: func(cmd *cobra.Command, args []string) error { + var err error + auth, err = flagsResults.CheckAuth() + if err != nil { + return err + } + return nil + }, + } + flags := registryImagePullRaw.Flags() + flags.SetInterspersed(false) + flagsResults.RegisterFlags(flags) + _ = registryImagePullRaw.MarkFlagRequired("images") + return registryImagePullRaw +} diff --git a/pkg/registry/cmd/status.go b/pkg/registry/cmd/status.go new file mode 100644 index 000000000..1e3926ba9 --- /dev/null +++ b/pkg/registry/cmd/status.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "os" + + "github.com/docker/docker/api/types" + "github.com/spf13/cobra" + + "github.com/labring/sealos/pkg/registry" + "github.com/labring/sealos/pkg/utils/logger" +) + +func newRegistryStatusCmd() *cobra.Command { + preValidate := func() map[string]types.AuthConfig { + cfg, err := registry.GetAuthInfo() + if err != nil { + logger.Error("auth info is error: %+v", err) + os.Exit(1) + } + return cfg + } + var auth map[string]types.AuthConfig + var is registry.Registry + flagsResults := registryStatusResults{} + var registryStatusCmd = &cobra.Command{ + Use: "status", + Short: "registry status", + Example: fmt.Sprintf(`%[1]s registry status`, rootCmd.Root().CommandPath()), + RunE: func(cmd *cobra.Command, args []string) error { + is.Status(flagsResults.json) + return nil + }, + PreRun: func(cmd *cobra.Command, args []string) { + auth = preValidate() + is = registry.NewImage(auth) + }, + } + flags := registryStatusCmd.Flags() + flags.SetInterspersed(false) + flagsResults.RegisterFlags(flags) + return registryStatusCmd +} diff --git a/pkg/registry/filter.go b/pkg/registry/filter.go new file mode 100644 index 000000000..d5ea744a5 --- /dev/null +++ b/pkg/registry/filter.go @@ -0,0 +1,180 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "errors" + "fmt" + "strings" + + "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/maps" + strings2 "github.com/labring/sealos/pkg/utils/strings" +) + +const none = "" + +type FilterType string + +const ( + FilterTypeName = "name" + FilterTypeTag = "tag" +) + +type FilterStrategy string + +const ( + FilterStrategyNone = "none" + FilterStrategyPrefix = "prefix" + FilterStrategySuffix = "suffix" + FilterStrategyDefault = "default" + FilterStrategyEquals = "equals" + FilterStrategyAll = "all" + FilterStrategyUnknown = "unknown" +) + +type Filter struct { + Name string + Tag string + nameStrategy FilterStrategy + tagStrategy FilterStrategy +} + +func (f *Filter) Validate() error { + check := func(filter string, t FilterType) (FilterStrategy, error) { + if filter == none { + if t == FilterTypeName { + return FilterStrategyUnknown, errors.New("your filter repo not allow is none") + } + return FilterStrategyNone, nil + } + + if filter == "" { + return FilterStrategyAll, nil + } + if len(filter) == 1 && filter == "*" { + return FilterStrategyAll, nil + } + if strings.Contains(filter, "*") { + if filter[0] == '*' && filter[len(filter)-1] == '*' { + return FilterStrategyDefault, nil + } + if filter[0] == '*' { + if strings.LastIndex(filter, "*") == 0 { + return FilterStrategySuffix, nil + } + return FilterStrategyUnknown, fmt.Errorf("your filter must has one char '*' , example *ccc ") + } + if filter[len(filter)-1] == '*' { + if strings.LastIndex(filter, "*") == len(filter)-1 { + return FilterStrategyPrefix, nil + } + return FilterStrategyUnknown, fmt.Errorf("your filter must has one char '*' , example ccc* ") + } + return FilterStrategyUnknown, fmt.Errorf("not spport char '*' in filter middle") + } + return FilterStrategyEquals, nil + } + + errorlist := make([]error, 0) + strategy, err := check(f.Name, FilterTypeName) + if err != nil { + errorlist = append(errorlist, err) + } + f.nameStrategy = strategy + + strategy, err = check(f.Tag, FilterTypeTag) + if err != nil { + errorlist = append(errorlist, err) + } + f.tagStrategy = strategy + + if len(errorlist) == 0 { + return nil + } + + return fmt.Errorf("your filter has some errors: %+v", errorlist) +} + +func (f *Filter) Run(data []string, t FilterType) []string { + if t == FilterTypeName { + return filter(data, f.Name, f.nameStrategy) + } + return filter(data, f.Tag, f.tagStrategy) +} + +func newFilter(filter string) *Filter { + filterMap := maps.StringToMap(filter, ",") + for key := range filterMap { + if !strings2.InList(key, []string{FilterTypeName, FilterTypeTag}) { + logger.Warn("filter key not support: %s", key) + } + } + return &Filter{ + Name: filterMap[FilterTypeName], + Tag: filterMap[FilterTypeTag], + } +} + +func filter(data []string, filter string, strategy FilterStrategy) []string { + prefix := func(data []string, filter string) []string { + res := make([]string, 0) + for _, d := range data { + if strings.HasPrefix(d, filter) { + res = append(res, d) + } + } + return res + } + + suffix := func(data []string, filter string) []string { + res := make([]string, 0) + for _, d := range data { + if strings.HasSuffix(d, filter) { + res = append(res, d) + } + } + return res + } + + contains := func(data []string, filter string) []string { + res := make([]string, 0) + for _, d := range data { + if strings.Contains(d, filter) { + res = append(res, d) + } + } + return res + } + + switch strategy { + case FilterStrategyDefault: + return contains(data, filter[1:len(filter)-1]) + case FilterStrategyPrefix: + return prefix(data, filter[:len(filter)-1]) + case FilterStrategySuffix: + return suffix(data, filter[1:]) + case FilterStrategyNone: + return []string{} + case FilterStrategyAll: + return data + case FilterStrategyEquals: + return []string{filter} + default: + } + return []string{} +} diff --git a/pkg/registry/filter_test.go b/pkg/registry/filter_test.go new file mode 100644 index 000000000..2a8283c43 --- /dev/null +++ b/pkg/registry/filter_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "reflect" + "testing" +) + +func TestFilter_Validate(t *testing.T) { + tests := []struct { + name string + fields *Filter + wantErr bool + }{ + { + name: "default", + fields: newFilter(""), + wantErr: false, + }, + { + name: "name", + fields: newFilter("name="), + wantErr: true, + }, + { + name: "name-prefix", + fields: newFilter("name=aa*"), + wantErr: false, + }, + { + name: "name-suffix", + fields: newFilter("name=*cccc"), + wantErr: false, + }, + { + name: "name-error", + fields: newFilter("name=ccc*cccc"), + wantErr: true, + }, + { + name: "name-eque", + fields: newFilter("name=cccc"), + wantErr: false, + }, + { + name: "name-default1", + fields: newFilter("name=*"), + wantErr: false, + }, + { + name: "name-default2", + fields: newFilter("name="), + wantErr: false, + }, + + { + name: "tag", + fields: newFilter("tag="), + wantErr: false, + }, + { + name: "tag-prefix", + fields: newFilter("tag=aa*"), + wantErr: false, + }, + { + name: "tag-suffix", + fields: newFilter("tag=*cccc"), + wantErr: false, + }, + { + name: "tag-error", + fields: newFilter("tag=ccc*cccc"), + wantErr: true, + }, + { + name: "tag-default1", + fields: newFilter("tag=*"), + wantErr: false, + }, + { + name: "tag-default2", + fields: newFilter("tag="), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := tt.fields + if err := f.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestFilter_Run(t *testing.T) { + type args struct { + data []string + t FilterType + } + tests := []struct { + name string + fields *Filter + args args + want []string + }{ + { + name: "default", + fields: newFilter("name=a"), + args: args{ + data: []string{"a", "b", "c"}, + t: FilterTypeName, + }, + want: []string{"a"}, + }, + { + name: "default", + fields: newFilter("name=c*"), + args: args{ + data: []string{"abcc", "bdcff", "c"}, + t: FilterTypeName, + }, + want: []string{"c"}, + }, + { + name: "default", + fields: newFilter("name=*labring*"), + args: args{ + data: []string{"labring/lvscare"}, + t: FilterTypeName, + }, + want: []string{"labring/lvscare"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := tt.fields + if err := f.Validate(); err != nil { + t.Error(err) + return + } + if got := f.Run(tt.args.data, tt.args.t); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Run() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/registry/interface.go b/pkg/registry/interface.go index 8d68cbb43..ed67e7aea 100644 --- a/pkg/registry/interface.go +++ b/pkg/registry/interface.go @@ -27,8 +27,8 @@ import ( type Registry interface { // SaveImages is not concurrently safe SaveImages(images []string, dir string, platform v1.Platform) ([]string, error) - ListRegistry() - ListImages(registryName, search string) + Status(json bool) + ListImages(registryName, search string, json bool) RmiImage(registryName, imageName string) error } diff --git a/pkg/registry/list.go b/pkg/registry/list.go index 94f085c78..b489a9bbe 100644 --- a/pkg/registry/list.go +++ b/pkg/registry/list.go @@ -17,14 +17,18 @@ limitations under the License. package registry import ( + "fmt" + "github.com/modood/table" "github.com/opencontainers/go-digest" + "k8s.io/apimachinery/pkg/util/json" "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/utils/registry" ) -func (is *DefaultImage) ListImages(registryName string, search string) { +func (is *DefaultImage) ListImages(registryName, search string, enableJSON bool) { + logger.Debug("param registryName: %s, filter: %s", registryName, search) authInfo := is.auths[registryName] var listImage []imageOutputParams reg, err := registry.NewRegistryForDomain(registryName, authInfo.Username, authInfo.Password) @@ -32,60 +36,76 @@ func (is *DefaultImage) ListImages(registryName string, search string) { logger.Error("Failed to list image : %v", err) return } - repos, _ := reg.Repositories() + filter := newFilter(search) + if err = filter.Validate(); err != nil { + logger.Error("Failed to list image using filter : %v", err) + return + } + + logger.Debug("filter name=%s,strategy=%s", filter.Name, filter.nameStrategy) + logger.Debug("filter tag=%s,strategy=%s", filter.Tag, filter.tagStrategy) + + var repos []string + if filter.nameStrategy == FilterStrategyEquals { + repos = []string{filter.Name} + } else { + repos, err = reg.Repositories(func(data []string) []string { + return filter.Run(data, FilterTypeName) + }) + if err != nil { + logger.Error("list image is error: %+v", err) + return + } + } + + var imageVersionList bool + defer func() { + if !enableJSON { + logger.Info("Image count %d", len(repos)) + if imageVersionList { + logger.Info("Images Version count %d", len(listImage)) + } + } + }() for _, repo := range repos { tags, _ := reg.Tags(repo) - for _, tag := range tags { - var imageID digest.Digest - imageID, _ = reg.ManifestDigest(repo, tag) + tags = filter.Run(tags, FilterTypeTag) + if len(tags) == 0 { listImage = append(listImage, imageOutputParams{ RegistryName: registryName, - Repository: repo, - Tag: tag, - ImageID: imageID.String(), - }) - } - } - table.OutputA(listImage) - logger.Info("Images count %d", len(listImage)) -} - -func (is *DefaultImage) ListRegistry() { - var listRegistry []registryOutputParams - for domain, auth := range is.auths { - reg, err := registry.NewRegistryForDomain(domain, auth.Username, auth.Password) - if err != nil { - listRegistry = append(listRegistry, registryOutputParams{ - Name: domain, - URL: "unknow://" + domain, - UserName: auth.Username, - Password: auth.Password, - Healthy: "failed", + ImageName: repo, + Tag: "", + ImageID: "", }) } else { - listRegistry = append(listRegistry, registryOutputParams{ - Name: domain, - URL: reg.URL, - UserName: auth.Username, - Password: auth.Password, - Healthy: "ok", - }) + imageVersionList = true + for _, tag := range tags { + var imageID digest.Digest + imageID, _ = reg.ManifestDigest(repo, tag) + listImage = append(listImage, imageOutputParams{ + RegistryName: registryName, + ImageName: repo, + Tag: tag, + ImageID: imageID.String(), + }) + } } } - table.OutputA(listRegistry) -} - -type registryOutputParams struct { - Name string - URL string - UserName string - Password string - Healthy string + if enableJSON { + marshalled, err := json.Marshal(listImage) + if err != nil { + logger.Error("Failed to Marshal Json : %v", err) + return + } + fmt.Println(string(marshalled)) + return + } + table.OutputA(listImage) } type imageOutputParams struct { RegistryName string - Repository string + ImageName string Tag string ImageID string } diff --git a/pkg/registry/list_test.go b/pkg/registry/list_test.go index 4fe7f23e0..a49de25c4 100644 --- a/pkg/registry/list_test.go +++ b/pkg/registry/list_test.go @@ -47,7 +47,7 @@ func TestDefaultImage_ListRegistry(t *testing.T) { is := &DefaultImage{ auths: tt.fields.auths, } - is.ListRegistry() + is.Status(false) }) } } @@ -84,7 +84,7 @@ func TestDefaultImage_ListImages(t *testing.T) { is := &DefaultImage{ auths: tt.fields.auths, } - is.ListImages(tt.args.registryName, "") + is.ListImages(tt.args.registryName, "name=*,tag=", false) }) } } diff --git a/pkg/registry/save.go b/pkg/registry/save.go index 4808d7bb3..12271a417 100644 --- a/pkg/registry/save.go +++ b/pkg/registry/save.go @@ -58,6 +58,7 @@ const ( ) func (is *DefaultImage) SaveImages(images []string, dir string, platform v1.Platform) ([]string, error) { + logger.Debug("search images platform: %s , dir: %s, image list: %+v", strings.Join([]string{platform.OS, platform.Architecture, platform.Variant}, ","), dir, images) //init a pipe for display pull message reader, writer := io.Pipe() defer func() { diff --git a/pkg/registry/status.go b/pkg/registry/status.go new file mode 100644 index 000000000..fba3b3d88 --- /dev/null +++ b/pkg/registry/status.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + + "github.com/modood/table" + "k8s.io/apimachinery/pkg/util/json" + + "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/registry" +) + +func (is *DefaultImage) Status(enableJSON bool) { + var listRegistry []registryOutputParams + if len(is.auths) == 0 && !enableJSON { + logger.Warn("your registry not login") + } + for domain, auth := range is.auths { + reg, err := registry.NewRegistryForDomain(domain, auth.Username, auth.Password) + if err != nil { + listRegistry = append(listRegistry, registryOutputParams{ + Name: domain, + URL: "unknow://" + domain, + UserName: auth.Username, + Password: auth.Password, + Healthy: "failed", + }) + } else { + listRegistry = append(listRegistry, registryOutputParams{ + Name: domain, + URL: reg.URL, + UserName: auth.Username, + Password: auth.Password, + Healthy: "ok", + }) + } + } + if enableJSON { + marshalled, err := json.Marshal(listRegistry) + if err != nil { + logger.Error("Failed to Marshal Json : %v", err) + return + } + fmt.Println(string(marshalled)) + return + } + table.OutputA(listRegistry) +} + +type registryOutputParams struct { + Name string + URL string + UserName string + Password string + Healthy string +}