opensource/sealos
3
0
Fork 0
mirror of https://github.com/labring/sealos.git synced 2026-08-30 17:58:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature(main): fix registry find manifest logic (#2314)

Browse Source 
* feature(main): fix registry find manifest logic

Signed-off-by: cuisongliu <cuisongliu@qq.com>
This commit is contained in:
cuisongliu
2023-01-03 23:14:06 +08:00
committed by GitHub
parent 506ccc32ef
commit 3b626a0727
19 changed files with 1073 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/sealctl/cmd/registry.go
+4 -29
View File
@@ -17,45 +17,20 @@ limitations under the License.
package cmd
import (
"runtime"
"github.com/spf13/cobra"
"github.com/labring/sealos/pkg/registry/cmd"
)
var (
registryPullRegistryDir string
registryPullArch string
registryPullMaxPullProcs int
)
func init() {
rootCmd.AddCommand(newRegistryImageCmd())
}
func newRegistryImageCmd() *cobra.Command {
var registryImageCmd = &cobra.Command{
Use: "registry",
Short: "registry images manager",
}
registryImageCmd.AddCommand(cmd.NewRegistryListCmd())
registryImageCmd.AddCommand(cmd.NewRegistryImageCmd())
registryImageCmd.AddCommand(cmd.NewRegistryPruneCmd())
registryImageCmd.AddCommand(newRegistryImageSaveCmd())
cmd.RegisterRootCommand(registryImageCmd, rootCmd)
return registryImageCmd
}
func newRegistryImageSaveCmd() *cobra.Command {
var registryImagePullCmd = &cobra.Command{
Use: "save",
Short: "save registry images to local registry dir",
}
registryImagePullCmd.PersistentFlags().StringVar(&registryPullArch, "arch", runtime.GOARCH, "pull images arch")
registryImagePullCmd.PersistentFlags().StringVar(&registryPullRegistryDir, "data-dir", "/var/lib/registry", "registry data dir path")
registryImagePullCmd.PersistentFlags().IntVar(&registryPullMaxPullProcs, "max-pull-procs", 5, "maximum number of goroutines for pulling")
registryImagePullCmd.AddCommand(cmd.NewSaveCmd(registryPullRegistryDir, registryPullArch, registryPullMaxPullProcs, "default"))
registryImagePullCmd.AddCommand(cmd.NewSaveCmd(registryPullRegistryDir, registryPullArch, registryPullMaxPullProcs, "raw"))
return registryImagePullCmd
}
func init() {
rootCmd.AddCommand(newRegistryImageCmd())
}
docs/4.0/docs/examples/sealctl-registry.md
+227
View File
@@ -0,0 +1,227 @@
# Using sealctl to manage registry images
## Save images to local registry directory
### default type
The default mode scans images in the context directory. The logic of scanning images is the logic of build images, and the directories of charts, manifests, and images scan images.
```
registry images manager save to local dir by default type
Usage:
sealctl registry save default [CONTEXT] [flags]
Flags:
--arch string pull images arch (default "arm64")
--data-dir string registry data dir path (default "/var/lib/registry")
-h, --help help for default
--max-pull-procs int maximum number of goroutines for pulling (default 5)
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
sealctl registry save default .
```
### raw type
raw type using images as image list.
```
registry images manager save to local dir by raw type
Usage:
sealctl registry save raw [flags]
Flags:
--arch string pull images arch (default "arm64")
--data-dir string registry data dir path (default "/var/lib/registry")
-h, --help help for raw
--images strings images list
--max-pull-procs int maximum number of goroutines for pulling (default 5)
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
sealctl registry save raw --images nginx
```
## Check registry status
To use this command, you need to `sealctl login` or `sealos login` first.
```
registry status
Usage:
sealctl registry status [flags]
Examples:
sealctl registry status
Flags:
-h, --help help for status
--json output in JSON format
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
$ sealctl registry status
+--------------------+---------------------------+----------+----------+---------+
| Name | URL | UserName | Password | Healthy |
+--------------------+---------------------------+----------+----------+---------+
| 192.168.64.63:5000 | http://192.168.64.63:5000 | admin | passw0rd | ok |
+--------------------+---------------------------+----------+----------+---------+
```
## List registry image
```
registry image list
Usage:
sealctl registry images [flags]
Examples:
Example:
sealctl registry images --filter name=public*
sealctl registry images --filter tag=*1.1*
sealctl registry images --filter tag=*sec
sealctl registry images --filter name=public,tag=v1.1.1
sealctl registry images --filter tag=<none>
Flags:
--filter string Filter support 'name' and 'tag' , strategy support prefix (eg key*),suffix(eg *key),equals(eg key),empty(eg <none>),like(eg *key*)
-h, --help help for images
--json output in JSON format
-n, --name string registry name (default "sealos.hub:5000")
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
$ sealctl registry images -n 192.168.64.63:5000 --filter name=kube*,tag=*
+--------------------+-------------------------+---------+-------------------------------------------------------------------------+
| RegistryName | ImageName | Tag | ImageID |
+--------------------+-------------------------+---------+-------------------------------------------------------------------------+
| 192.168.64.63:5000 | kube-apiserver | v1.19.0 | sha256:522d17d35a8994637d27d1232bebd35cfae8e3e21ab359431403f2b8023e332c |
| 192.168.64.63:5000 | kube-controller-manager | v1.19.0 | sha256:6c11a3d4d06385f7547a5ea0c3f0d5e7b12baa558111d01406ac1f778cb3f00b |
| 192.168.64.63:5000 | kube-proxy | v1.19.0 | sha256:c752ecbd04bc4517168a19323bb60fb45324eee1e480b2b97d3fd6ea0a54f42d |
| 192.168.64.63:5000 | kube-scheduler | v1.19.0 | sha256:529a1566960a5b3024f2c94128e1cbd882ca1804f222ec5de99b25567858ecb9 |
+--------------------+-------------------------+---------+-------------------------------------------------------------------------+
2023-01-01T12:31:49 info Image count 4
2023-01-01T12:31:49 info Images Version count 4
```
### Filtering
Keys for `--filter`:
- `name`
- `tag`
Expressions:
- `*val`: suffix
- `val*`: prefix
- `value`: equals
- `<none>`: empty
- `*val*`: contains
## Remove registry image
```
registry rmi image
Usage:
sealctl registry rmi [flags]
Examples:
sealctl registry rmi labring/lvscare:v4.1.3
Flags:
-h, --help help for rmi
-n, --name string registry name (default "sealos.hub:5000")
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
$ sealctl registry rmi 192.168.64.63:5000/kube-apiserver:v1.19.0 -n 192.168.64.63:5000
```
## Prune registry
This command can only be run on a registry node, which used the binary deploy mode. Does not support nodes using container deploy mode.
```
registry `garbage-collect` deletes layers not referenced by any manifests
Usage:
sealctl registry prune [flags]
Flags:
-c, --config string registry config path (default "/etc/registry/registry_config.yml")
-u, --delete-untagged delete manifests that are not currently referenced via tag
-d, --dry-run do everything except remove the blobs
-h, --help help for prune
Global Flags:
--debug enable debug logger
--root string storage root dir (default "/var/lib/containers/storage")
--runroot string storage state dir (default "/run/containers/storage")
--show-path enable show code path
--storage-driver string storage-driver (default "overlay")
--storage-opt strings storage driver option
```
#### Usage
```shell
$ sealctl registry prune
```
fork/github.com/heroku/docker-registry-client/registry/json.go
+4 -6
View File
@@ -3,10 +3,9 @@ package registry
import (
"encoding/json"
"errors"
"fmt"
http2 "github.com/labring/sealos/pkg/utils/http"
"net/http"
"regexp"
"strings"
)
var (
@@ -17,6 +16,9 @@ var (
// next page URL while updating pointed-to variable with a parsed JSON
// value. When there are no more pages it returns `ErrNoMorePages`.
func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) {
if _, ok := http2.IsURL(url); !ok {
url = registry.URL + url
}
resp, err := registry.Client.Get(url)
if err != nil {
return "", err
@@ -45,10 +47,6 @@ func getNextLink(resp *http.Response) (string, error) {
for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
parts := nextLinkRE.FindStringSubmatch(link)
if parts != nil {
// support 2.7+ distribution
if strings.HasPrefix(parts[1], "/") {
return fmt.Sprintf("%s://%s%s", resp.Request.URL.Scheme, resp.Request.URL.Host, parts[1]), nil
}
return parts[1], nil
}
}
fork/github.com/heroku/docker-registry-client/registry/manifest.go
+24 -8
View File
@@ -2,6 +2,7 @@ package registry
import (
"bytes"
"github.com/containers/image/v5/manifest"
"io"
"net/http"
@@ -49,8 +50,14 @@ func (registry *Registry) ManifestV2(repository, reference string) (*schema2.Des
if err != nil {
return nil, err
}
req.Header.Set("Accept", schema2.MediaTypeManifest)
headers := map[string][]string{
"Accept": manifest.DefaultRequestedManifestMIMETypes,
}
for n, h := range headers {
for _, hh := range h {
req.Header.Add(n, hh)
}
}
resp, err := registry.Client.Do(req)
if err != nil {
return nil, err
@@ -74,12 +81,21 @@ func (registry *Registry) ManifestDigest(repository, reference string) (digest.D
url := registry.url("/v2/%s/manifests/%s", repository, reference)
registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
resp, err := registry.Client.Head(url)
//req, _ := http.NewRequest("HEAD", url, nil)
//req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
//resp, err := registry.Client.Do(req)
if resp != nil {
defer resp.Body.Close()
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return "", err
}
headers := map[string][]string{
"Accept": manifest.DefaultRequestedManifestMIMETypes,
}
for n, h := range headers {
for _, hh := range h {
req.Header.Add(n, hh)
}
}
resp, err := registry.Client.Do(req)
if err != nil {
return "", err
}
if err != nil {
return "", err
fork/github.com/heroku/docker-registry-client/registry/repositories.go
+3 -4
View File
@@ -4,7 +4,7 @@ type repositoriesResponse struct {
Repositories []string `json:"repositories"`
}
func (registry *Registry) Repositories() ([]string, error) {
func (registry *Registry) Repositories(filterFunc func(data []string) []string) ([]string, error) {
url := "/v2/_catalog?n=50"
repos := make([]string, 0, 10)
@@ -12,15 +12,14 @@ func (registry *Registry) Repositories() ([]string, error) {
var response repositoriesResponse
//var last string
for {
url = registry.URL + url
registry.Logf("registry.repositories url=%s", url)
url, err = registry.getPaginatedJSON(url, &response)
switch err {
case ErrNoMorePages:
repos = append(repos, response.Repositories...)
repos = append(repos, filterFunc(response.Repositories)...)
return repos, nil
case nil:
repos = append(repos, response.Repositories...)
repos = append(repos, filterFunc(response.Repositories)...)
continue
default:
return nil, err
pkg/registry/cmd/flags.go
+105
View File
@@ -0,0 +1,105 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"runtime"
"github.com/pkg/errors"
"github.com/docker/docker/api/types"
"github.com/spf13/pflag"
"github.com/labring/sealos/pkg/registry"
"github.com/labring/sealos/pkg/utils/file"
)
const defaultRegistryName = "sealos.hub:5000"
type imagesResults struct {
registryName string
filter string
json bool
}
func (opts *imagesResults) RegisterFlags(fs *pflag.FlagSet) {
fs.SetInterspersed(false)
fs.BoolVar(&opts.json, "json", opts.json, "output in JSON format")
fs.StringVarP(&opts.registryName, "name", "n", defaultRegistryName, "registry name")
fs.StringVar(&opts.filter, "filter", opts.filter, "Filter support 'name' and 'tag' , strategy support prefix (eg key*),suffix(eg *key),equals(eg key),empty(eg <none>),like(eg *key*)")
}
type rmiResults struct {
registryName string
}
func (opts *rmiResults) RegisterFlags(fs *pflag.FlagSet) {
fs.SetInterspersed(false)
fs.StringVarP(&opts.registryName, "name", "n", defaultRegistryName, "registry name")
}
type registryStatusResults struct {
json bool
}
func (opts *registryStatusResults) RegisterFlags(fs *pflag.FlagSet) {
fs.SetInterspersed(false)
fs.BoolVar(&opts.json, "json", opts.json, "output in JSON format")
}
type registrySaveResults struct {
registryPullRegistryDir string
registryPullArch string
registryPullMaxPullProcs int
}
func (opts *registrySaveResults) RegisterFlags(fs *pflag.FlagSet) {
fs.SetInterspersed(false)
fs.StringVar(&opts.registryPullArch, "arch", runtime.GOARCH, "pull images arch")
fs.StringVar(&opts.registryPullRegistryDir, "data-dir", "/var/lib/registry", "registry data dir path")
fs.IntVar(&opts.registryPullMaxPullProcs, "max-pull-procs", 5, "maximum number of goroutines for pulling")
}
func (opts *registrySaveResults) CheckAuth() (map[string]types.AuthConfig, error) {
if !file.IsExist(opts.registryPullRegistryDir) {
_ = os.MkdirAll(opts.registryPullRegistryDir, 0755)
}
cfg, err := registry.GetAuthInfo()
if err != nil {
return nil, errors.Wrap(err, "auth info is error")
}
return cfg, nil
}
type registrySaveRawResults struct {
*registrySaveResults
images []string
}
func (opts *registrySaveRawResults) RegisterFlags(fs *pflag.FlagSet) {
opts.registrySaveResults.RegisterFlags(fs)
fs.StringSliceVar(&opts.images, "images", []string{}, "images list")
}
type registrySaveDefaultResults struct {
*registrySaveResults
}
func (opts *registrySaveDefaultResults) RegisterFlags(fs *pflag.FlagSet) {
opts.registrySaveResults.RegisterFlags(fs)
}
pkg/registry/cmd/list.go
+23 -53
View File
@@ -17,16 +17,17 @@ limitations under the License.
package cmd
import (
"fmt"
"os"
"github.com/docker/docker/api/types"
"github.com/spf13/cobra"
"github.com/labring/sealos/pkg/registry"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/docker/docker/api/types"
"github.com/spf13/cobra"
)
func NewRegistryListCmd() *cobra.Command {
func newRegistryListImageCmd() *cobra.Command {
preValidate := func() map[string]types.AuthConfig {
cfg, err := registry.GetAuthInfo()
if err != nil {
@@ -37,63 +38,32 @@ func NewRegistryListCmd() *cobra.Command {
}
var auth map[string]types.AuthConfig
var is registry.Registry
flagsResults := imagesResults{}
var registryImageListCmd = &cobra.Command{
Use: "list",
Short: "registry list",
Example: "sealctl registry list",
Use: "images",
Short: "registry image list",
Example: fmt.Sprintf(`Example:
%[1]s registry images --filter name=public*
%[1]s registry images --filter tag=*1.1*
%[1]s registry images --filter tag=*sec
%[1]s registry images --filter name=public,tag=v1.1.1
%[1]s registry images --filter tag=<none>`, rootCmd.CommandPath()),
Args: cobra.ExactArgs(0),
RunE: func(cmd *cobra.Command, args []string) error {
is.ListRegistry()
is.ListImages(flagsResults.registryName, flagsResults.filter, flagsResults.json)
return nil
},
PreRun: func(cmd *cobra.Command, args []string) {
PreRunE: func(cmd *cobra.Command, args []string) error {
auth = preValidate()
is = registry.NewImage(auth)
},
}
return registryImageListCmd
}
func NewRegistryImageCmd() *cobra.Command {
var registryName string
var registryImageCmd = &cobra.Command{
Use: "image",
Short: "registry images manager",
}
registryImageCmd.PersistentFlags().StringVarP(&registryName, "name", "n", "sealos.hub:5000", "registry name")
registryImageCmd.AddCommand(NewRegistryListImageCmd(registryName))
registryImageCmd.AddCommand(NewRegistryImageRmiCmd(registryName))
return registryImageCmd
}
func NewRegistryListImageCmd(registryName string) *cobra.Command {
preValidate := func() map[string]types.AuthConfig {
cfg, err := registry.GetAuthInfo()
if err != nil {
logger.Error("auth info is error: %+v", err)
os.Exit(1)
}
return cfg
}
var auth map[string]types.AuthConfig
var is registry.Registry
var registryImageListCmd = &cobra.Command{
Use: "list",
Short: "registry list image",
Example: "sealctl registry image list",
Args: cobra.ExactArgs(0),
RunE: func(cmd *cobra.Command, args []string) error {
is.ListImages(registryName, registryName)
if _, ok := auth[flagsResults.registryName]; !ok {
return fmt.Errorf("not found %s in auth info", flagsResults.registryName)
}
return nil
},
PreRun: func(cmd *cobra.Command, args []string) {
auth = preValidate()
is = registry.NewImage(auth)
},
}
flags := registryImageListCmd.Flags()
flags.SetInterspersed(false)
flagsResults.RegisterFlags(flags)
return registryImageListCmd
}
pkg/registry/cmd/prune.go
+1 -1
View File
@@ -26,7 +26,7 @@ import (
"github.com/labring/sealos/pkg/utils/exec"
)
func NewRegistryPruneCmd() *cobra.Command {
func newRegistryPruneCmd() *cobra.Command {
var configPath string
var dryRun, deleteUntagged bool
var registryImagePruneCmd = &cobra.Command{
pkg/registry/cmd/registry.go
+45
View File
@@ -0,0 +1,45 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"os"
"github.com/containers/buildah/pkg/parse"
"github.com/spf13/cobra"
)
var (
rootCmd *cobra.Command
)
func subCommands() []*cobra.Command {
return []*cobra.Command{
newRegistryStatusCmd(),
newRegistryListImageCmd(),
newRegistryImageRmiCmd(),
newRegistryPruneCmd(),
newRegistryImageSaveCmd(),
}
}
func RegisterRootCommand(cmd *cobra.Command, root *cobra.Command) {
rootCmd = root
os.Setenv("TMPDIR", parse.GetTempDir())
cmd.SilenceUsage = true
cmd.AddCommand(subCommands()...)
}
pkg/registry/cmd/rmi.go
+11 -8
View File
@@ -32,7 +32,7 @@ import (
"github.com/labring/sealos/pkg/utils/logger"
)
func NewRegistryImageRmiCmd(registryName string) *cobra.Command {
func newRegistryImageRmiCmd() *cobra.Command {
preValidate := func() map[string]types.AuthConfig {
cfg, err := registry.GetAuthInfo()
if err != nil {
@@ -43,17 +43,18 @@ func NewRegistryImageRmiCmd(registryName string) *cobra.Command {
}
var auth map[string]types.AuthConfig
var is registry.Registry
var registryImageListCmd = &cobra.Command{
flagsResults := rmiResults{}
var registryImageRMICmd = &cobra.Command{
Use: "rmi",
Short: "registry rmi image",
Example: "sealctl registry image rmi labring/lvscare:v4.1.3",
Example: fmt.Sprintf(`%[1]s registry rmi labring/lvscare:v4.1.3`, rootCmd.Root().CommandPath()),
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
imageList := args
errorList := field.ErrorList{}
for i := range imageList {
imageList[i] = strings.ReplaceAll(imageList[i], fmt.Sprintf("%s/", registryName), "")
err := is.RmiImage(registryName, imageList[i])
imageList[i] = strings.ReplaceAll(imageList[i], fmt.Sprintf("%s/", flagsResults.registryName), "")
err := is.RmiImage(flagsResults.registryName, imageList[i])
if err != nil {
errorList = append(errorList, field.Invalid(field.NewPath("image"), imageList[i], err.Error()))
}
@@ -63,12 +64,14 @@ func NewRegistryImageRmiCmd(registryName string) *cobra.Command {
PreRunE: func(cmd *cobra.Command, args []string) error {
auth = preValidate()
is = registry.NewImage(auth)
if registryName == "" {
if flagsResults.registryName == "" {
return errors.New("registryName not allow empty")
}
return nil
},
}
return registryImageListCmd
flags := registryImageRMICmd.Flags()
flags.SetInterspersed(false)
flagsResults.RegisterFlags(flags)
return registryImageRMICmd
}
pkg/registry/cmd/save.go
+84 -78
View File
@@ -20,98 +20,104 @@ import (
"context"
"os"
"github.com/pkg/errors"
"github.com/docker/docker/api/types"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"github.com/labring/sealos/pkg/buildimage"
"github.com/labring/sealos/pkg/registry"
"github.com/labring/sealos/pkg/utils/file"
"github.com/labring/sealos/pkg/utils/logger"
)
func NewSaveCmd(registryPullRegistryDir, registryPullArch string, registryPullMaxPullProcs int, cmdType string) *cobra.Command {
preValidate := func() map[string]types.AuthConfig {
if !file.IsExist(registryPullRegistryDir) {
_ = os.MkdirAll(registryPullRegistryDir, 0755)
}
cfg, err := registry.GetAuthInfo()
if err != nil {
logger.Error("auth info is error: %+v", err)
os.Exit(1)
}
return cfg
}
pullRawCmd := func() *cobra.Command {
var imageFile string
var auth map[string]types.AuthConfig
var registryImagePullRaw = &cobra.Command{
Use: "raw",
Short: "registry images manager save to local dir by raw type",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var (
imageContext = "."
)
if len(args) != 0 {
imageContext = args[0]
}
images, err := buildimage.List(imageContext)
if err != nil {
logger.Error("get images list is error: %s", err.Error())
os.Exit(1)
}
is := registry.NewImageSaver(context.Background(), registryPullMaxPullProcs, auth)
outImages, err := is.SaveImages(images, registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: registryPullArch})
if err != nil {
logger.Error("pull registry images is error: %s", err.Error())
os.Exit(1)
}
logger.Info("pull images list save to local : %+v", outImages)
},
PreRun: func(cmd *cobra.Command, args []string) {
auth = preValidate()
if !file.IsExist(imageFile) {
logger.Error("ImageFile path is not exist")
os.Exit(1)
}
},
}
registryImagePullRaw.PersistentFlags().StringVarP(&imageFile, "image-file", "f", "ImageFile", "ImageFile path")
return registryImagePullRaw
}
pullDefaultCmd := func() *cobra.Command {
var images []string
var auth map[string]types.AuthConfig
var registryImagePullDefault = &cobra.Command{
Use: "default",
Short: "registry images manager pull to local dir by default type",
Run: func(cmd *cobra.Command, args []string) {
is := registry.NewImageSaver(context.Background(), registryPullMaxPullProcs, auth)
outImages, err := is.SaveImages(images, registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: registryPullArch})
if err != nil {
logger.Error("pull registry images is error: %s", err.Error())
os.Exit(1)
}
logger.Info("pull images list save to local : %+v", outImages)
},
PreRun: func(cmd *cobra.Command, args []string) {
auth = preValidate()
},
}
registryImagePullDefault.PersistentFlags().StringSliceVar(&images, "images", []string{}, "images list")
return registryImagePullDefault
func newRegistryImageSaveCmd() *cobra.Command {
var registryImagePullCmd = &cobra.Command{
Use: "save",
Short: "save registry images to local registry dir",
}
registryImagePullCmd.AddCommand(saveCmd("default"))
registryImagePullCmd.AddCommand(saveCmd("raw"))
return registryImagePullCmd
}
func saveCmd(cmdType string) *cobra.Command {
switch cmdType {
case "raw":
return pullRawCmd()
return saveRawCmd()
case "default":
return pullDefaultCmd()
return saveDefaultCmd()
default:
return pullDefaultCmd()
return saveDefaultCmd()
}
}
func saveDefaultCmd() *cobra.Command {
var images []string
var auth map[string]types.AuthConfig
flagsResults := registrySaveDefaultResults{
registrySaveResults: new(registrySaveResults),
}
var registryImagePullDefault = &cobra.Command{
Use: "default [CONTEXT]",
Short: "registry images manager save to local dir by default type",
Run: func(cmd *cobra.Command, args []string) {
is := registry.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
if err != nil {
logger.Error("pull registry images is error: %s", err.Error())
os.Exit(1)
}
logger.Info("pull images list save to local : %+v", outImages)
},
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
images, err = buildimage.List(args[0])
if err != nil {
return err
}
auth, err = flagsResults.CheckAuth()
if err != nil {
return err
}
return nil
},
}
flags := registryImagePullDefault.Flags()
flags.SetInterspersed(false)
flagsResults.RegisterFlags(flags)
return registryImagePullDefault
}
func saveRawCmd() *cobra.Command {
var auth map[string]types.AuthConfig
flagsResults := registrySaveRawResults{
registrySaveResults: new(registrySaveResults),
}
var registryImagePullRaw = &cobra.Command{
Use: "raw",
Short: "registry images manager save to local dir by raw type",
RunE: func(cmd *cobra.Command, args []string) error {
is := registry.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
outImages, err := is.SaveImages(flagsResults.images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
if err != nil {
return errors.Wrap(err, "pull registry images is error")
}
logger.Info("pull images list save to local : %+v", outImages)
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
auth, err = flagsResults.CheckAuth()
if err != nil {
return err
}
return nil
},
}
flags := registryImagePullRaw.Flags()
flags.SetInterspersed(false)
flagsResults.RegisterFlags(flags)
_ = registryImagePullRaw.MarkFlagRequired("images")
return registryImagePullRaw
}
pkg/registry/cmd/status.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"os"
"github.com/docker/docker/api/types"
"github.com/spf13/cobra"
"github.com/labring/sealos/pkg/registry"
"github.com/labring/sealos/pkg/utils/logger"
)
func newRegistryStatusCmd() *cobra.Command {
preValidate := func() map[string]types.AuthConfig {
cfg, err := registry.GetAuthInfo()
if err != nil {
logger.Error("auth info is error: %+v", err)
os.Exit(1)
}
return cfg
}
var auth map[string]types.AuthConfig
var is registry.Registry
flagsResults := registryStatusResults{}
var registryStatusCmd = &cobra.Command{
Use: "status",
Short: "registry status",
Example: fmt.Sprintf(`%[1]s registry status`, rootCmd.Root().CommandPath()),
RunE: func(cmd *cobra.Command, args []string) error {
is.Status(flagsResults.json)
return nil
},
PreRun: func(cmd *cobra.Command, args []string) {
auth = preValidate()
is = registry.NewImage(auth)
},
}
flags := registryStatusCmd.Flags()
flags.SetInterspersed(false)
flagsResults.RegisterFlags(flags)
return registryStatusCmd
}
pkg/registry/filter.go
+180
View File
@@ -0,0 +1,180 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"errors"
"fmt"
"strings"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/maps"
strings2 "github.com/labring/sealos/pkg/utils/strings"
)
const none = "<none>"
type FilterType string
const (
FilterTypeName = "name"
FilterTypeTag = "tag"
)
type FilterStrategy string
const (
FilterStrategyNone = "none"
FilterStrategyPrefix = "prefix"
FilterStrategySuffix = "suffix"
FilterStrategyDefault = "default"
FilterStrategyEquals = "equals"
FilterStrategyAll = "all"
FilterStrategyUnknown = "unknown"
)
type Filter struct {
Name string
Tag string
nameStrategy FilterStrategy
tagStrategy FilterStrategy
}
func (f *Filter) Validate() error {
check := func(filter string, t FilterType) (FilterStrategy, error) {
if filter == none {
if t == FilterTypeName {
return FilterStrategyUnknown, errors.New("your filter repo not allow is none")
}
return FilterStrategyNone, nil
}
if filter == "" {
return FilterStrategyAll, nil
}
if len(filter) == 1 && filter == "*" {
return FilterStrategyAll, nil
}
if strings.Contains(filter, "*") {
if filter[0] == '*' && filter[len(filter)-1] == '*' {
return FilterStrategyDefault, nil
}
if filter[0] == '*' {
if strings.LastIndex(filter, "*") == 0 {
return FilterStrategySuffix, nil
}
return FilterStrategyUnknown, fmt.Errorf("your filter must has one char '*' , example *ccc ")
}
if filter[len(filter)-1] == '*' {
if strings.LastIndex(filter, "*") == len(filter)-1 {
return FilterStrategyPrefix, nil
}
return FilterStrategyUnknown, fmt.Errorf("your filter must has one char '*' , example ccc* ")
}
return FilterStrategyUnknown, fmt.Errorf("not spport char '*' in filter middle")
}
return FilterStrategyEquals, nil
}
errorlist := make([]error, 0)
strategy, err := check(f.Name, FilterTypeName)
if err != nil {
errorlist = append(errorlist, err)
}
f.nameStrategy = strategy
strategy, err = check(f.Tag, FilterTypeTag)
if err != nil {
errorlist = append(errorlist, err)
}
f.tagStrategy = strategy
if len(errorlist) == 0 {
return nil
}
return fmt.Errorf("your filter has some errors: %+v", errorlist)
}
func (f *Filter) Run(data []string, t FilterType) []string {
if t == FilterTypeName {
return filter(data, f.Name, f.nameStrategy)
}
return filter(data, f.Tag, f.tagStrategy)
}
func newFilter(filter string) *Filter {
filterMap := maps.StringToMap(filter, ",")
for key := range filterMap {
if !strings2.InList(key, []string{FilterTypeName, FilterTypeTag}) {
logger.Warn("filter key not support: %s", key)
}
}
return &Filter{
Name: filterMap[FilterTypeName],
Tag: filterMap[FilterTypeTag],
}
}
func filter(data []string, filter string, strategy FilterStrategy) []string {
prefix := func(data []string, filter string) []string {
res := make([]string, 0)
for _, d := range data {
if strings.HasPrefix(d, filter) {
res = append(res, d)
}
}
return res
}
suffix := func(data []string, filter string) []string {
res := make([]string, 0)
for _, d := range data {
if strings.HasSuffix(d, filter) {
res = append(res, d)
}
}
return res
}
contains := func(data []string, filter string) []string {
res := make([]string, 0)
for _, d := range data {
if strings.Contains(d, filter) {
res = append(res, d)
}
}
return res
}
switch strategy {
case FilterStrategyDefault:
return contains(data, filter[1:len(filter)-1])
case FilterStrategyPrefix:
return prefix(data, filter[:len(filter)-1])
case FilterStrategySuffix:
return suffix(data, filter[1:])
case FilterStrategyNone:
return []string{}
case FilterStrategyAll:
return data
case FilterStrategyEquals:
return []string{filter}
default:
}
return []string{}
}
pkg/registry/filter_test.go
+163
View File
@@ -0,0 +1,163 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"reflect"
"testing"
)
func TestFilter_Validate(t *testing.T) {
tests := []struct {
name string
fields *Filter
wantErr bool
}{
{
name: "default",
fields: newFilter(""),
wantErr: false,
},
{
name: "name",
fields: newFilter("name=<none>"),
wantErr: true,
},
{
name: "name-prefix",
fields: newFilter("name=aa*"),
wantErr: false,
},
{
name: "name-suffix",
fields: newFilter("name=*cccc"),
wantErr: false,
},
{
name: "name-error",
fields: newFilter("name=ccc*cccc"),
wantErr: true,
},
{
name: "name-eque",
fields: newFilter("name=cccc"),
wantErr: false,
},
{
name: "name-default1",
fields: newFilter("name=*"),
wantErr: false,
},
{
name: "name-default2",
fields: newFilter("name="),
wantErr: false,
},
{
name: "tag",
fields: newFilter("tag=<none>"),
wantErr: false,
},
{
name: "tag-prefix",
fields: newFilter("tag=aa*"),
wantErr: false,
},
{
name: "tag-suffix",
fields: newFilter("tag=*cccc"),
wantErr: false,
},
{
name: "tag-error",
fields: newFilter("tag=ccc*cccc"),
wantErr: true,
},
{
name: "tag-default1",
fields: newFilter("tag=*"),
wantErr: false,
},
{
name: "tag-default2",
fields: newFilter("tag="),
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := tt.fields
if err := f.Validate(); (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestFilter_Run(t *testing.T) {
type args struct {
data []string
t FilterType
}
tests := []struct {
name string
fields *Filter
args args
want []string
}{
{
name: "default",
fields: newFilter("name=a"),
args: args{
data: []string{"a", "b", "c"},
t: FilterTypeName,
},
want: []string{"a"},
},
{
name: "default",
fields: newFilter("name=c*"),
args: args{
data: []string{"abcc", "bdcff", "c"},
t: FilterTypeName,
},
want: []string{"c"},
},
{
name: "default",
fields: newFilter("name=*labring*"),
args: args{
data: []string{"labring/lvscare"},
t: FilterTypeName,
},
want: []string{"labring/lvscare"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := tt.fields
if err := f.Validate(); err != nil {
t.Error(err)
return
}
if got := f.Run(tt.args.data, tt.args.t); !reflect.DeepEqual(got, tt.want) {
t.Errorf("Run() = %v, want %v", got, tt.want)
}
})
}
}
pkg/registry/interface.go
+2 -2
View File
@@ -27,8 +27,8 @@ import (
type Registry interface {
// SaveImages is not concurrently safe
SaveImages(images []string, dir string, platform v1.Platform) ([]string, error)
ListRegistry()
ListImages(registryName, search string)
Status(json bool)
ListImages(registryName, search string, json bool)
RmiImage(registryName, imageName string) error
}
pkg/registry/list.go
+63 -43
View File
@@ -17,14 +17,18 @@ limitations under the License.
package registry
import (
"fmt"
"github.com/modood/table"
"github.com/opencontainers/go-digest"
"k8s.io/apimachinery/pkg/util/json"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/registry"
)
func (is *DefaultImage) ListImages(registryName string, search string) {
func (is *DefaultImage) ListImages(registryName, search string, enableJSON bool) {
logger.Debug("param registryName: %s, filter: %s", registryName, search)
authInfo := is.auths[registryName]
var listImage []imageOutputParams
reg, err := registry.NewRegistryForDomain(registryName, authInfo.Username, authInfo.Password)
@@ -32,60 +36,76 @@ func (is *DefaultImage) ListImages(registryName string, search string) {
logger.Error("Failed to list image : %v", err)
return
}
repos, _ := reg.Repositories()
filter := newFilter(search)
if err = filter.Validate(); err != nil {
logger.Error("Failed to list image using filter : %v", err)
return
}
logger.Debug("filter name=%s,strategy=%s", filter.Name, filter.nameStrategy)
logger.Debug("filter tag=%s,strategy=%s", filter.Tag, filter.tagStrategy)
var repos []string
if filter.nameStrategy == FilterStrategyEquals {
repos = []string{filter.Name}
} else {
repos, err = reg.Repositories(func(data []string) []string {
return filter.Run(data, FilterTypeName)
})
if err != nil {
logger.Error("list image is error: %+v", err)
return
}
}
var imageVersionList bool
defer func() {
if !enableJSON {
logger.Info("Image count %d", len(repos))
if imageVersionList {
logger.Info("Images Version count %d", len(listImage))
}
}
}()
for _, repo := range repos {
tags, _ := reg.Tags(repo)
for _, tag := range tags {
var imageID digest.Digest
imageID, _ = reg.ManifestDigest(repo, tag)
tags = filter.Run(tags, FilterTypeTag)
if len(tags) == 0 {
listImage = append(listImage, imageOutputParams{
RegistryName: registryName,
Repository: repo,
Tag: tag,
ImageID: imageID.String(),
})
}
}
table.OutputA(listImage)
logger.Info("Images count %d", len(listImage))
}
func (is *DefaultImage) ListRegistry() {
var listRegistry []registryOutputParams
for domain, auth := range is.auths {
reg, err := registry.NewRegistryForDomain(domain, auth.Username, auth.Password)
if err != nil {
listRegistry = append(listRegistry, registryOutputParams{
Name: domain,
URL: "unknow://" + domain,
UserName: auth.Username,
Password: auth.Password,
Healthy: "failed",
ImageName: repo,
Tag: "<none>",
ImageID: "<none>",
})
} else {
listRegistry = append(listRegistry, registryOutputParams{
Name: domain,
URL: reg.URL,
UserName: auth.Username,
Password: auth.Password,
Healthy: "ok",
})
imageVersionList = true
for _, tag := range tags {
var imageID digest.Digest
imageID, _ = reg.ManifestDigest(repo, tag)
listImage = append(listImage, imageOutputParams{
RegistryName: registryName,
ImageName: repo,
Tag: tag,
ImageID: imageID.String(),
})
}
}
}
table.OutputA(listRegistry)
}
type registryOutputParams struct {
Name string
URL string
UserName string
Password string
Healthy string
if enableJSON {
marshalled, err := json.Marshal(listImage)
if err != nil {
logger.Error("Failed to Marshal Json : %v", err)
return
}
fmt.Println(string(marshalled))
return
}
table.OutputA(listImage)
}
type imageOutputParams struct {
RegistryName string
Repository string
ImageName string
Tag string
ImageID string
}
pkg/registry/list_test.go
+2 -2
View File
@@ -47,7 +47,7 @@ func TestDefaultImage_ListRegistry(t *testing.T) {
is := &DefaultImage{
auths: tt.fields.auths,
}
is.ListRegistry()
is.Status(false)
})
}
}
@@ -84,7 +84,7 @@ func TestDefaultImage_ListImages(t *testing.T) {
is := &DefaultImage{
auths: tt.fields.auths,
}
is.ListImages(tt.args.registryName, "")
is.ListImages(tt.args.registryName, "name=*,tag=<none>", false)
})
}
}
pkg/registry/save.go
+1
View File
@@ -58,6 +58,7 @@ const (
)
func (is *DefaultImage) SaveImages(images []string, dir string, platform v1.Platform) ([]string, error) {
logger.Debug("search images platform: %s , dir: %s, image list: %+v", strings.Join([]string{platform.OS, platform.Architecture, platform.Variant}, ","), dir, images)
//init a pipe for display pull message
reader, writer := io.Pipe()
defer func() {
pkg/registry/status.go
+72
View File
@@ -0,0 +1,72 @@
/*
Copyright 2022 cuisongliu@qq.com.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"fmt"
"github.com/modood/table"
"k8s.io/apimachinery/pkg/util/json"
"github.com/labring/sealos/pkg/utils/logger"
"github.com/labring/sealos/pkg/utils/registry"
)
func (is *DefaultImage) Status(enableJSON bool) {
var listRegistry []registryOutputParams
if len(is.auths) == 0 && !enableJSON {
logger.Warn("your registry not login")
}
for domain, auth := range is.auths {
reg, err := registry.NewRegistryForDomain(domain, auth.Username, auth.Password)
if err != nil {
listRegistry = append(listRegistry, registryOutputParams{
Name: domain,
URL: "unknow://" + domain,
UserName: auth.Username,
Password: auth.Password,
Healthy: "failed",
})
} else {
listRegistry = append(listRegistry, registryOutputParams{
Name: domain,
URL: reg.URL,
UserName: auth.Username,
Password: auth.Password,
Healthy: "ok",
})
}
}
if enableJSON {
marshalled, err := json.Marshal(listRegistry)
if err != nil {
logger.Error("Failed to Marshal Json : %v", err)
return
}
fmt.Println(string(marshalled))
return
}
table.OutputA(listRegistry)
}
type registryOutputParams struct {
Name string
URL string
UserName string
Password string
Healthy string
}