Merge pull request #973 in YUNIONIO/onecloud from ~QIUJIAN/onecloud:hotfix/qj-fix-aws-image-fetch-info to release/2.6.0

* commit '878bc69358620916304e5c28d6b15d7c20d88756':
  minor updates
  改进:对Aws系统镜像分类处理
This commit is contained in:
邱剑
2019-01-26 20:10:17 +08:00
10 changed files with 810 additions and 116 deletions
+133 -58
View File
@@ -4,16 +4,16 @@ import (
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go/service/ec2"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/util/timeutils"
"time"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/pkg/util/timeutils"
)
type ImageStatusType string
@@ -24,11 +24,19 @@ const (
ImageStatusCreateFailed ImageStatusType = "failed"
)
type TImageOwnerType string
const (
ImageOwnerTypeSystem = TImageOwnerType("system")
ImageOwnerTypeSelf = TImageOwnerType("self")
ImageOwnerTypeOther = TImageOwnerType("other")
)
var (
ImageOwnerAmazone = "amazon"
ImageOwnerSelf = "self"
ImageOwnerMicrosoft = "microsoft"
ImageOwnerMarketplace = "aws-marketplace"
ImageOwnerAll = []TImageOwnerType(nil)
ImageOwnerSelf = []TImageOwnerType{ImageOwnerTypeSelf}
ImageOwnerSystem = []TImageOwnerType{ImageOwnerTypeSystem}
ImageOwnerSelfSystem = []TImageOwnerType{ImageOwnerTypeSystem, ImageOwnerTypeSelf}
)
type ImageImportTask struct {
@@ -51,17 +59,27 @@ type SImage struct {
Description string
ImageId string
ImageName string
OSName string
OSType string
ImageType string
// IsSupportCloudinit bool
IsSupportIoOptimized bool
Platform string
SizeGB int
Status ImageStatusType
OwnerType string
EnaSupport bool
Platform string
SizeGB int
Status ImageStatusType
OwnerType string
// Usage string
RootDevice RootDevice
Public bool
Hypervisor string
VirtualizationType string
OwnerId string
ProductCodes []*ec2.ProductCode
OSVersion string
OSDist string
OSBuildId string
}
func (self *SImage) GetId() string {
@@ -111,18 +129,7 @@ func (self *SImage) Refresh() error {
}
func (self *SImage) GetImageType() string {
switch self.OwnerType {
case ImageOwnerSelf:
return cloudprovider.CachedImageTypeCustomized
case ImageOwnerAmazone:
return cloudprovider.CachedImageTypeSystem
case ImageOwnerMicrosoft:
return cloudprovider.CachedImageTypeShared
case ImageOwnerMarketplace:
return cloudprovider.CachedImageTypeMarket
default:
return cloudprovider.CachedImageTypeShared
}
return self.ImageType
}
func (self *SImage) GetSize() int64 {
@@ -138,15 +145,15 @@ func (self *SImage) GetOsArch() string {
}
func (self *SImage) GetOsDist() string {
return self.Platform
return self.OSDist
}
func (self *SImage) GetOsVersion() string {
return self.OSName
return self.OSVersion
}
func (self *SImage) GetMinOsDiskSizeGb() int {
return 10
return self.SizeGB
}
func (self *SImage) GetImageFormat() string {
@@ -169,11 +176,11 @@ func (self *SImage) GetMetadata() *jsonutils.JSONDict {
if len(self.OSType) > 0 {
data.Add(jsonutils.NewString(self.OSType), "os_name")
}
if len(self.Platform) > 0 {
data.Add(jsonutils.NewString(self.Platform), "os_distribution")
if len(self.OSDist) > 0 {
data.Add(jsonutils.NewString(self.OSDist), "os_distribution")
}
if len(self.OSName) > 0 {
data.Add(jsonutils.NewString(self.OSName), "os_version")
if len(self.OSVersion) > 0 {
data.Add(jsonutils.NewString(self.OSVersion), "os_version")
}
return data
}
@@ -239,22 +246,22 @@ func (self *SRegion) GetImage(imageId string) (*SImage, error) {
return nil, fmt.Errorf("GetImage image id should not be empty")
}
images, err := self.GetImages("", nil, []string{imageId}, "")
images, err := self.getImages("", ImageOwnerAll, []string{imageId}, "", "", nil, "")
if err != nil {
return nil, err
}
if len(images) == 0 {
return nil, fmt.Errorf("image %s not found", imageId)
return nil, cloudprovider.ErrNotFound
}
return &images[0], nil
}
func (self *SRegion) GetImageByName(name string) (*SImage, error) {
func (self *SRegion) GetImageByName(name string, owners []TImageOwnerType) (*SImage, error) {
if len(name) == 0 {
return nil, fmt.Errorf("image name should not be empty")
}
images, err := self.GetImages("", nil, nil, name)
images, err := self.getImages("", owners, nil, name, "hvm", nil, "")
if err != nil {
return nil, err
}
@@ -277,15 +284,53 @@ func (self *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) {
func getRootDiskSize(image *ec2.Image) (int, error) {
rootDeivce := *image.RootDeviceName
for _, volume := range image.BlockDeviceMappings {
if len(rootDeivce) > 0 && *volume.DeviceName == rootDeivce {
if len(rootDeivce) > 0 && *volume.DeviceName == rootDeivce && volume.Ebs != nil && volume.Ebs.VolumeSize != nil {
return int(*volume.Ebs.VolumeSize), nil
}
}
return 0, fmt.Errorf("image size not found: %s", image.String())
}
func (self *SRegion) GetImages(status ImageStatusType, owners []*string, imageId []string, name string) ([]SImage, error) {
func getLatestImage(images []SImage) SImage {
var latestBuild string
latestBuildIdx := -1
for i := range images {
if latestBuildIdx < 0 || comapreImageBuildIds(latestBuild, images[i]) < 0 {
latestBuild = images[i].OSBuildId
latestBuildIdx = i
}
}
return images[latestBuildIdx]
}
func (self *SRegion) GetImages(status ImageStatusType, owners []TImageOwnerType, imageId []string, name string, virtualizationType string, ownerIds []string, volumeType string, latest bool) ([]SImage, error) {
images, err := self.getImages(status, owners, imageId, name, virtualizationType, ownerIds, volumeType)
if err != nil {
return nil, err
}
if !latest {
return images, err
}
noVersionImages := make([]SImage, 0)
versionedImages := make(map[string][]SImage)
for i := range images {
key := fmt.Sprintf("%s%s", images[i].OSDist, images[i].OSVersion)
if len(key) == 0 {
noVersionImages = append(noVersionImages, images[i])
continue
}
if _, ok := versionedImages[key]; !ok {
versionedImages[key] = make([]SImage, 0)
}
versionedImages[key] = append(versionedImages[key], images[i])
}
for key := range versionedImages {
noVersionImages = append(noVersionImages, getLatestImage(versionedImages[key]))
}
return noVersionImages, nil
}
func (self *SRegion) getImages(status ImageStatusType, owners []TImageOwnerType, imageId []string, name string, virtualizationType string, ownerIds []string, volumeType string) ([]SImage, error) {
params := &ec2.DescribeImagesInput{}
filters := make([]*ec2.Filter, 0)
if len(status) > 0 {
@@ -296,8 +341,18 @@ func (self *SRegion) GetImages(status ImageStatusType, owners []*string, imageId
filters = AppendSingleValueFilter(filters, "name", name)
}
if len(owners) > 0 {
params.SetOwners(owners)
if len(virtualizationType) > 0 {
filters = AppendSingleValueFilter(filters, "virtualization-type", virtualizationType)
}
if len(volumeType) > 0 {
filters = AppendSingleValueFilter(filters, "block-device-mapping.volume-type", volumeType)
}
filters = AppendSingleValueFilter(filters, "image-type", "machine")
if len(owners) > 0 || len(ownerIds) > 0 {
params.SetOwners(imageOwnerTypes2Strings(owners, ownerIds))
}
if len(imageId) > 0 {
@@ -317,7 +372,9 @@ func (self *SRegion) GetImages(status ImageStatusType, owners []*string, imageId
}
images := []SImage{}
for _, image := range ret.Images {
for i := range ret.Images {
image := ret.Images[i]
if err := FillZero(image); err != nil {
return nil, err
}
@@ -327,7 +384,9 @@ func (self *SRegion) GetImages(status ImageStatusType, owners []*string, imageId
size, err := getRootDiskSize(image)
if err != nil {
log.Debugf(err.Error())
// fail to get disk size, ignore the image
/// log.Debugf(err.Error())
continue
}
var rootDevice RootDevice
@@ -348,22 +407,38 @@ func (self *SRegion) GetImages(status ImageStatusType, owners []*string, imageId
createTime, _ := timeutils.ParseTimeStr(*image.CreationDate)
images = append(images, SImage{
storageCache: self.getStoragecache(),
Architecture: *image.Architecture,
Description: *image.Description,
ImageId: *image.ImageId,
ImageName: tagspec.GetNameTag(),
ImageType: *image.ImageType,
OwnerType: *image.ImageOwnerAlias,
IsSupportIoOptimized: *image.EnaSupport,
Platform: *image.Platform,
Status: ImageStatusType(*image.State),
CreationTime: createTime,
SizeGB: size,
RootDevice: rootDevice,
OSType: osType,
})
name := tagspec.GetNameTag()
if len(name) == 0 && image.Name != nil {
name = *image.Name
}
sImage := SImage{
storageCache: self.getStoragecache(),
Architecture: *image.Architecture,
Description: *image.Description,
ImageId: *image.ImageId,
Public: *image.Public,
ImageName: name,
OSType: osType,
// ImageType: *image.ImageType,
OwnerType: *image.ImageOwnerAlias,
EnaSupport: *image.EnaSupport,
Platform: *image.Platform,
Status: ImageStatusType(*image.State),
CreationTime: createTime,
SizeGB: size,
RootDevice: rootDevice,
VirtualizationType: *image.VirtualizationType,
Hypervisor: *image.Hypervisor,
ProductCodes: image.ProductCodes,
OwnerId: *image.OwnerId,
}
sImage.ImageType = getImageType(sImage)
sImage.OSType = getImageOSType(sImage)
sImage.OSDist = getImageOSDist(sImage)
sImage.OSVersion = getImageOSVersion(sImage)
sImage.OSBuildId = getImageOSBuildID(sImage)
images = append(images, sImage)
}
return images, nil
+572
View File
@@ -0,0 +1,572 @@
package aws
import (
"fmt"
"regexp"
"strconv"
"strings"
"yunion.io/x/pkg/util/regutils"
"yunion.io/x/onecloud/pkg/cloudprovider"
)
func getSystemOwnerIds() []string {
keys := make([]string, len(awsImagePublishers))
idx := 0
for key := range awsImagePublishers {
keys[idx] = key
idx += 1
}
return keys
}
func stringArray2StringPtrArray(arr []string) []*string {
ret := make([]*string, len(arr))
for i := range arr {
ret[i] = &arr[i]
}
return ret
}
func imageOwnerTypes2Strings(owners []TImageOwnerType, rawIds []string) []*string {
ownerIds := make([]string, 0)
for i := range owners {
switch owners[i] {
case ImageOwnerTypeSelf:
ownerIds = append(ownerIds, "self")
case ImageOwnerTypeSystem:
sysOwnerIds := getSystemOwnerIds()
ownerIds = append(ownerIds, sysOwnerIds...)
}
}
ownerIds = append(ownerIds, rawIds...)
return stringArray2StringPtrArray(ownerIds)
}
type SAWSImagePublisherInfo struct {
GetOSType func(image SImage) string
GetOSDist func(image SImage) string
GetOSVersion func(image SImage) string
GetOSBuildID func(image SImage) string
CompareBuilds func(v1, v2 string) int
}
var rhel = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
return "RHEL"
},
GetOSVersion: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
if len(parts) >= 2 {
parts = strings.Split(parts[1], "_")
return parts[0]
}
return ""
},
GetOSBuildID: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
if len(parts) >= 2 {
return parts[2]
}
return ""
},
}
var (
debianDatePattern = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}-`)
)
var debian = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
return "Debian"
},
GetOSVersion: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
if len(parts) >= 5 {
return parts[1]
}
return ""
},
GetOSBuildID: func(image SImage) string {
dateStr := debianDatePattern.FindString(image.ImageName)
if len(dateStr) > 2 {
return dateStr[1 : len(dateStr)-1]
}
return ""
},
}
var (
centosDatePattern = regexp.MustCompile(`\s+\d{4,8}(_\d+)?`)
)
var centos = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
if strings.Index(image.ImageName, "Atomic") > 0 {
return "CentOS Atomic"
} else {
return "CentOS"
}
},
GetOSVersion: func(image SImage) string {
parts := strings.Split(image.ImageName, " ")
if strings.Index(image.ImageName, "Atomic") > 0 {
if regutils.MatchInteger(parts[3]) {
return parts[3]
} else {
return "7"
}
} else if strings.HasPrefix(image.ImageName, "CentOS Linux ") {
return parts[2]
} else {
return parts[1]
}
return ""
},
GetOSBuildID: func(image SImage) string {
build := centosDatePattern.FindString(image.ImageName)
build = strings.TrimSpace(build)
if strings.HasPrefix(build, "201") {
build = build[2:]
}
return build
},
}
var ubuntuReleases = map[string]string{
"warty": "4.10",
"hoary": "5.04",
"breezy": "5.10",
"dapper": "6.06",
"edgy": "6.10",
"feisty": "7.04",
"gutsy": "7.10",
"hardy": "8.04",
"intrepid": "8.10",
"jaunty": "9.04",
"karmic": "9.10",
"lucid": "10.04",
"maverick": "10.10",
"natty": "11.04",
"oneiric": "11.10",
"precise": "12.04",
"quantal": "12.10",
"raring": "13.04",
"saucy": "13.10",
"trusty": "14.04",
"utopic": "14.10",
"vivid": "15.04",
"wily": "15.10",
"xenial": "16.04",
"yakkety": "16.10",
"zesty": "17.04",
"artful": "17.10",
"bionic": "18.04",
"cosmic": "18.10",
"disco": "19.04",
}
var ubuntuReleasePattern = regexp.MustCompile(`-\d+\.\d+-`)
var ubuntu = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
if strings.HasPrefix(image.ImageName, "ubuntu-minimal/") {
return "Ubuntu Minimal"
}
if strings.HasPrefix(image.ImageName, "ubuntu/") {
return "Ubuntu"
}
if strings.HasPrefix(image.ImageName, "ubuntu-rolling-") || strings.HasPrefix(image.ImageName, "ubuntu-core") || strings.Index(image.ImageName, "core-edge") > 0 {
return "Ubuntu Core"
}
return "Ubuntu"
},
GetOSVersion: func(image SImage) string {
relStr := ubuntuReleasePattern.FindString(image.ImageName)
if len(relStr) > 2 {
return relStr[1 : len(relStr)-1]
}
parts := strings.Split(image.ImageName, "/")
if len(parts) >= 4 {
parts = strings.Split(parts[3], "-")
if len(parts) >= 2 {
relName := strings.ToLower(parts[1])
if _, ok := ubuntuReleases[relName]; ok {
return ubuntuReleases[relName]
} else {
return relName
}
}
}
if strings.HasPrefix(image.ImageName, "ubuntu-rolling-") {
parts := strings.Split(image.ImageName, "-")
if len(parts) > 3 {
return strings.ToLower(parts[2])
}
}
return ""
},
GetOSBuildID: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
return parts[len(parts)-1]
},
}
var (
SUSE_SLES = regexp.MustCompile(`suse-sles-\d+-v?\d+-`)
SUSE_SLES_SP = regexp.MustCompile(`suse-sles-\d+-sp\d+-v?\d+-`)
SUSE_SLES_RIGHTLINK = regexp.MustCompile(`suse-sles-\d+-rightscale-v?\d+-`)
SUSE_SLES_RIGHTLINK_SP = regexp.MustCompile(`suse-sles-\d+-sp\d+-rightscale-v?\d+-`)
SUSE_SLES_SAPCAL = regexp.MustCompile(`suse-sles-\d+-sapcal-v?\d+-`)
SUSE_SLES_SAPCAL_SP = regexp.MustCompile(`suse-sles-\d+-sp\d+-sapcal-v?\d+-`)
SUSE_SLES_BYOS = regexp.MustCompile(`suse-sles-\d+-byos-v?\d+-`)
SUSE_SLES_BYOS_SP = regexp.MustCompile(`suse-sles-\d+-sp\d+-byos-v?\d+-`)
SUSE_SLES_SAP = regexp.MustCompile(`suse-sles-sap-\d+-v\d+-`)
SUSE_SLES_SAP_SP = regexp.MustCompile(`suse-sles-sap-\d+-sp\d+-v\d+-`)
SUSE_SLES_SAP_BYOS = regexp.MustCompile(`suse-sles-sap-\d+-byos-v?\d+-`)
SUSE_SLES_SAP_BYOS_SP = regexp.MustCompile(`suse-sles-sap-\d+-sp\d+-byos-v?\d+-`)
SUSE_CAASP_CLUSTER_BYOS = regexp.MustCompile(`suse-caasp-\d+-\d+-cluster-byos-v?\d+-`)
SUSE_CAASP_ADMIN_BYOS = regexp.MustCompile(`suse-caasp-\d+-\d+-admin-byos-v?\d+-`)
SUSE_MANAGER_SERVER_BYOS = regexp.MustCompile(`suse-manager-\d+-\d+-server-byos-v?\d+-`)
SUSE_MANAGER_PROXY_BYOS = regexp.MustCompile(`suse-manager-\d+-\d+-proxy-byos-v?\d+-`)
)
func getBuildId(ver string) string {
if ver[0] == 'v' {
return ver[1:]
} else {
return ver
}
}
var suse = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
switch {
case SUSE_SLES.MatchString(image.ImageName), SUSE_SLES_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server"
case SUSE_SLES_RIGHTLINK.MatchString(image.ImageName), SUSE_SLES_RIGHTLINK_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server with RightLink"
case SUSE_SLES_SAPCAL.MatchString(image.ImageName), SUSE_SLES_SAPCAL_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server for SAP CAL"
case SUSE_SLES_BYOS.MatchString(image.ImageName), SUSE_SLES_BYOS_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server BYOS"
case SUSE_SLES_SAP.MatchString(image.ImageName), SUSE_SLES_SAP_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server for SAP Application"
case SUSE_SLES_SAP_BYOS.MatchString(image.ImageName), SUSE_SLES_SAP_BYOS_SP.MatchString(image.ImageName):
return "SUSE Linux Enterpise Server for SAP Application BYOS"
case SUSE_CAASP_CLUSTER_BYOS.MatchString(image.ImageName):
return "SUSE CaaSP Cluster Node"
case SUSE_CAASP_ADMIN_BYOS.MatchString(image.ImageName):
return "SUSE CaaSP Admin Node"
case SUSE_MANAGER_SERVER_BYOS.MatchString(image.ImageName):
return "SUSE Manager Server"
case SUSE_MANAGER_PROXY_BYOS.MatchString(image.ImageName):
return "SUSE Manager Proxy"
}
return "SUSE"
},
GetOSVersion: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
switch {
case SUSE_SLES.MatchString(image.ImageName):
return parts[2]
case SUSE_SLES_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3][2:])
case SUSE_SLES_RIGHTLINK.MatchString(image.ImageName):
return parts[2]
case SUSE_SLES_RIGHTLINK_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3][2:])
case SUSE_SLES_SAPCAL.MatchString(image.ImageName):
return parts[2]
case SUSE_SLES_SAPCAL_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3][2:])
case SUSE_SLES_BYOS.MatchString(image.ImageName):
return parts[2]
case SUSE_SLES_BYOS_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3][2:])
case SUSE_SLES_SAP.MatchString(image.ImageName):
return parts[3]
case SUSE_SLES_SAP_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[3], parts[4][2:])
case SUSE_SLES_SAP_BYOS.MatchString(image.ImageName):
return parts[3]
case SUSE_SLES_SAP_BYOS_SP.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[3], parts[4][2:])
case SUSE_CAASP_CLUSTER_BYOS.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3])
case SUSE_CAASP_ADMIN_BYOS.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3])
case SUSE_MANAGER_SERVER_BYOS.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3])
case SUSE_MANAGER_PROXY_BYOS.MatchString(image.ImageName):
return fmt.Sprintf("%s.%s", parts[2], parts[3])
}
return ""
},
GetOSBuildID: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
switch {
case SUSE_SLES.MatchString(image.ImageName):
return getBuildId(parts[3])
case SUSE_SLES_SP.MatchString(image.ImageName):
return getBuildId(parts[4])
case SUSE_SLES_RIGHTLINK.MatchString(image.ImageName):
return getBuildId(parts[4])
case SUSE_SLES_RIGHTLINK_SP.MatchString(image.ImageName):
return getBuildId(parts[5])
case SUSE_SLES_SAPCAL.MatchString(image.ImageName):
return getBuildId(parts[4])
case SUSE_SLES_SAPCAL_SP.MatchString(image.ImageName):
return getBuildId(parts[5])
case SUSE_SLES_BYOS.MatchString(image.ImageName):
return getBuildId(parts[4])
case SUSE_SLES_BYOS_SP.MatchString(image.ImageName):
return getBuildId(parts[5])
case SUSE_SLES_SAP.MatchString(image.ImageName):
return getBuildId(parts[4])
case SUSE_SLES_SAP_SP.MatchString(image.ImageName):
return getBuildId(parts[5])
case SUSE_SLES_SAP_BYOS.MatchString(image.ImageName):
return getBuildId(parts[5])
case SUSE_SLES_SAP_BYOS_SP.MatchString(image.ImageName):
return getBuildId(parts[6])
case SUSE_CAASP_CLUSTER_BYOS.MatchString(image.ImageName):
return getBuildId(parts[6])
case SUSE_CAASP_ADMIN_BYOS.MatchString(image.ImageName):
return getBuildId(parts[6])
case SUSE_MANAGER_SERVER_BYOS.MatchString(image.ImageName):
return getBuildId(parts[6])
case SUSE_MANAGER_PROXY_BYOS.MatchString(image.ImageName):
return getBuildId(parts[6])
}
return ""
},
}
var coreosVersionTable = map[string]int{
"alpha": 0,
"beta": 1,
"stable": 2,
}
var coreos = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
return "CoreOS"
},
GetOSVersion: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
subparts := strings.Split(parts[2], ".")
return subparts[0]
},
GetOSBuildID: func(image SImage) string {
parts := strings.Split(image.ImageName, "-")
return fmt.Sprintf("%s-%s", parts[1], parts[2])
},
CompareBuilds: func(v1, v2 string) int {
parts1 := strings.Split(v1, "-")
parts2 := strings.Split(v2, "-")
majorV1 := coreosVersionTable[parts1[0]]
majorV2 := coreosVersionTable[parts2[0]]
if majorV1 != majorV2 {
return majorV1 - majorV2
}
parts1 = strings.Split(parts1[1], ".")
parts2 = strings.Split(parts2[1], ".")
for i := 0; i < len(parts1) && i < len(parts2); i += 1 {
n1, _ := strconv.Atoi(parts1[i])
n2, _ := strconv.Atoi(parts2[i])
if n1 != n2 {
return n1 - n2
}
}
return len(parts1) - len(parts2)
},
}
var (
sqlServerPattern = regexp.MustCompile(`-SQL_(\d+)_(\w+)-`)
dotnetcorePattern = regexp.MustCompile(`-dotnetcore-`)
)
var windowsServer = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
if strings.HasPrefix(image.ImageName, "ubuntu-") || strings.HasPrefix(image.ImageName, "amzn-ami-") || strings.HasPrefix(image.ImageName, "amzn2-ami-") {
return "Linux"
} else {
return "Windows"
}
},
GetOSDist: func(image SImage) string {
osStr := "Windows Server"
if strings.HasPrefix(image.ImageName, "ubuntu-") {
osStr = "Ubuntu"
} else if strings.HasPrefix(image.ImageName, "amzn-ami-") || strings.HasPrefix(image.ImageName, "amzn2-ami-") {
osStr = "Amazon Linux"
}
apps := make([]string, 0)
matchApp := sqlServerPattern.FindStringSubmatch(image.ImageName)
if len(matchApp) > 0 {
apps = append(apps, fmt.Sprintf("SQL Server %s %s", matchApp[1], matchApp[2]))
}
if dotnetcorePattern.MatchString(image.ImageName) {
apps = append(apps, ".Net Core")
}
if len(apps) > 0 {
osStr = fmt.Sprintf("%s with %s", osStr, strings.Join(apps, " "))
}
return osStr
},
GetOSVersion: func(image SImage) string {
if strings.HasPrefix(image.ImageName, "ubuntu-") {
return ubuntu.GetOSVersion(image)
}
if strings.HasPrefix(image.ImageName, "amzn-ami-") || strings.HasPrefix(image.ImageName, "amzn2-ami-") {
return amazon.GetOSVersion(image)
}
parts := strings.Split(image.ImageName, "-")
return strings.Join(parts[1:len(parts)-1], " ")
},
GetOSBuildID: func(image SImage) string {
if strings.HasPrefix(image.ImageName, "ubuntu-") {
return ubuntu.GetOSBuildID(image)
}
if strings.HasPrefix(image.ImageName, "amzn-ami-") || strings.HasPrefix(image.ImageName, "amzn2-ami-") {
return amazon.GetOSBuildID(image)
}
parts := strings.Split(image.ImageName, "-")
return parts[len(parts)-1]
},
}
var (
amazonVersionPattern = regexp.MustCompile(`-(\d{4})\.(\d{2})\.(rc-\d+|\d+)(\.(\d+))?`)
amazonVersionPattern2 = regexp.MustCompile(`-(\d{1,2})\.(\d{1,2})\.(\d{8})(\.(\d+))?`)
)
var amazon = SAWSImagePublisherInfo{
GetOSType: func(image SImage) string {
return "Linux"
},
GetOSDist: func(image SImage) string {
if strings.HasPrefix(image.ImageName, "amzn-ami-minimal-") || strings.HasPrefix(image.ImageName, "amzn2-ami-minimal-") {
return "Amazon Linux Minimal"
} else {
return "Amazon Linux"
}
},
GetOSVersion: func(image SImage) string {
verStrs := amazonVersionPattern2.FindStringSubmatch(image.ImageName)
if len(verStrs) > 3 {
return fmt.Sprintf("%s.%s.%s", verStrs[1], verStrs[2], verStrs[3][:6])
}
verStrs = amazonVersionPattern.FindStringSubmatch(image.ImageName)
if len(verStrs) > 3 {
return fmt.Sprintf("%s.%s.%s", verStrs[1], verStrs[2], verStrs[3])
}
return ""
},
GetOSBuildID: func(image SImage) string {
verStrs := amazonVersionPattern2.FindStringSubmatch(image.ImageName)
if len(verStrs) > 5 && len(verStrs[5]) > 0 {
return fmt.Sprintf("%s.%s", verStrs[3], verStrs[5])
} else if len(verStrs) > 3 {
return verStrs[3]
}
verStrs = amazonVersionPattern.FindStringSubmatch(image.ImageName)
if len(verStrs) > 5 {
return verStrs[5]
}
return ""
},
}
var awsImagePublishers = map[string]SAWSImagePublisherInfo{
"841258680906": rhel, // china
"309956199498": rhel, // international
"673060587306": debian, // china
"379101102735": debian, // international
"718707510307": centos, // china
"410186602215": centos, // international
"837727238323": ubuntu, // china
"099720109477": ubuntu, // internaltional
"841869936221": suse, // china
"013907871322": suse, // international
"280032941352": coreos, // china
"595879546273": coreos, // international
"016951021795": windowsServer, // china
"801119661308": windowsServer, // international
"141808717104": amazon, // china
"137112412989": amazon, // international
}
func getImageOSType(image SImage) string {
ownerInfo, ok := awsImagePublishers[image.OwnerId]
if ok {
return ownerInfo.GetOSType(image)
}
return image.OSType
}
func getImageOSDist(image SImage) string {
ownerInfo, ok := awsImagePublishers[image.OwnerId]
if ok {
return ownerInfo.GetOSDist(image)
}
return ""
}
func getImageOSVersion(image SImage) string {
ownerInfo, ok := awsImagePublishers[image.OwnerId]
if ok {
return ownerInfo.GetOSVersion(image)
}
return ""
}
func getImageOSBuildID(image SImage) string {
ownerInfo, ok := awsImagePublishers[image.OwnerId]
if ok {
return ownerInfo.GetOSBuildID(image)
}
return ""
}
func comapreImageBuildIds(ver1 string, img2 SImage) int {
ownerInfo, ok := awsImagePublishers[img2.OwnerId]
if ok && ownerInfo.CompareBuilds != nil {
return ownerInfo.CompareBuilds(ver1, img2.OSBuildId)
}
return strings.Compare(ver1, img2.OSBuildId)
}
func getImageType(image SImage) string {
_, ok := awsImagePublishers[image.OwnerId]
if ok {
return cloudprovider.CachedImageTypeSystem
}
if !image.Public {
return cloudprovider.CachedImageTypeCustomized
}
return cloudprovider.CachedImageTypeMarket
}
+35
View File
@@ -0,0 +1,35 @@
package aws
import (
"regexp"
"testing"
)
func TestImageDistMatch(t *testing.T) {
cases := []struct {
pattern *regexp.Regexp
match string
}{
{SUSE_SLES, "suse-sles-11-v20161021-hvm-ssd-x86_64"},
{SUSE_SLES_SP, "suse-sles-11-sp4-v20161021-hvm-ssd-x86_64"},
{SUSE_SLES_RIGHTLINK, "suse-sles-11-rightscale-v20160804-hvm-ssd-x86_64"},
{SUSE_SLES_RIGHTLINK_SP, "suse-sles-11-sp4-rightscale-v20160804-hvm-ssd-x86_64"},
{SUSE_SLES_SAPCAL, "suse-sles-11-sapcal-20150729-hvm-mag-x86_64"},
{SUSE_SLES_SAPCAL_SP, "suse-sles-11-sp4-sapcal-20150729-hvm-mag-x86_64"},
{SUSE_SLES_BYOS, "suse-sles-15-byos-v20180806-hvm-ssd-x86_64"},
{SUSE_SLES_BYOS_SP, "suse-sles-15-sp4-byos-v20180806-hvm-ssd-x86_64"},
{SUSE_SLES_SAP, "suse-sles-sap-12-v20180706-hvm-ssd-x86_64"},
{SUSE_SLES_SAP_SP, "suse-sles-sap-12-sp2-v20180706-hvm-ssd-x86_64"},
{SUSE_SLES_SAP_BYOS, "suse-sles-sap-12-byos-v20180706-hvm-ssd-x86_64"},
{SUSE_SLES_SAP_BYOS_SP, "suse-sles-sap-12-sp2-byos-v20180706-hvm-ssd-x86_64"},
{SUSE_CAASP_CLUSTER_BYOS, "suse-caasp-2-1-cluster-byos-v20180815-hvm-ssd-x86_64"},
{SUSE_CAASP_ADMIN_BYOS, "suse-caasp-2-1-admin-byos-v20180524-hvm-ssd-x86_64"},
{SUSE_MANAGER_SERVER_BYOS, "suse-manager-3-1-server-byos-v20170627-hvm-ssd-x86_64"},
{SUSE_MANAGER_PROXY_BYOS, "suse-manager-3-1-proxy-byos-v20180215-hvm-ssd-x86_64"},
}
for _, c := range cases {
if !c.pattern.MatchString(c.match) {
t.Errorf("not match %s %s", c.pattern, c.match)
}
}
}
+6 -3
View File
@@ -168,9 +168,12 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict {
data.Add(jsonutils.NewString(self.host.zone.GetGlobalId()), "zone_ext_id")
if len(self.ImageId) > 0 {
if image, err := self.host.zone.region.GetImage(self.ImageId); err != nil {
image, err := self.host.zone.region.GetImage(self.ImageId)
if err != nil {
log.Errorf("Failed to find image %s for instance %s zone %s", self.ImageId, self.GetId(), self.ZoneId)
} else if meta := image.GetMetadata(); meta != nil {
}
meta := image.GetMetadata()
if meta != nil {
data.Update(meta)
}
}
@@ -528,7 +531,7 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit
NetworkInterfaces: networkInterfaces,
VpcAttributes: vpcattr,
ProductCodes: productCodes,
OSName: image.OSName, // todo: 这里在model层回写OSName信息
OSName: image.OSType, // todo: 这里在model层回写OSName信息
OSType: image.OSType,
// ExpiredTime:
// VlanId:
+22 -19
View File
@@ -2,24 +2,27 @@ package aws
import "yunion.io/x/onecloud/pkg/cloudprovider"
// https://docs.aws.amazon.com/general/latest/gr/rande.html
var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
"ap-south-1": {Latitude: 19.0759837, Longitude: 72.8776559, City: "", CountryCode: ""},
"ap-northeast-3": {Latitude: 34.6937378, Longitude: 135.5021651, City: "", CountryCode: ""},
"us-east-1": {Latitude: 37.4315734, Longitude: -78.6568942, City: "", CountryCode: ""},
"us-east-2": {Latitude: 40.4172871, Longitude: -82.90712300000001, City: "", CountryCode: ""},
"ap-southeast-2": {Latitude: -33.8688197, Longitude: 151.2092955, City: "", CountryCode: ""},
"cn-northwest-1": {Latitude: 37.198731, Longitude: 106.1580937, City: "", CountryCode: "CN"},
"eu-west-1": {Latitude: 53.41291, Longitude: -8.24389, City: "", CountryCode: ""},
"eu-central-1": {Latitude: 50.1109221, Longitude: 8.6821267, City: "", CountryCode: ""},
"sa-east-1": {Latitude: -23.5505199, Longitude: -46.63330939999999, City: "", CountryCode: ""},
"ap-southeast-1": {Latitude: 1.352083, Longitude: 103.819836, City: "", CountryCode: ""},
"ca-central-1": {Latitude: 56.130366, Longitude: -106.346771, City: "", CountryCode: ""},
"ap-northeast-2": {Latitude: 37.566535, Longitude: 126.9779692, City: "", CountryCode: ""},
"us-west-2": {Latitude: 43.8041334, Longitude: -120.5542012, City: "", CountryCode: ""},
"us-gov-west-1": {Latitude: 37.09024, Longitude: -95.712891, City: "", CountryCode: ""},
"us-west-1": {Latitude: 38.8375215, Longitude: -120.8958242, City: "", CountryCode: ""},
"cn-north-1": {Latitude: 39.90419989999999, Longitude: 116.4073963, City: "", CountryCode: ""},
"ap-northeast-1": {Latitude: 35.7090259, Longitude: 139.7319925, City: "", CountryCode: ""},
"eu-west-2": {Latitude: 51.5073509, Longitude: -0.1277583, City: "", CountryCode: ""},
"eu-west-3": {Latitude: 48.856614, Longitude: 2.3522219, City: "", CountryCode: ""},
"us-east-2": {Latitude: 40.4172871, Longitude: -82.90712300000001, City: "Ohio", CountryCode: "US"},
"us-east-1": {Latitude: 37.4315734, Longitude: -78.6568942, City: "N. Virginia", CountryCode: "US"},
"us-west-1": {Latitude: 38.8375215, Longitude: -120.8958242, City: "N. California", CountryCode: "US"},
"us-west-2": {Latitude: 43.8041334, Longitude: -120.5542012, City: "Oregon", CountryCode: "US"},
"ap-south-1": {Latitude: 19.0759837, Longitude: 72.8776559, City: "Mumbai", CountryCode: "IN"},
"ap-northeast-3": {Latitude: 34.6937378, Longitude: 135.5021651, City: "Osaka-Local", CountryCode: "JP"},
"ap-northeast-2": {Latitude: 37.566535, Longitude: 126.9779692, City: "Seoul", CountryCode: "KR"},
"ap-southeast-1": {Latitude: 1.352083, Longitude: 103.819836, City: "Singapore", CountryCode: "SG"},
"ap-southeast-2": {Latitude: -33.8688197, Longitude: 151.2092955, City: "Sydney", CountryCode: "AU"},
"ap-northeast-1": {Latitude: 35.7090259, Longitude: 139.7319925, City: "Tokyo", CountryCode: "JP"},
"ca-central-1": {Latitude: 56.130366, Longitude: -106.346771, City: "Central", CountryCode: "CA"},
"cn-north-1": {Latitude: 39.90419989999999, Longitude: 116.4073963, City: "Beijing", CountryCode: "CN"},
"cn-northwest-1": {Latitude: 37.198731, Longitude: 106.1580937, City: "Ningxia", CountryCode: "CN"},
"eu-central-1": {Latitude: 50.1109221, Longitude: 8.6821267, City: "Frankfurt", CountryCode: "DE"},
"eu-west-1": {Latitude: 53.41291, Longitude: -8.24389, City: "Ireland", CountryCode: "IE"},
"eu-west-2": {Latitude: 51.5073509, Longitude: -0.1277583, City: "London", CountryCode: "GB"},
"eu-west-3": {Latitude: 48.856614, Longitude: 2.3522219, City: "Paris", CountryCode: "FR"},
"eu-north-1": {Latitude: 59.1946, Longitude: 18.47, City: "Stockholm", CountryCode: "SE"},
"sa-east-1": {Latitude: -23.5505199, Longitude: -46.63330939999999, City: "San Paulo", CountryCode: "BR"},
"us-gov-west-1": {Latitude: 37.09024, Longitude: -95.712891, City: "us-gov-west", CountryCode: "US"},
}
+15 -11
View File
@@ -7,20 +7,24 @@ import (
func init() {
type ImageListOptions struct {
Status string `help:"image status type" choices:"Creating|Available|UnAvailable|CreateFailed"`
Owner []string `help:"Owner type" choices:"amazon|self|microsoft|aws-marketplace"`
Id []string `help:"Image ID"`
Name string `help:"image name"`
Status string `help:"image status type" choices:"Creating|Available|UnAvailable|CreateFailed"`
Owner string `help:"Owner type, e.g. self, system or all" choices:"self|system|all"`
VirtualizationType string `help:"virtualization type" choices:"hvm|paravirtual"`
Id []string `help:"Image ID"`
Name string `help:"image name"`
RawOwner []string `help:"raw owner id"`
VolumeType string `help:"image volume type" choices:"gp2|io1|st1|sc1|standard"`
Latest bool `help:"show latest image only"`
}
shellutils.R(&ImageListOptions{}, "image-list", "List images", func(cli *aws.SRegion, args *ImageListOptions) error {
var owners []*string
if len(args.Owner) > 0 {
owners = make([]*string, 0)
for i := range args.Owner {
owners = append(owners, &args.Owner[i])
}
var owners []aws.TImageOwnerType
switch args.Owner {
case "self":
owners = aws.ImageOwnerSelf
case "system":
owners = aws.ImageOwnerSystem
}
images, e := cli.GetImages(aws.ImageStatusType(args.Status), owners, args.Id, args.Name)
images, e := cli.GetImages(aws.ImageStatusType(args.Status), owners, args.Id, args.Name, args.VirtualizationType, args.RawOwner, args.VolumeType, args.Latest)
if e != nil {
return e
}
+19 -20
View File
@@ -71,15 +71,12 @@ func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImag
return nil, fmt.Errorf("GetIImageById image id should not be empty")
}
parts, err := self.region.GetImages(ImageStatusType(""), nil, []string{extId}, "")
part, err := self.region.GetImage(extId)
if err != nil {
return nil, err
}
if len(parts) == 0 {
return nil, cloudprovider.ErrNotFound
}
parts[0].storageCache = self
return &parts[0], nil
part.storageCache = self
return part, nil
}
func (self *SStoragecache) GetPath() string {
@@ -91,20 +88,22 @@ func (self *SStoragecache) GetManagerId() string {
}
func (self *SStoragecache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) {
if imageId, err := self.region.createIImage(snapshotId, imageName, imageDesc); err != nil {
imageId, err := self.region.createIImage(snapshotId, imageName, imageDesc)
if err != nil {
return nil, err
} else if image, err := self.region.GetImage(imageId); err != nil {
return nil, err
} else {
image.storageCache = self
iimage := make([]cloudprovider.ICloudImage, 1)
iimage[0] = image
//todo : implement me
if err := cloudprovider.WaitStatus(iimage[0], "avaliable", 15*time.Second, 3600*time.Second); err != nil {
return nil, err
}
return iimage[0], nil
}
image, err := self.region.GetImage(imageId)
if err != nil {
return nil, err
}
image.storageCache = self
iimage := make([]cloudprovider.ICloudImage, 1)
iimage[0] = image
//todo : implement me
if err := cloudprovider.WaitStatus(iimage[0], "avaliable", 15*time.Second, 3600*time.Second); err != nil {
return nil, err
}
return iimage[0], nil
}
func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) {
@@ -131,7 +130,7 @@ func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.To
}
func (self *SStoragecache) fetchImages() error {
images, err := self.region.GetImages(ImageStatusType(""), []*string{&ImageOwnerSelf, &ImageOwnerAmazone}, nil, "")
images, err := self.region.GetImages("", ImageOwnerSelfSystem, nil, "", "hvm", nil, "", true)
if err != nil {
return err
}
@@ -213,7 +212,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
// check image name, avoid name conflict
for {
_, err = self.region.GetImageByName(imageName)
_, err = self.region.GetImageByName(imageName, ImageOwnerSelf)
if err != nil {
if err == cloudprovider.ErrNotFound {
break