diff --git a/Gopkg.lock b/Gopkg.lock index bcd51fb9d7..1b124ff00a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -884,6 +884,18 @@ pruneopts = "UT" revision = "d188e65d659ef53fcdb0691c12f1bba64928b649" +[[projects]] + branch = "master" + digest = "1:119cf7d2c3bc4a3c675e8c30cadf00c8c2ab34a20ba373211dd3a7d66f3e5f32" + name = "github.com/tredoe/osutil" + packages = [ + "user/crypt", + "user/crypt/common", + "user/crypt/sha512_crypt", + ] + pruneopts = "UT" + revision = "7d3ee1afa71c90fd1514c8f557ae6c5f414208eb" + [[projects]] digest = "1:98e5cda86f67cd1ac95389d98670b66dea8cae480fe6292b83bccccfe60b4106" name = "github.com/ugorji/go" @@ -1223,11 +1235,11 @@ [[projects]] branch = "master" - digest = "1:49ffc35ec8d3f7789393cd132acd359e8ac1f5d38c7a2b91c484b041840c62c0" + digest = "1:36db56d9ed25cc9cbd34d5553c14d5e1d4ef6501feaddb752c66aeb294e481e6" name = "yunion.io/x/jsonutils" packages = ["."] pruneopts = "UT" - revision = "d1290e94d4753c1748fc7c89f472a523cc0a5c08" + revision = "191bb9c0726440a0b239c344c70b9df567536302" [[projects]] branch = "master" @@ -1362,6 +1374,7 @@ "github.com/moul/http2curl", "github.com/serialx/hashring", "github.com/stretchr/testify/assert", + "github.com/tredoe/osutil/user/crypt/sha512_crypt", "github.com/vmware/govmomi", "github.com/vmware/govmomi/object", "github.com/vmware/govmomi/property", diff --git a/Gopkg.toml b/Gopkg.toml index 2feb622afe..b70428fdfc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -109,3 +109,11 @@ [[constraint]] name = "github.com/360EntSecGroup-Skylar/excelize" version = "v1.3.0" + +[[constraint]] + branch = "master" + name = "github.com/tredoe/osutil" + +[[constraint]] + branch = "master" + name = "golang.org/x/crypto" diff --git a/Makefile b/Makefile index ee09d3df04..372dde24df 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ install: prepare_dir done -build: prepare_dir +build: prepare_dir fmt @for PKG in $(CMDS); do \ echo build $$PKG; \ $(GO_BUILD) -o $(BIN_DIR)/`basename $${PKG}` $$PKG; \ @@ -58,11 +58,11 @@ test: prepare_dir done -cmd/%: prepare_dir +cmd/%: prepare_dir fmt $(GO_BUILD) -o $(BIN_DIR)/$(shell basename $@) $(REPO_PREFIX)/$@ -pkg/%: prepare_dir +pkg/%: prepare_dir fmt $(GO_INSTALL) $(REPO_PREFIX)/$@ diff --git a/cmd/climc/shell/account_balances.go b/cmd/climc/shell/account_balances.go index 4346049bcb..377209a4c1 100644 --- a/cmd/climc/shell/account_balances.go +++ b/cmd/climc/shell/account_balances.go @@ -14,7 +14,7 @@ func init() { StatMonth string `help:"stat_month of the query"` StartDate string `help:"start_date of the query"` EndDate string `help:"end_date of the query"` - QueryType string `help:"query_type of the query"` + QueryType string `help:"query_type of the query"` Platform string `help:"platform of the query"` ProjectId string `help:"project_id of the query"` } diff --git a/cmd/climc/shell/ansible.go b/cmd/climc/shell/ansible.go index 38ba093498..69dbec5ea7 100644 --- a/cmd/climc/shell/ansible.go +++ b/cmd/climc/shell/ansible.go @@ -11,11 +11,11 @@ import ( ) type AnsibleHostsOptions struct { - List bool `help:"List all ansible inventory"` - Host string `help:"List of a host"` + List bool `help:"List all ansible inventory"` + Host string `help:"List of a host"` PrivateKey string `help:"path to private key to use for ansible"` - Port int `help:"optional port, if port is not 22"` - User string `help:"username to try"` + Port int `help:"optional port, if port is not 22"` + User string `help:"username to try"` UserBecome string `help:"username to sudo"` } @@ -35,7 +35,6 @@ func serverGetNameIP(srv jsonutils.JSONObject) (string, string, error) { return host, ipList[0], nil } - func doList(s *mcclient.ClientSession, args *AnsibleHostsOptions) error { hostVars := jsonutils.NewDict() hosts := jsonutils.NewArray() @@ -86,7 +85,6 @@ func doList(s *mcclient.ClientSession, args *AnsibleHostsOptions) error { return nil } - func doHost(s *mcclient.ClientSession, host string, args *AnsibleHostsOptions) error { srv, err := modules.Servers.Get(s, host, nil) if err != nil { diff --git a/cmd/climc/shell/capabilities.go b/cmd/climc/shell/capabilities.go index a1a02ff842..d51cfe763b 100644 --- a/cmd/climc/shell/capabilities.go +++ b/cmd/climc/shell/capabilities.go @@ -7,7 +7,6 @@ import ( func init() { type CapabilitiesOptions struct { - } R(&CapabilitiesOptions{}, "capabilities", "Show backend capabilities", func(s *mcclient.ClientSession, args *CapabilitiesOptions) error { result, err := modules.Capabilities.List(s, nil) diff --git a/cmd/climc/shell/cloudaccounts.go b/cmd/climc/shell/cloudaccounts.go index f1f0b83dde..3de4da23c8 100644 --- a/cmd/climc/shell/cloudaccounts.go +++ b/cmd/climc/shell/cloudaccounts.go @@ -38,6 +38,7 @@ func init() { AccessURL string `helo:"hello" metavar:"Azure choices: "` Desc string `help:"Description"` Enabled bool `help:"Enabled the account automatically"` + Import bool `help:"Import all sub account automatically"` } R(&CloudaccountCreateOptions{}, "cloud-account-create", "Create a cloud account", func(s *mcclient.ClientSession, args *CloudaccountCreateOptions) error { params := jsonutils.NewDict() @@ -48,6 +49,9 @@ func init() { if args.Enabled { params.Add(jsonutils.JSONTrue, "enabled") } + if args.Import { + params.Add(jsonutils.JSONTrue, "import") + } if len(args.AccessURL) > 0 { params.Add(jsonutils.NewString(args.AccessURL), "access_url") } @@ -147,6 +151,19 @@ func init() { return nil }) + type CloudaccountImportOptions struct { + ID string `help:"ID or Name of cloud account"` + Enabled bool `help:"Import sub accounts with enabled status"` + } + R(&CloudaccountImportOptions{}, "cloud-account-import", "Import sub cloud account", func(s *mcclient.ClientSession, args *CloudaccountImportOptions) error { + result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "import", jsonutils.Marshal(map[string]bool{"enabled": args.Enabled})) + if err != nil { + return err + } + printObject(result) + return nil + }) + type CloudaccountUpdateCredentialOptions struct { ID string `help:"ID or Name of cloud account"` ACCOUNT string `help:"new account"` diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 716d557c08..e802a77279 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -74,12 +74,18 @@ func init() { return nil }) - R(&DiskDetailOptions{}, "disk-delete", "Delete a disk", func(s *mcclient.ClientSession, args *DiskDetailOptions) error { - disk, e := modules.Disks.Delete(s, args.ID, nil) - if e != nil { - return e + type DiskDeleteOptions struct { + ID []string `help:"ID of disks to delete" metavar:"DISK"` + OverridePendingDelete bool `help:"Delete disk directly instead of pending delete"` + } + + R(&DiskDeleteOptions{}, "disk-delete", "Delete a disk", func(s *mcclient.ClientSession, args *DiskDeleteOptions) error { + params := jsonutils.NewDict() + if args.OverridePendingDelete { + params.Add(jsonutils.JSONTrue, "override_pending_delete") } - printObject(disk) + ret := modules.Disks.BatchDeleteWithParam(s, args.ID, params, nil) + printBatchResults(ret, modules.Disks.GetColumns(s)) return nil }) @@ -202,12 +208,16 @@ func init() { return nil }) type DiskResetOptions struct { - DISK string `help:"ID or name of disk"` - SNAPSHOT string `help:"snapshots ID of disk` + DISK string `help:"ID or name of disk"` + SNAPSHOT string `help:"snapshots ID of disk` + AutoStart bool `help:"Autostart guest"` } R(&DiskResetOptions{}, "disk-reset", "Resize a disk", func(s *mcclient.ClientSession, args *DiskResetOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.SNAPSHOT), "snapshot_id") + if args.AutoStart { + params.Add(jsonutils.JSONTrue, "auto_start") + } disk, err := modules.Disks.PerformAction(s, args.DISK, "disk-reset", params) if err != nil { return err diff --git a/cmd/climc/shell/isolatedevices.go b/cmd/climc/shell/isolatedevices.go index 48dc75c5e9..72515beb51 100644 --- a/cmd/climc/shell/isolatedevices.go +++ b/cmd/climc/shell/isolatedevices.go @@ -13,7 +13,7 @@ func init() { Unused bool `help:"Only show unused devices"` Gpu bool `help:"Only show gpu devices"` Host string `help:"Host ID or Name"` - Zone string `help:"Zone ID or Name"` + Zone string `help:"Zone ID or Name"` } R(&DeviceListOptions{}, "isolated-device-list", "List isolated devices like GPU", func(s *mcclient.ClientSession, args *DeviceListOptions) error { var params *jsonutils.JSONDict diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index d6957bfb8e..0f947d663a 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -3,6 +3,7 @@ package shell import ( "fmt" + "io/ioutil" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -59,6 +60,14 @@ func init() { params.Add(jsonutils.JSONFalse, "reset_password") } + if len(opts.UserDataFile) > 0 { + userdata, err := ioutil.ReadFile(opts.UserDataFile) + if err != nil { + return err + } + params.Add(jsonutils.NewString(string(userdata)), "user_data") + } + count := options.IntV(opts.Count) if options.BoolV(opts.DryRun) { results, err := modules.SchedManager.DoScheduleListResult(s, params, count) @@ -88,12 +97,22 @@ func init() { return nil }) - R(&options.ServerIdOptions{}, "server-logininfo", "Get login info of a server", func(s *mcclient.ClientSession, opts *options.ServerIdOptions) error { + R(&options.ServerLoginInfoOptions{}, "server-logininfo", "Get login info of a server", func(s *mcclient.ClientSession, opts *options.ServerLoginInfoOptions) error { srvid, e := modules.Servers.GetId(s, opts.ID, nil) if e != nil { return e } - i, e := modules.Servers.GetLoginInfo(s, srvid, nil) + + params := jsonutils.NewDict() + if len(opts.Key) > 0 { + privateKey, e := ioutil.ReadFile(opts.Key) + if e != nil { + return e + } + params.Add(jsonutils.NewString(string(privateKey)), "private_key") + } + + i, e := modules.Servers.GetLoginInfo(s, srvid, params) if e != nil { return e } @@ -467,4 +486,23 @@ func init() { printObject(result) return nil }) + + type ServerUserDataOptions struct { + ID string `help:"ID or name of server"` + FILE string `help:"Path to user data file"` + } + R(&ServerUserDataOptions{}, "server-set-user-data", "Update server user_data", func(s *mcclient.ClientSession, args *ServerUserDataOptions) error { + params := jsonutils.NewDict() + content, err := ioutil.ReadFile(args.FILE) + if err != nil { + return err + } + params.Add(jsonutils.NewString(string(content)), "user_data") + result, err := modules.Servers.PerformAction(s, args.ID, "user-data", params) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/cmd/climc/shell/specs.go b/cmd/climc/shell/specs.go index 68f8f1e9aa..011ba7bee7 100644 --- a/cmd/climc/shell/specs.go +++ b/cmd/climc/shell/specs.go @@ -15,7 +15,7 @@ func init() { Model string `help:"Specified model specs" choices:"hosts|isolated_devices|guests"` HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv"` Gpu bool `help:"Only show gpu devices"` - Zone string `help:"Filter by zone id or name"` + Zone string `help:"Filter by zone id or name"` } R(&ListOptions{}, "spec", "List all kinds of model specs", func(s *mcclient.ClientSession, args *ListOptions) error { var params *jsonutils.JSONDict diff --git a/cmd/climc/shell/sshkeypairs.go b/cmd/climc/shell/sshkeypairs.go new file mode 100644 index 0000000000..d68aa940b5 --- /dev/null +++ b/cmd/climc/shell/sshkeypairs.go @@ -0,0 +1,42 @@ +package shell + +import ( + "fmt" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + type SshkeypairQueryOptions struct { + Project string `help:"get keypair for specific project"` + Admin bool `help:"get admin keypair, sysadmin ONLY option"` + } + R(&SshkeypairQueryOptions{}, "sshkeypair-show", "Get ssh keypairs", func(s *mcclient.ClientSession, args *SshkeypairQueryOptions) error { + query := jsonutils.NewDict() + if args.Admin { + query.Add(jsonutils.JSONTrue, "admin") + } + var keys jsonutils.JSONObject + if len(args.Project) == 0 { + listResult, err := modules.Sshkeypairs.List(s, query) + if err != nil { + return err + } + keys = listResult.Data[0] + } else { + result, err := modules.Sshkeypairs.GetById(s, args.Project, query) + if err != nil { + return err + } + keys = result + } + privKey, _ := keys.GetString("private_key") + pubKey, _ := keys.GetString("public_key") + + fmt.Print(privKey) + fmt.Print(pubKey) + + return nil + }) +} diff --git a/cmd/climc/shell/utils.go b/cmd/climc/shell/utils.go index b66d7ee548..67061edbe6 100644 --- a/cmd/climc/shell/utils.go +++ b/cmd/climc/shell/utils.go @@ -5,8 +5,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/excelutils" + "yunion.io/x/onecloud/pkg/util/printutils" ) func printList(list *modules.ListResult, columns []string) { @@ -35,7 +35,7 @@ func exportList(list *modules.ListResult, file string, exportKeys string, export if len(exportKeys) > 0 { keys = strings.Split(exportKeys, ",") texts = strings.Split(exportTexts, ",") - }else { + } else { keys = columns texts = columns } diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index 0297e57216..7168d07723 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -9,7 +9,7 @@ import ( "yunion.io/x/sqlchemy" ) -func fetchById(manager IModelManager, idStr string) (IModel, error) { +func FetchById(manager IModelManager, idStr string) (IModel, error) { q := manager.Query() q = manager.FilterById(q, idStr) count := q.Count() @@ -31,7 +31,11 @@ func fetchById(manager IModelManager, idStr string) (IModel, error) { } } -func fetchByName(manager IModelManager, owner string, idStr string) (IModel, error) { +func FetchByName(manager IModelManager, userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + var owner string + if userCred != nil { + owner = manager.GetOwnerId(userCred) + } q := manager.Query() q = manager.FilterByName(q, idStr) q = manager.FilterByOwner(q, owner) @@ -54,10 +58,10 @@ func fetchByName(manager IModelManager, owner string, idStr string) (IModel, err } } -func fetchByIdOrName(manager IModelManager, ownerProjId string, idStr string) (IModel, error) { - obj, err := fetchById(manager, idStr) +func FetchByIdOrName(manager IModelManager, userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + obj, err := FetchById(manager, idStr) if err == sql.ErrNoRows { - return fetchByName(manager, ownerProjId, idStr) + return FetchByName(manager, userCred, idStr) } else { return obj, err } diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 4b0a7a3ac5..202bfc4a26 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -39,12 +39,12 @@ type IModelManager interface { FilterByName(q *sqlchemy.SQuery, name string) *sqlchemy.SQuery FilterByOwner(q *sqlchemy.SQuery, owner string) *sqlchemy.SQuery - GetOwnerId(userCred mcclient.TokenCredential) string + GetOwnerId(userCred mcclient.IIdentityProvider) string // RawFetchById(idStr string) (IModel, error) FetchById(idStr string) (IModel, error) - FetchByName(ownerProjId string, idStr string) (IModel, error) - FetchByIdOrName(ownerProjId string, idStr string) (IModel, error) + FetchByName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) + FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) // create hooks AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 4e17ff01e9..b06106bccb 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -108,7 +108,7 @@ func (manager *SModelBaseManager) FilterByOwner(q *sqlchemy.SQuery, owner string return q } -func (manager *SModelBaseManager) GetOwnerId(userCred mcclient.TokenCredential) string { +func (manager *SModelBaseManager) GetOwnerId(userCred mcclient.IIdentityProvider) string { return "" } @@ -116,11 +116,11 @@ func (manager *SModelBaseManager) FetchById(idStr string) (IModel, error) { return nil, sql.ErrNoRows } -func (manager *SModelBaseManager) FetchByName(ownerProjId string, idStr string) (IModel, error) { +func (manager *SModelBaseManager) FetchByName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { return nil, sql.ErrNoRows } -func (manager *SModelBaseManager) FetchByIdOrName(ownerProjId string, idStr string) (IModel, error) { +func (manager *SModelBaseManager) FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { return nil, sql.ErrNoRows } diff --git a/pkg/cloudcommon/db/standalone.go b/pkg/cloudcommon/db/standalone.go index d7d2b74a14..e97b61e0d6 100644 --- a/pkg/cloudcommon/db/standalone.go +++ b/pkg/cloudcommon/db/standalone.go @@ -70,15 +70,15 @@ func (manager *SStandaloneResourceBaseManager) ValidateName(name string) error { } func (manager *SStandaloneResourceBaseManager) FetchById(idStr string) (IModel, error) { - return fetchById(manager, idStr) + return FetchById(manager, idStr) } -func (manager *SStandaloneResourceBaseManager) FetchByName(ownerProjId string, idStr string) (IModel, error) { - return fetchByName(manager, ownerProjId, idStr) +func (manager *SStandaloneResourceBaseManager) FetchByName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + return FetchByName(manager, userCred, idStr) } -func (manager *SStandaloneResourceBaseManager) FetchByIdOrName(ownerProjId string, idStr string) (IModel, error) { - return fetchByIdOrName(manager, ownerProjId, idStr) +func (manager *SStandaloneResourceBaseManager) FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + return FetchByIdOrName(manager, userCred, idStr) } func (manager *SStandaloneResourceBaseManager) FetchByExternalId(idStr string) (IStandaloneModel, error) { diff --git a/pkg/cloudcommon/db/tenantcache.go b/pkg/cloudcommon/db/tenantcache.go index 9fdf0866ea..bb756a7b63 100644 --- a/pkg/cloudcommon/db/tenantcache.go +++ b/pkg/cloudcommon/db/tenantcache.go @@ -35,7 +35,7 @@ func init() { } func (manager *STenantCacheManager) FetchTenantByIdOrName(ctx context.Context, idStr string) (*STenant, error) { - tenant, err := manager.FetchByIdOrName("", idStr) + tenant, err := manager.FetchByIdOrName(nil, idStr) if err != nil { if err == sql.ErrNoRows { return manager.fetchTenantFromKeystone(ctx, idStr) @@ -63,7 +63,7 @@ func (manager *STenantCacheManager) FetchTenantById(ctx context.Context, idStr s } func (manager *STenantCacheManager) FetchTenantByName(ctx context.Context, idStr string) (*STenant, error) { - tenant, err := manager.FetchByName("", idStr) + tenant, err := manager.FetchByName(nil, idStr) if err != nil { if err == sql.ErrNoRows { return manager.fetchTenantFromKeystone(ctx, idStr) diff --git a/pkg/cloudcommon/db/usercache.go b/pkg/cloudcommon/db/usercache.go index 917b5c8754..1fe3f560c5 100644 --- a/pkg/cloudcommon/db/usercache.go +++ b/pkg/cloudcommon/db/usercache.go @@ -29,7 +29,7 @@ func init() { } func (manager *SUserCacheManager) FetchUserByIdOrName(idStr string) (*SUser, error) { - obj, err := manager.SKeystoneCacheObjectManager.FetchByIdOrName("", idStr) + obj, err := manager.SKeystoneCacheObjectManager.FetchByIdOrName(nil, idStr) if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (manager *SUserCacheManager) FetchUserById(idStr string) (*SUser, error) { } func (manager *SUserCacheManager) FetchUserByName(idStr string) (*SUser, error) { - obj, err := manager.SKeystoneCacheObjectManager.FetchByName("", idStr) + obj, err := manager.SKeystoneCacheObjectManager.FetchByName(nil, idStr) if err != nil { return nil, err } diff --git a/pkg/cloudcommon/db/virtualjointbase.go b/pkg/cloudcommon/db/virtualjointbase.go index 63e10c4c69..23565cc777 100644 --- a/pkg/cloudcommon/db/virtualjointbase.go +++ b/pkg/cloudcommon/db/virtualjointbase.go @@ -110,7 +110,7 @@ func (manager *SVirtualJointResourceBaseManager) ListItemFilter(ctx context.Cont } tenant, _ := query.GetString("tenant") if len(tenant) > 0 { - tc, _ := TenantCacheManager.FetchByIdOrName("", tenant) + tc, _ := TenantCacheManager.FetchTenantByIdOrName(ctx, tenant) if tc == nil { return nil, httperrors.NewTenantNotFoundError(fmt.Sprintf("tenant %s not found", tenant)) } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 6bf85de87f..16a9b44572 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -56,15 +56,15 @@ func (manager *SVirtualResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, ow return q } -func (manager *SVirtualResourceBaseManager) FetchByName(ownerProjId string, idStr string) (IModel, error) { - return fetchByName(manager, ownerProjId, idStr) +func (manager *SVirtualResourceBaseManager) FetchByName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + return FetchByName(manager, userCred, idStr) } -func (manager *SVirtualResourceBaseManager) FetchByIdOrName(ownerProjId string, idStr string) (IModel, error) { - return fetchByIdOrName(manager, ownerProjId, idStr) +func (manager *SVirtualResourceBaseManager) FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (IModel, error) { + return FetchByIdOrName(manager, userCred, idStr) } -func (manager *SVirtualResourceBaseManager) GetOwnerId(userCred mcclient.TokenCredential) string { +func (manager *SVirtualResourceBaseManager) GetOwnerId(userCred mcclient.IIdentityProvider) string { return userCred.GetProjectId() } diff --git a/pkg/cloudcommon/validators/choices.go b/pkg/cloudcommon/validators/choices.go index 83188a7ef0..1ad80cba49 100644 --- a/pkg/cloudcommon/validators/choices.go +++ b/pkg/cloudcommon/validators/choices.go @@ -23,7 +23,7 @@ func (cs Choices) Has(choice string) bool { func (cs Choices) String() string { choices := make([]string, len(cs)) i := 0 - for choice, _ := range cs { + for choice := range cs { choices[i] = choice i++ } diff --git a/pkg/cloudcommon/validators/validators.go b/pkg/cloudcommon/validators/validators.go index 163f04a549..a612b150ed 100644 --- a/pkg/cloudcommon/validators/validators.go +++ b/pkg/cloudcommon/validators/validators.go @@ -322,11 +322,24 @@ type ValidatorModelIdOrName struct { Validator ModelKeyword string ProjectId string + UserId string ModelManager db.IModelManager Model db.IModel modelIdKey string } +func (v *ValidatorModelIdOrName) GetProjectId() string { + return v.ProjectId +} + +func (v *ValidatorModelIdOrName) GetUserId() string { + return v.UserId +} + +func (v *ValidatorModelIdOrName) GetTenantId() string { + return v.ProjectId +} + func (v *ValidatorModelIdOrName) getValue() interface{} { return v.Model } @@ -361,7 +374,7 @@ func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error { return newModelManagerError(v.ModelKeyword) } v.ModelManager = modelManager - model, err := modelManager.FetchByIdOrName(v.ProjectId, modelIdOrName) + model, err := modelManager.FetchByIdOrName(v, modelIdOrName) if err != nil { return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err) } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f93abdc326..912009d0a3 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -135,7 +135,7 @@ type ICloudHost interface { GetManagerId() string CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, - passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string) (ICloudVM, error) + passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string) (ICloudVM, error) } type ICloudVM interface { @@ -174,6 +174,8 @@ type ICloudVM interface { UpdateVM(name string) error + UpdateUserData(userData string) error + RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index d081599979..468ac48afc 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -139,6 +139,12 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu publicKey, _ := config.GetString("public_key") + adminPublicKey, _ := config.GetString("admin_public_key") + projectPublicKey, _ := config.GetString("project_public_key") + oUserData, _ := config.GetString("user_data") + + userData := generateUserData(adminPublicKey, projectPublicKey, oUserData) + resetPassword := jsonutils.QueryBoolean(config, "reset_password", false) passwd, _ := config.GetString("password") if resetPassword && len(passwd) == 0 { @@ -176,7 +182,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu } iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgrpId) + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgrpId, userData) if err != nil { return nil, err } @@ -282,6 +288,13 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + if len(userData) > 0 { + err := iVM.UpdateUserData(userData) + if err != nil { + log.Errorf("update userdata fail %s", err) + } + } + err := iVM.DeployVM(name, passwd, publicKey, deleteKeypair, description) if err != nil { return nil, err @@ -306,6 +319,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return data, nil }) } else if action == "rebuild" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) if err != nil || iVM == nil { log.Errorf("cannot find vm %s", err) @@ -313,6 +327,13 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + if len(userData) > 0 { + err := iVM.UpdateUserData(userData) + if err != nil { + log.Errorf("update userdata fail %s", err) + } + } + diskId, err := iVM.RebuildRoot(desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) if err != nil { return nil, err diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 90cfe6d6ba..c557b70bc5 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -70,6 +70,13 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue if resetPassword && len(passwd) == 0 { passwd = seclib2.RandomPassword2(12) } + + adminPublicKey, _ := config.GetString("admin_public_key") + projectPublicKey, _ := config.GetString("project_public_key") + oUserData, _ := config.GetString("user_data") + + userData := generateUserData(adminPublicKey, projectPublicKey, oUserData) + desc := SManagedVMCreateConfig{} if err := config.Unmarshal(&desc, "desc"); err != nil { return err @@ -98,7 +105,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue } if iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgrpId); err != nil { + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgrpId, userData); err != nil { return nil, err } else { log.Debugf("VMcreated %s, wait status running ...", iVM.GetGlobalId()) diff --git a/pkg/compute/guestdrivers/userdata.go b/pkg/compute/guestdrivers/userdata.go new file mode 100644 index 0000000000..0f711a2099 --- /dev/null +++ b/pkg/compute/guestdrivers/userdata.go @@ -0,0 +1,32 @@ +package guestdrivers + +import ( + "yunion.io/x/onecloud/pkg/util/ansible" + "yunion.io/x/onecloud/pkg/util/cloudinit" +) + +func generateUserData(adminPublicKey, projectPublicKey, oUserData string) string { + var oCloudConfig *cloudinit.SCloudConfig = nil + + if len(oUserData) > 0 { + oCloudConfig, _ = cloudinit.ParseUserDataBase64(oUserData) + } + + ansibleUser := cloudinit.NewUser(ansible.PUBLIC_CLOUD_ANSIBLE_USER) + ansibleUser.SshKey(adminPublicKey).SshKey(projectPublicKey).SudoPolicy(cloudinit.USER_SUDO_NOPASSWD) + + cloudConfig := cloudinit.SCloudConfig{ + DisableRoot: 0, + SshPwauth: 1, + + Users: []cloudinit.SUser{ + ansibleUser, + }, + } + + if oCloudConfig != nil { + cloudConfig.Merge(oCloudConfig) + } + + return cloudConfig.UserDataBase64() +} diff --git a/pkg/compute/handlers.go b/pkg/compute/handlers.go index c2e3a05ab5..b3ba3f50a9 100644 --- a/pkg/compute/handlers.go +++ b/pkg/compute/handlers.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/onecloud/pkg/compute/capabilities" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/specs" + "yunion.io/x/onecloud/pkg/compute/sshkeys" "yunion.io/x/onecloud/pkg/compute/usages" ) @@ -21,6 +22,7 @@ func InitHandlers(app *appsrv.Application) { usages.AddUsageHandler("", app) capabilities.AddCapabilityHandler("", app) specs.AddSpecHandler("", app) + sshkeys.AddSshKeysHandler("", app) taskman.AddTaskHandler("", app) diff --git a/pkg/compute/models/billingresource.go b/pkg/compute/models/billingresource.go index 612919afdc..8633f4b3a1 100644 --- a/pkg/compute/models/billingresource.go +++ b/pkg/compute/models/billingresource.go @@ -4,12 +4,12 @@ import "time" const ( BILLING_TYPE_POSTPAID = "postpaid" - BILLING_TYPE_PREPAID = "prepaid" + BILLING_TYPE_PREPAID = "prepaid" ) type SBillingResourceBase struct { - BillingType string `width:"36" charset:"ascii" nullable:"true" default:"postpaid" list:"user" create:"optional"` - ExpiredAt time.Time `nullable:"true" list:"user" create:"optional"` + BillingType string `width:"36" charset:"ascii" nullable:"true" default:"postpaid" list:"user" create:"optional"` + ExpiredAt time.Time `nullable:"true" list:"user" create:"optional"` } func (self *SBillingResourceBase) GetChargeType() string { diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index ca33d54bbb..9cd3c5e66d 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -14,6 +14,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -49,7 +50,7 @@ type SCloudaccount struct { Provider string `width:"64" charset:"ascii" list:"admin" create:"admin_required"` } -func (self *SCloudaccount) getCloudproviders() []SCloudprovider { +func (self *SCloudaccount) GetCloudproviders() []SCloudprovider { cloudproviders := []SCloudprovider{} q := CloudproviderManager.Query().Equals("cloudaccount_id", self.Id) if err := db.FetchModelObjects(CloudproviderManager, q, &cloudproviders); err != nil { @@ -62,7 +63,7 @@ func (self *SCloudaccount) ValidateDeleteCondition(ctx context.Context) error { if self.Enabled { return httperrors.NewInvalidStatusError("account is enabled") } - if len(self.getCloudproviders()) > 0 { + if len(self.GetCloudproviders()) > 0 { return httperrors.NewNotEmptyError("Not an empty cloud account") } return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx) @@ -95,7 +96,7 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr return nil, httperrors.NewConflictError("The account has been registered") } - if subAccount, err := getSubAccounts(name, url, account, secret, provider); err != nil { + if subAccount, err := GetSubAccounts(name, url, account, secret, provider); err != nil { return nil, err } else if accounts, err := subAccount.GetArray("data"); err != nil { return nil, err @@ -108,8 +109,8 @@ func (self *SCloudaccountManager) ValidateCreateData(ctx context.Context, userCr func (self *SCloudaccount) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data) self.savePassword(self.Secret) - - if subAccounts, err := data.GetArray("accounts"); err == nil && len(subAccounts) > 0 { + _import, _ := data.Bool("import") + if subAccounts, err := data.GetArray("accounts"); err == nil && _import && len(subAccounts) > 0 { for _, subAccount := range subAccounts { name, _ := subAccount.GetString("name") account, _ := subAccount.GetString("account") @@ -183,7 +184,7 @@ func (self *SCloudaccount) PerformSync(ctx context.Context, userCred mcclient.To return nil, httperrors.NewInputParameterError("invalid input %s", err) } if self.CanSync() || syncRange.Force { - err = self.startSyncCloudProviderInfoTask(ctx, userCred, nil, "") + err = self.startSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "") } return nil, err } @@ -220,6 +221,18 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred changed = true } if (len(account) > 0 && account != self.Account) || (len(accessUrl) > 0 && accessUrl != self.AccessUrl) { + for _, cloudprovider := range self.GetCloudproviders() { + if cloudprovider.Account == self.Account { + if len(account) > 0 { + if _, err = cloudprovider.GetModelManager().TableSpec().Update(&cloudprovider, func() error { + cloudprovider.Account = account + return nil + }); err != nil { + return nil, err + } + } + } + } _, err = self.GetModelManager().TableSpec().Update(self, func() error { if len(account) > 0 { self.Account = account @@ -242,15 +255,16 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred } func (self *SCloudaccount) startSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error { - for _, cloudprovider := range self.getCloudproviders() { - params := jsonutils.NewDict() - if syncRange != nil { - params.Add(jsonutils.Marshal(syncRange), "sync_range") - } - if cloudprovider.Enabled { - cloudprovider.startSyncCloudProviderInfoTask(ctx, userCred, nil, "") - } + params := jsonutils.NewDict() + if syncRange != nil { + params.Add(jsonutils.Marshal(syncRange), "sync_range") } + task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountSyncInfoTask", self, userCred, params, parentTaskId, "", nil) + if err != nil { + log.Errorf("startSyncCloudAccountInfoTask newTask error %s", err) + return err + } + task.ScheduleRun(nil) return nil } @@ -288,10 +302,16 @@ func (self *SCloudaccount) AllowPerformImport(ctx context.Context, userCred mccl return userCred.IsSystemAdmin() } -func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SCloudaccount) GetSubAccounts() (jsonutils.JSONObject, error) { if secret, err := self.getPassword(); err != nil { return nil, err - } else if subAccounts, err := getSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider); err != nil { + } else { + return GetSubAccounts(self.Name, self.AccessUrl, self.Account, secret, self.Provider) + } +} + +func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if subAccounts, err := self.GetSubAccounts(); err != nil { return nil, err } else if accounts, err := subAccounts.GetArray("data"); err != nil { return nil, err @@ -321,74 +341,16 @@ func (self *SCloudaccount) PerformImport(ctx context.Context, userCred mcclient. return nil, err } if enabled { - newCloudprovider.startSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "") + newCloudprovider.StartSyncCloudProviderInfoTask(ctx, userCred, &SSyncRange{FullSync: true}, "") } } } } } - return jsonutils.NewDict(), nil + return nil, nil } -// func (self *SCloudaccount) PerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { -// if !self.Enabled { -// return nil, httperrors.NewInvalidStatusError("Account disabled") -// } -// if provider, err := self.GetDriver(); err != nil { -// return nil, err -// } else if _subAccounts, err := provider.GetSubAccounts(); err != nil { -// return nil, err -// } else { -// result := jsonutils.NewDict() -// data := jsonutils.NewArray() -// accounts := []string{} -// for _, account := range self.getCloudproviders() { -// accounts = append(accounts, account.Account) -// _account := jsonutils.NewDict() -// _account.Add(jsonutils.NewString(account.Account), "account") -// _account.Add(jsonutils.NewString(account.Name), "name") -// _account.Add(jsonutils.JSONTrue, "exist") -// data.Add(_account) -// } -// if _subAccounts != nil { -// if subAccounts, err := _subAccounts.GetArray("data"); err != nil { -// return nil, err -// } else { -// for _, subAccount := range subAccounts { -// if account, err := subAccount.GetString("account"); err != nil { -// log.Errorf("Get subAccount error %v", err) -// } else if !utils.IsInStringArray(account, accounts) { -// _account := subAccount.(*jsonutils.JSONDict) -// _account.Add(jsonutils.JSONFalse, "exist") -// data.Add(_account) -// } -// } -// } -// } -// result.Add(data, "data") -// result.Add(jsonutils.NewInt(int64(data.Length())), "total") -// return result, nil -// } -// } - -// func (manager *SCloudaccountManager) AllowPerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { -// return userCred.IsSystemAdmin() -// } - -// func (manager *SCloudaccountManager) PerformGetSubAccounts(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { -// name, _ := data.GetString("name") -// accessUrl, _ := data.GetString("access_url") -// account, _ := data.GetString("account") -// secret, _ := data.GetString("secret") -// _provider, _ := data.GetString("provider") -// if provider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, _provider); err != nil { -// return nil, err -// } else { -// return provider.GetSubAccounts() -// } -// } - -func getSubAccounts(name, accessUrl, account, secret, provider string) (jsonutils.JSONObject, error) { +func GetSubAccounts(name, accessUrl, account, secret, provider string) (jsonutils.JSONObject, error) { if provider, err := cloudprovider.GetProvider("", name, accessUrl, account, secret, provider); err != nil { return nil, err } else { @@ -403,27 +365,28 @@ func (self *SCloudaccount) SaveSysInfo(info jsonutils.JSONObject) { }) } -func (manager *SCloudaccountManager) FetchCloudproviderById(providerId string) *SCloudprovider { - providerObj, err := manager.FetchById(providerId) +func (manager *SCloudaccountManager) FetchCloudaccountById(accountId string) *SCloudaccount { + providerObj, err := manager.FetchById(accountId) if err != nil { log.Errorf("%s", err) return nil } - return providerObj.(*SCloudprovider) + return providerObj.(*SCloudaccount) } -func (manager *SCloudaccountManager) FetchCloudproviderByIdOrName(providerId string) *SCloudprovider { - providerObj, err := manager.FetchByIdOrName("", providerId) +func (manager *SCloudaccountManager) FetchCloudaccountByIdOrName(accountId string) *SCloudaccount { + providerObj, err := manager.FetchByIdOrName(nil, accountId) if err != nil { if err != sql.ErrNoRows { log.Errorf("%s", err) } return nil } - return providerObj.(*SCloudprovider) + return providerObj.(*SCloudaccount) } func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + extra.Add(jsonutils.Marshal(self.GetCloudproviders()), "accounts") return extra } @@ -446,7 +409,7 @@ func (manager *SCloudaccountManager) InitializeData() error { } newAccounts := map[string]string{} for _, cloudprovider := range cloudproviders { - Account, providerAccount, providerName := cloudprovider.Account, cloudprovider.Account, cloudprovider.Name + Account, providerAccount, providerName := cloudprovider.Account, "", cloudprovider.Name if cloudprovider.Provider == CLOUD_PROVIDER_AZURE { if accountInfo := strings.Split(cloudprovider.Account, "/"); len(accountInfo) == 2 { if _, ok := newAccounts[accountInfo[0]]; ok { diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index aeba260c8e..6f7776c911 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -4,12 +4,16 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" @@ -51,7 +55,9 @@ type SCloudprovider struct { Account string `width:"128" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(64, charset='ascii'), nullable=False) Secret string `width:"256" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // Column(VARCHAR(256, charset='ascii'), nullable=False) - CloudaccountId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" key_index:"true""` + CloudaccountId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" key_index:"true"` + + ProjectId string `name:"tenant_id" width:"128" charset:"ascii" nullable:"true" list:"admin"` LastSync time.Time `get:"admin" list:"admin"` // = Column(DateTime, nullable=True) @@ -122,6 +128,35 @@ func (self *SCloudprovider) CanSync() bool { } } +func (self *SCloudprovider) SyncProject() (err error) { + projectId := "" + if len(self.ProjectId) == 0 && len(self.Name) > 0 && self.Provider == CLOUD_PROVIDER_AZURE { + s := auth.GetAdminSession(options.Options.Region, "") + if project, err := modules.Projects.GetByName(s, self.Name, nil); err == nil { + if projectId, err = project.GetString("id"); err != nil { + return err + } + } else if strings.Index(err.Error(), "404 NotFoundError") > 0 { + if project, err := modules.Projects.Create(s, jsonutils.Marshal(map[string]string{"name": self.Name})); err != nil { + return err + } else if projectId, err = project.GetString("id"); err != nil { + return err + } + } else { + return err + } + if len(projectId) > 0 { + if _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.ProjectId = projectId + return nil + }); err != nil { + return err + } + } + } + return nil +} + type SSyncRange struct { Force bool FullSync bool @@ -148,7 +183,7 @@ func (sr *SSyncRange) NeedSyncInfo() bool { func (sr *SSyncRange) normalizeRegionIds() error { for i := 0; i < len(sr.Region); i += 1 { - obj, err := CloudregionManager.FetchByIdOrName("", sr.Region[i]) + obj, err := CloudregionManager.FetchByIdOrName(nil, sr.Region[i]) if err != nil { if err == sql.ErrNoRows { return httperrors.NewResourceNotFoundError("Region %s not found", sr.Region[i]) @@ -163,7 +198,7 @@ func (sr *SSyncRange) normalizeRegionIds() error { func (sr *SSyncRange) normalizeZoneIds() error { for i := 0; i < len(sr.Zone); i += 1 { - obj, err := ZoneManager.FetchByIdOrName("", sr.Zone[i]) + obj, err := ZoneManager.FetchByIdOrName(nil, sr.Zone[i]) if err != nil { if err == sql.ErrNoRows { return httperrors.NewResourceNotFoundError("Zone %s not found", sr.Zone[i]) @@ -178,7 +213,7 @@ func (sr *SSyncRange) normalizeZoneIds() error { func (sr *SSyncRange) normalizeHostIds() error { for i := 0; i < len(sr.Host); i += 1 { - obj, err := HostManager.FetchByIdOrName("", sr.Host[i]) + obj, err := HostManager.FetchByIdOrName(nil, sr.Host[i]) if err != nil { if err == sql.ErrNoRows { return httperrors.NewResourceNotFoundError("Host %s not found", sr.Host[i]) @@ -227,12 +262,15 @@ func (self *SCloudprovider) PerformSync(ctx context.Context, userCred mcclient.T return nil, httperrors.NewInputParameterError("invalid input %s", err) } if self.CanSync() || syncRange.Force { - err = self.startSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "") + err = self.StartSyncCloudProviderInfoTask(ctx, userCred, &syncRange, "") } return nil, err } -func (self *SCloudprovider) startSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error { +func (self *SCloudprovider) StartSyncCloudProviderInfoTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange *SSyncRange, parentTaskId string) error { + if err := self.SyncProject(); err != nil { + log.Errorf("Sync cloudprovider project error: %v", err) + } params := jsonutils.NewDict() if syncRange != nil { params.Add(jsonutils.Marshal(syncRange), "sync_range") @@ -265,12 +303,10 @@ type SAccount struct { } func (self *SCloudprovider) getCloudaccount() (*SCloudaccount, error) { - cloudaccount := &SCloudaccount{} - q := CloudaccountManager.Query().Equals("id", self.CloudaccountId) - if err := db.FetchModelObjects(CloudaccountManager, q, cloudaccount); err != nil { - return nil, err + if cloudaccount := CloudaccountManager.FetchCloudaccountById(self.CloudaccountId); cloudaccount != nil { + return cloudaccount, nil } - return cloudaccount, nil + return nil, fmt.Errorf("Failed to find cloud account for cloud provider %s", self.Name) } func (self *SCloudprovider) getAccount() (*SAccount, error) { @@ -283,7 +319,7 @@ func (self *SCloudprovider) getAccount() (*SAccount, error) { } else { account.Secret = passwd } - if account.Account != self.Account { + if len(self.Account) > 0 && self.Account != cloudaccount.Account { account.Account = fmt.Sprintf("%s/%s", account.Account, self.Account) } return &account, nil @@ -320,7 +356,7 @@ func (manager *SCloudproviderManager) FetchCloudproviderById(providerId string) } func (manager *SCloudproviderManager) FetchCloudproviderByIdOrName(providerId string) *SCloudprovider { - providerObj, err := manager.FetchByIdOrName("", providerId) + providerObj, err := manager.FetchByIdOrName(nil, providerId) if err != nil { if err != sql.ErrNoRows { log.Errorf("%s", err) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 5cc92da4f2..34abdd376c 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -38,6 +38,7 @@ const ( DISK_STARTALLOC = "start_alloc" DISK_ALLOCATING = "allocating" DISK_READY = "ready" + DISK_RESET = "reset" DISK_DEALLOC = "deallocating" DISK_DEALLOC_FAILED = "dealloc_failed" DISK_UNKNOWN = "unknown" @@ -155,7 +156,7 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu storageStr := jsonutils.GetAnyString(queryDict, []string{"storage", "storage_id"}) if len(storageStr) > 0 { - storageObj, err := StorageManager.FetchByIdOrName(userCred.GetProjectId(), storageStr) + storageObj, err := StorageManager.FetchByIdOrName(userCred, storageStr) if err != nil { return nil, httperrors.NewResourceNotFoundError("storage %s not found: %s", storageStr, err) } @@ -391,10 +392,8 @@ func (self *SDisk) CleanUpDiskSnapshots(ctx context.Context, userCred mcclient.T convertSnapshots := jsonutils.NewArray() deleteSnapshots := jsonutils.NewArray() for i := 0; i < len(dest); i++ { - if dest[i].CreatedBy == MANUAL && !dest[i].FakeDeleted { - if !dest[i].OutOfChain { - convertSnapshots.Add(jsonutils.NewString(dest[i].Id)) - } + if !dest[i].FakeDeleted && !dest[i].OutOfChain { + convertSnapshots.Add(jsonutils.NewString(dest[i].Id)) } else { deleteSnapshots.Add(jsonutils.NewString(dest[i].Id)) } @@ -416,6 +415,9 @@ func (self *SDisk) AllowPerformDiskReset(ctx context.Context, userCred mcclient. } func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status != DISK_READY { + return nil, httperrors.NewInvalidStatusError("Cannot reset disk in status %s", self.Status) + } snapshotId, err := data.GetString("snapshot_id") if err != nil { return nil, err @@ -436,13 +438,16 @@ func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.Token if snapshot.Status != SNAPSHOT_READY { return nil, httperrors.NewBadRequestError("Cannot reset disk with snapshot in status %s", snapshot.Status) } - self.StartResetDisk(ctx, userCred, snapshotId) + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) + self.StartResetDisk(ctx, userCred, snapshotId, autoStart) return nil, nil } -func (self *SDisk) StartResetDisk(ctx context.Context, userCred mcclient.TokenCredential, snapshotId string) error { +func (self *SDisk) StartResetDisk(ctx context.Context, userCred mcclient.TokenCredential, snapshotId string, autoStart bool) error { + self.SetStatus(userCred, DISK_RESET, "") params := jsonutils.NewDict() params.Set("snapshot_id", jsonutils.NewString(snapshotId)) + params.Set("auto_start", jsonutils.NewBool(autoStart)) task, err := taskman.TaskManager.NewTask(ctx, "DiskResetTask", self, userCred, params, "", "", nil) if err != nil { return err @@ -732,7 +737,11 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke self.ExpiredAt = extDisk.GetExpiredAt() self.ProjectId = userCred.GetProjectId() - + if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + self.ProjectId = provider.ProjectId + } + } return nil }) if err != nil { @@ -765,7 +774,11 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl disk.ExternalId = extDisk.GetGlobalId() disk.StorageId = storage.Id disk.ProjectId = userCred.GetProjectId() - + if manageId := extDisk.GetIStorge().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + disk.ProjectId = provider.ProjectId + } + } disk.DiskFormat = extDisk.GetDiskFormat() disk.DiskSize = extDisk.GetDiskSizeMB() disk.AutoDelete = extDisk.GetIsAutoDelete() @@ -1249,7 +1262,7 @@ func (manager *SDiskManager) AutoDiskSnapshot(ctx context.Context, userCred mccl continue } // name - name := guests[0].Name + time.Now().Format("2006-01-02#15:04:05") + name := "Auto-" + guests[0].Name + time.Now().Format("2006-01-02#15:04:05") snap, err := SnapshotManager.CreateSnapshot(ctx, userCred, AUTO, disk.Id, guests[0].Id, "", name) if err != nil { log.Errorln(err) diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index 5c5ad97792..c01b30df1f 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -84,7 +84,7 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem managerFilter, _ := query.GetString("manager") if len(managerFilter) > 0 { - managerI, err := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter) + managerI, err := CloudproviderManager.FetchByIdOrName(userCred, managerFilter) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter) @@ -97,7 +97,7 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem regionFilter, _ := query.GetString("region") if len(regionFilter) > 0 { - regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionFilter) + regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionFilter) @@ -232,7 +232,12 @@ func (self *SElasticip) SyncWithCloudEip(userCred mcclient.TokenCredential, ext self.ExternalId = ext.GetGlobalId() // self.ManagerId = ext.GetManagerId() self.IsEmulated = ext.IsEmulated() - // self.ProjectId = userCred.GetProjectId() + self.ProjectId = userCred.GetProjectId() + if manageId := ext.GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + self.ProjectId = provider.ProjectId + } + } self.ChargeType = ext.GetInternetChargeType() return nil @@ -258,7 +263,11 @@ func (manager *SElasticipManager) newFromCloudEip(userCred mcclient.TokenCredent eip.ChargeType = extEip.GetInternetChargeType() eip.ProjectId = userCred.GetProjectId() - + if manageId := extEip.GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + eip.ProjectId = provider.ProjectId + } + } err := manager.TableSpec().Insert(&eip) if err != nil { log.Errorf("newFromCloudEip fail %s", err) @@ -362,7 +371,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr if len(regionStr) == 0 { return nil, httperrors.NewInputParameterError("Missing region/region_id") } - region, err := CloudregionManager.FetchByIdOrName("", regionStr) + region, err := CloudregionManager.FetchByIdOrName(nil, regionStr) if err != nil { if err != sql.ErrNoRows { return nil, httperrors.NewGeneralError(err) @@ -377,7 +386,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr return nil, httperrors.NewInputParameterError("Missing manager/manager_id") } - provider, err := CloudproviderManager.FetchByIdOrName("", managerStr) + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) if err != nil { if err != sql.ErrNoRows { return nil, httperrors.NewGeneralError(err) @@ -490,7 +499,7 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient. return nil, httperrors.NewInputParameterError("Unsupported %s", instanceType) } - vmObj, err := GuestManager.FetchByIdOrName(userCred.GetProjectId(), instanceId) + vmObj, err := GuestManager.FetchByIdOrName(userCred, instanceId) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("server %s not found", instanceId) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index b2cc9b7df5..0f951c74dc 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -578,4 +578,4 @@ func (manager *SGuestnetworkManager) getRecentlyReleasedIPAddresses(networkId st } } return ret -} \ No newline at end of file +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 7d53b31407..4a4a823959 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/base64" "fmt" "net/http" "strconv" @@ -29,14 +30,17 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/seclib2" + + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/compute/sshkeys" ) const ( @@ -222,7 +226,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ hostFilter, _ := queryDict.GetString("host") if len(hostFilter) > 0 { - host, _ := HostManager.FetchByIdOrName("", hostFilter) + host, _ := HostManager.FetchByIdOrName(nil, hostFilter) if host == nil { return nil, httperrors.NewResourceNotFoundError("host %s not found", hostFilter) } @@ -231,7 +235,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ secgrpFilter, _ := queryDict.GetString("secgroup") if len(secgrpFilter) > 0 { - secgrp, _ := SecurityGroupManager.FetchByIdOrName("", secgrpFilter) + secgrp, _ := SecurityGroupManager.FetchByIdOrName(nil, secgrpFilter) if secgrp == nil { return nil, httperrors.NewResourceNotFoundError("secgroup %s not found", secgrpFilter) } @@ -240,7 +244,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ zoneFilter, _ := queryDict.GetString("zone") if len(zoneFilter) > 0 { - zone, _ := ZoneManager.FetchByIdOrName("", zoneFilter) + zone, _ := ZoneManager.FetchByIdOrName(nil, zoneFilter) if zone == nil { return nil, httperrors.NewResourceNotFoundError("zone %s not found", zoneFilter) } @@ -253,7 +257,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ wireFilter, _ := queryDict.GetString("wire") if len(wireFilter) > 0 { - wire, _ := WireManager.FetchByIdOrName("", wireFilter) + wire, _ := WireManager.FetchByIdOrName(nil, wireFilter) if wire == nil { return nil, httperrors.NewResourceNotFoundError("wire %s not found", wireFilter) } @@ -265,7 +269,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ networkFilter, _ := queryDict.GetString("network") if len(networkFilter) > 0 { - netI, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), networkFilter) + netI, _ := NetworkManager.FetchByIdOrName(userCred, networkFilter) if netI == nil { return nil, httperrors.NewResourceNotFoundError("network %s not found", networkFilter) } @@ -279,7 +283,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ diskFilter, _ := queryDict.GetString("disk") if len(diskFilter) > 0 { - diskI, _ := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskFilter) + diskI, _ := DiskManager.FetchByIdOrName(userCred, diskFilter) if diskI == nil { return nil, httperrors.NewResourceNotFoundError("disk %s not found", diskFilter) } @@ -312,7 +316,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ managerFilter, _ := queryDict.GetString("manager") if len(managerFilter) > 0 { - managerI, _ := CloudproviderManager.FetchByIdOrName(userCred.GetProjectId(), managerFilter) + managerI, _ := CloudproviderManager.FetchByIdOrName(userCred, managerFilter) if managerI == nil { return nil, httperrors.NewResourceNotFoundError("cloud provider %s not found", managerFilter) } @@ -323,7 +327,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ regionFilter, _ := queryDict.GetString("region") if len(regionFilter) > 0 { - regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionFilter) + regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionFilter) @@ -555,7 +559,7 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To err = self.checkUpdateQuota(ctx, userCred, vcpuCount, vmemSize) if err != nil { - return nil, err + return nil, httperrors.NewOutOfQuotaError(err.Error()) } if data.Contains("name") { @@ -640,7 +644,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if len(bmName) == 0 { bmName, _ = data.GetString("prefer_baremetal") } - bmObj, err := HostManager.FetchByIdOrName("", bmName) + bmObj, err := HostManager.FetchByIdOrName(nil, bmName) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("Host %s not found", bmName) @@ -804,7 +808,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m keypairId, _ = data.GetString("keypair_id") } if len(keypairId) > 0 { - keypairObj, err := KeypairManager.FetchByIdOrName(userCred.GetUserId(), keypairId) + keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Keypair %s not found", keypairId) } @@ -815,7 +819,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if data.Contains("secgroup") { secGrpId, _ := data.GetString("secgroup") - secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secGrpId) + secGrpObj, err := SecurityGroupManager.FetchByIdOrName(userCred, secGrpId) if err != nil { return nil, httperrors.NewResourceNotFoundError("Secgroup %s not found", secGrpId) } @@ -937,6 +941,11 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred if osProfileJson != nil { guest.setOSProfile(ctx, userCred, osProfileJson) } + + userData, _ := data.GetString("user_data") + if len(userData) > 0 { + guest.setUserData(ctx, userCred, userData) + } } func (guest *SGuest) setApptags(ctx context.Context, appTags []string, userCred mcclient.TokenCredential) { @@ -1344,6 +1353,12 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token self.Machine = extVM.GetMachine() self.HostId = host.Id self.ProjectId = userCred.GetProjectId() + if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + self.ProjectId = provider.ProjectId + } + } + self.Hypervisor = extVM.GetHypervisor() self.IsEmulated = extVM.IsEmulated() @@ -1411,7 +1426,13 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. guest.ExpiredAt = extVM.GetExpiredAt() guest.HostId = host.Id + guest.ProjectId = userCred.GetProjectId() + if manageId := extVM.GetIHost().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + guest.ProjectId = provider.ProjectId + } + } metaData := extVM.GetMetadata() @@ -1764,9 +1785,10 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { doRestart = true var kpId string - if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { + + if kwargs.Contains("keypair") { keypair, _ := kwargs.GetString("keypair") - iKp, err := KeypairManager.FetchByIdOrName(userCred.GetProjectId(), keypair) + iKp, err := KeypairManager.FetchByIdOrName(userCred, keypair) if err != nil { return nil, err } @@ -1776,11 +1798,18 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr kp := iKp.(*SKeypair) kpId = kp.Id } + if self.KeypairId != kpId { + okey := self.getKeypair() + if okey != nil { + kwargs.Set("delete_public_key", jsonutils.NewString(okey.PublicKey)) + } + self.GetModelManager().TableSpec().Update(self, func() error { self.KeypairId = kpId return nil }) + kwargs.Set("reset_password", jsonutils.JSONTrue) } } @@ -1795,6 +1824,7 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr } return nil, nil } + return nil, httperrors.NewServerStatusError("Cannot deploy in status %s", self.Status) } @@ -1830,7 +1860,7 @@ func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.Tok if diskId, err := data.GetString("disk_id"); err != nil { return nil, err } else { - if disk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId); err != nil { + if disk, err := DiskManager.FetchByIdOrName(userCred, diskId); err != nil { return nil, err } else if disk == nil { return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) @@ -2451,7 +2481,7 @@ func (self *SGuest) PerformAssignSecgroup(ctx context.Context, userCred mcclient } else { if secgrp, err := data.GetString("secgrp"); err != nil { return nil, err - } else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), secgrp); err != nil { + } else if sg, err := SecurityGroupManager.FetchByIdOrName(userCred, secgrp); err != nil { return nil, httperrors.NewNotFoundError("SecurityGroup %s not found", secgrp) } else { if _, err := self.GetModelManager().TableSpec().Update(self, func() error { @@ -2538,7 +2568,7 @@ func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.To keypairStr := jsonutils.GetAnyString(data, []string{"keypair", "keypair_id"}) if len(keypairStr) > 0 { - keypairObj, err := KeypairManager.FetchByIdOrName(userCred.GetUserId(), keypairStr) + keypairObj, err := KeypairManager.FetchByIdOrName(userCred, keypairStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("keypair %s not found", keypairStr) @@ -2680,7 +2710,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok return nil, err } keepDisk := jsonutils.QueryBoolean(data, "keep_disk", false) - iDisk, err := DiskManager.FetchByIdOrName(userCred.GetProjectId(), diskId) + iDisk, err := DiskManager.FetchByIdOrName(userCred, diskId) if err != nil { return nil, err } @@ -2735,7 +2765,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError(msg) } - iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device) + iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device) if err != nil { msg := fmt.Sprintf("Isolated device %s not found", device) logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) @@ -2785,7 +2815,7 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewBadRequestError(msg) } - iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device) + iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device) if err != nil { msg := fmt.Sprintf("Isolated device %s not found", device) logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) @@ -3289,10 +3319,29 @@ func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, host *SHost, para if keypair != nil { config.Add(jsonutils.NewString(keypair.PublicKey), "public_key") } + deletePubKey, _ := params.GetString("delete_public_key") + if len(deletePubKey) > 0 { + config.Add(jsonutils.NewString(deletePubKey), "delete_public_key") + } } else { config.Add(jsonutils.JSONFalse, "reset_password") } + // add default public keys + _, adminPubKey, err := sshkeys.GetSshAdminKeypair(ctx) + if err != nil { + log.Errorf("fail to get ssh admin public key %s", err) + } + + _, projPubKey, err := sshkeys.GetSshProjectKeypair(ctx, self.ProjectId) + + if err != nil { + log.Errorf("fail to get ssh project public key %s", err) + } + + config.Add(jsonutils.NewString(adminPubKey), "admin_public_key") + config.Add(jsonutils.NewString(projPubKey), "project_public_key") + config.Add(jsonutils.NewString(deployAction), "action") onFinish := "shutdown" @@ -4343,7 +4392,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T if len(eipStr) == 0 { return nil, httperrors.NewInputParameterError("missing eip or eip_id") } - eipObj, err := ElasticipManager.FetchByIdOrName(userCred.GetProjectId(), eipStr) + eipObj, err := ElasticipManager.FetchByIdOrName(userCred, eipStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("eip %s not found", eipStr) @@ -4496,3 +4545,37 @@ func (self *SGuest) getDefaultStorageType() string { } return STORAGE_LOCAL } + +func (self *SGuest) setUserData(ctx context.Context, userCred mcclient.TokenCredential, data string) error { + data = base64.StdEncoding.EncodeToString([]byte(data)) + if len(data) > 16*1024 { + return fmt.Errorf("User data is limited to 16 KB.") + } + err := self.SetMetadata(ctx, "user_data", data, userCred) + if err != nil { + return err + } + return nil +} + +func (self *SGuest) AllowPerformUserData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformUserData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + userData, err := data.GetString("user_data") + if err != nil { + return nil, httperrors.NewInputParameterError("missing user_data %s", err) + } + err = self.setUserData(ctx, userCred, userData) + if err != nil { + return nil, httperrors.NewGeneralError(err) + } + if len(self.HostId) > 0 { + err = self.StartSyncTask(ctx, userCred, false, "") + if err != nil { + return nil, httperrors.NewGeneralError(err) + } + } + return nil, nil +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index bec3cec6e7..80c153b325 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -153,7 +153,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu schedTagStr := jsonutils.GetAnyString(query, []string{"schedtag", "schedtag_id"}) if len(schedTagStr) > 0 { - schedTag, _ := SchedtagManager.FetchByIdOrName("", schedTagStr) + schedTag, _ := SchedtagManager.FetchByIdOrName(nil, schedTagStr) if schedTag == nil { return nil, httperrors.NewResourceNotFoundError("Schedtag %s not found", schedTagStr) } @@ -163,7 +163,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu wireStr := jsonutils.GetAnyString(query, []string{"wire", "wire_id"}) if len(wireStr) > 0 { - wire, _ := WireManager.FetchByIdOrName("", wireStr) + wire, _ := WireManager.FetchByIdOrName(nil, wireStr) if wire == nil { return nil, httperrors.NewResourceNotFoundError("Wire %s not found", wireStr) } @@ -173,7 +173,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu storageStr := jsonutils.GetAnyString(query, []string{"storage", "storage_id"}) if len(storageStr) > 0 { - storage, _ := StorageManager.FetchByIdOrName("", storageStr) + storage, _ := StorageManager.FetchByIdOrName(nil, storageStr) if storage == nil { return nil, httperrors.NewResourceNotFoundError("Storage %s not found", storageStr) } @@ -183,7 +183,7 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { - zone, _ := ZoneManager.FetchByIdOrName("", zoneStr) + zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr) if zone == nil { return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr) } diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 077518d6f1..d45aaa3231 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -115,7 +115,7 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(ctx context.Context, q *sq } zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { - zone, _ := ZoneManager.FetchByIdOrName("", zoneStr) + zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr) if zone == nil { return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr) } diff --git a/pkg/compute/models/keypairs.go b/pkg/compute/models/keypairs.go index 84cb4f5b28..82ece4aa91 100644 --- a/pkg/compute/models/keypairs.go +++ b/pkg/compute/models/keypairs.go @@ -4,10 +4,15 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/sqlchemy" + + "golang.org/x/crypto/ssh" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/pkg/utils" ) type SKeypairManager struct { @@ -23,11 +28,11 @@ func init() { type SKeypair struct { db.SStandaloneResourceBase - Scheme string `width:"12" charset:"ascii" nullable:"true" default:"RSA" list:"user" create:"optional"` // Column(VARCHAR(length=12, charset='ascii'), nullable=True, default='RSA') - Fingerprint string `width:"48" charset:"ascii" nullable:"false" list:"user"` // Column(VARCHAR(length=48, charset='ascii'), nullable=False) - PrivateKey string `width:"2048" charset:"ascii" nullable:"false"` // Column(VARCHAR(length=2048, charset='ascii'), nullable=False) - PublicKey string `width:"1024" charset:"ascii" nullable:"false" list:"user"` // Column(VARCHAR(length=1024, charset='ascii'), nullable=False) - OwnerId string `width:"128" charset:"ascii" index:"true" nullable:"false"` // Column(VARCHAR(length=36, charset='ascii'), index=True, nullable=False) + Scheme string `width:"12" charset:"ascii" nullable:"true" default:"RSA" list:"user" create:"required"` // Column(VARCHAR(length=12, charset='ascii'), nullable=True, default='RSA') + Fingerprint string `width:"48" charset:"ascii" nullable:"false" list:"user" create:"required"` // Column(VARCHAR(length=48, charset='ascii'), nullable=False) + PrivateKey string `width:"2048" charset:"ascii" nullable:"false" create:"optional"` // Column(VARCHAR(length=2048, charset='ascii'), nullable=False) + PublicKey string `width:"1024" charset:"ascii" nullable:"false" list:"user" create:"required"` // Column(VARCHAR(length=1024, charset='ascii'), nullable=False) + OwnerId string `width:"128" charset:"ascii" index:"true" nullable:"false" create:"required"` // Column(VARCHAR(length=36, charset='ascii'), index=True, nullable=False) } func (manager *SKeypairManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -100,7 +105,40 @@ func (self *SKeypair) GetLinkedGuestsCount() int { } func (manager *SKeypairManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - // XXX: TODO + publicKey, _ := data.GetString("public_key") + if len(publicKey) == 0 { + scheme, _ := data.GetString("scheme") + if len(scheme) > 0 { + if !utils.IsInStringArray(scheme, []string{"RSA", "DSA"}) { + return nil, httperrors.NewInputParameterError("Unsupported scheme %s", scheme) + } + } else { + scheme = "RSA" + } + var privKey, pubKey string + var err error + if scheme == "RSA" { + privKey, pubKey, err = seclib2.GenerateRSASSHKeypair() + } else { + privKey, pubKey, err = seclib2.GenerateDSASSHKeypair() + } + if err != nil { + log.Errorf("fail to generate ssh keypair %s", err) + return nil, httperrors.NewGeneralError(err) + } + publicKey = pubKey + data.Set("public_key", jsonutils.NewString(pubKey)) + data.Set("private_key", jsonutils.NewString(privKey)) + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey)) + if err != nil { + log.Errorf("invalid public key %s", err) + return nil, httperrors.NewInputParameterError("invalid public") + } + data.Set("fingerprint", jsonutils.NewString(ssh.FingerprintLegacyMD5(pubKey))) + data.Set("scheme", jsonutils.NewString(seclib2.GetPublicKeyScheme(pubKey))) + data.Set("owner_id", jsonutils.NewString(userCred.GetUserId())) + return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) } @@ -124,6 +162,37 @@ func (self *SKeypair) GetOwnerProjectId() string { return self.OwnerId } -func (manager *SKeypairManager) GetOwnerId(userCred mcclient.TokenCredential) string { +func (manager *SKeypairManager) GetOwnerId(userCred mcclient.IIdentityProvider) string { return userCred.GetUserId() } + +func (manager *SKeypairManager) FetchByName(userCred mcclient.IIdentityProvider, idStr string) (db.IModel, error) { + return db.FetchByName(manager, userCred, idStr) +} + +func (manager *SKeypairManager) FetchByIdOrName(userCred mcclient.IIdentityProvider, idStr string) (db.IModel, error) { + return db.FetchByIdOrName(manager, userCred, idStr) +} + +func (keypair *SKeypair) AllowGetDetailsPrivatekey(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return keypair.OwnerId == userCred.GetUserId() +} + +func (keypair *SKeypair) GetDetailsPrivatekey(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + retval := jsonutils.NewDict() + if len(keypair.PrivateKey) > 0 { + retval.Add(jsonutils.NewString(keypair.PrivateKey), "private_key") + retval.Add(jsonutils.NewString(keypair.Name), "name") + retval.Add(jsonutils.NewString(keypair.Scheme), "scheme") + _, err := keypair.GetModelManager().TableSpec().Update(keypair, func() error { + keypair.PrivateKey = "" + return nil + }) + if err != nil { + return nil, err + } + + db.OpsLog.LogEvent(keypair, db.ACT_FETCH, nil, userCred) + } + return retval, nil +} diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index e13d486e45..ce539e5431 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -493,6 +493,11 @@ func (self *SNetwork) SyncWithCloudNetwork(userCred mcclient.TokenCredential, ex self.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() self.ProjectId = userCred.GetProjectId() + if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + self.ProjectId = provider.ProjectId + } + } return nil }) if err != nil { @@ -519,7 +524,11 @@ func (manager *SNetworkManager) newFromCloudNetwork(userCred mcclient.TokenCrede net.AllocTimoutSeconds = extNet.GetAllocTimeoutSeconds() net.ProjectId = userCred.GetProjectId() - + if manageId := extNet.GetIWire().GetIVpc().GetManagerId(); len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + net.ProjectId = provider.ProjectId + } + } err := manager.TableSpec().Insert(&net) if err != nil { log.Errorf("newFromCloudZone fail %s", err) @@ -691,7 +700,7 @@ func parseNetworkInfo(userCred mcclient.TokenCredential, info jsonutils.JSONObje } else if p == "[vip]" { netConfig.Vip = true } else { - netObj, err := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), p) + netObj, err := NetworkManager.FetchByIdOrName(userCred, p) if err != nil { return nil, err } @@ -710,7 +719,7 @@ func (self *SNetwork) getFreeAddressCount() int { func isValidNetworkInfo(userCred mcclient.TokenCredential, netConfig *SNetworkConfig) error { if len(netConfig.Network) > 0 { - netObj, err := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), netConfig.Network) + netObj, err := NetworkManager.FetchByIdOrName(userCred, netConfig.Network) if err != nil { return httperrors.NewResourceNotFoundError("Network %s not found %s", err) } @@ -971,7 +980,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred wireStr := jsonutils.GetAnyString(data, []string{"wire", "wire_id"}) if len(wireStr) > 0 { - wireObj, err := WireManager.FetchByIdOrName(userCred.GetProjectId(), wireStr) + wireObj, err := WireManager.FetchByIdOrName(userCred, wireStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("wire %s not found", wireStr) @@ -985,7 +994,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred if len(zoneStr) > 0 { vpcStr := jsonutils.GetAnyString(data, []string{"vpc", "vpc_id"}) if len(vpcStr) > 0 { - zoneObj, err := ZoneManager.FetchByIdOrName(userCred.GetProjectId(), zoneStr) + zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("zone %s not found", zoneStr) @@ -993,7 +1002,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred return nil, httperrors.NewInternalServerError("query zone %s error %s", zoneStr, err) } } - vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr) + vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("vpc %s not found", vpcStr) @@ -1277,7 +1286,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. } zoneStr, _ := query.GetString("zone") if len(zoneStr) > 0 { - zoneObj, err := ZoneManager.FetchByIdOrName(userCred.GetProjectId(), zoneStr) + zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr) if err != nil { return nil, httperrors.NewNotFoundError("Zone %s not found", zoneStr) } @@ -1286,7 +1295,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. } vpcStr, _ := query.GetString("vpc") if len(vpcStr) > 0 { - vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr) + vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr) if err != nil { return nil, httperrors.NewNotFoundError("VPC %s not found", vpcStr) } @@ -1295,7 +1304,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. } regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"}) if len(regionStr) > 0 { - region, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr) + region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("cloud region %s not found", regionStr) diff --git a/pkg/compute/models/quotas.go b/pkg/compute/models/quotas.go index ad73a7a1c9..152095735e 100644 --- a/pkg/compute/models/quotas.go +++ b/pkg/compute/models/quotas.go @@ -7,9 +7,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/pkg/tristate" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/pkg/tristate" ) var QuotaManager *quotas.SQuotaManager @@ -278,7 +278,7 @@ func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error { if sreq.IsolatedDevice > 0 && self.IsolatedDevice > squota.IsolatedDevice { return ErrOutOfIsolatedDevice } - if self.Snapshot > squota.Snapshot { + if sreq.Snapshot > 0 && self.Snapshot > squota.Snapshot { return ErrOutOfSnapshot } return nil diff --git a/pkg/compute/models/reservedips.go b/pkg/compute/models/reservedips.go index 1f24d09131..e3b2401f56 100644 --- a/pkg/compute/models/reservedips.go +++ b/pkg/compute/models/reservedips.go @@ -122,7 +122,7 @@ func (manager *SReservedipManager) ListItemFilter(ctx context.Context, q *sqlche } network, _ := query.GetString("network") if len(network) > 0 { - netObj, _ := NetworkManager.FetchByIdOrName(userCred.GetProjectId(), network) + netObj, _ := NetworkManager.FetchByIdOrName(userCred, network) if netObj == nil { return nil, httperrors.NewResourceNotFoundError(fmt.Sprintf("network %s not found", network)) } diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index e0a91987c1..79134250e9 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -59,7 +59,7 @@ func (manager *SSchedtagManager) AllowCreateItem(ctx context.Context, userCred m func (manager *SSchedtagManager) ValidateSchedtags(userCred mcclient.TokenCredential, schedtags map[string]string) (map[string]string, error) { ret := make(map[string]string) for tag, act := range schedtags { - schedtagObj, err := manager.FetchByIdOrName("", tag) + schedtagObj, err := manager.FetchByIdOrName(nil, tag) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("Invalid schedtag %s", tag) diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 24d83c9ba8..3514e720d2 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -98,7 +98,7 @@ func (manager *SSecurityGroupRuleManager) ListItemFilter(ctx context.Context, q return nil, err } if defsecgroup, _ := query.GetString("secgroup"); len(defsecgroup) > 0 { - if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), defsecgroup); secgroup != nil { + if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred, defsecgroup); secgroup != nil { sql = sql.Equals("secgroup_id", secgroup.GetId()) } else { return nil, httperrors.NewNotFoundError(fmt.Sprintf("Security Group %s not found", defsecgroup)) @@ -130,7 +130,7 @@ func (manager *SSecurityGroupRuleManager) ValidateCreateData( data *jsonutils.JSONDict, ) (*jsonutils.JSONDict, error) { if defsecgroup, _ := data.GetString("secgroup"); len(defsecgroup) > 0 { - if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred.GetProjectId(), defsecgroup); secgroup != nil { + if secgroup, _ := SecurityGroupManager.FetchByIdOrName(userCred, defsecgroup); secgroup != nil { data.Set("secgroup_id", jsonutils.NewString(secgroup.GetId())) } else { return nil, httperrors.NewNotFoundError(fmt.Sprintf("Security Group %s not found", defsecgroup)) diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 43ed994217..bef171376e 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -89,6 +89,32 @@ func (manager *SSnapshotManager) ListItemFilter(ctx context.Context, q *sqlchemy return q, nil } +func (self *SSnapshot) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SSnapshot) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + disk, _ := self.GetDisk() + if disk != nil { + extra.Add(jsonutils.NewString(disk.DiskType), "disk_type") + guests := disk.GetGuests() + if len(guests) == 1 { + extra.Add(jsonutils.NewString(guests[0].Id), "guest") + extra.Add(jsonutils.NewString(guests[0].Status), "guest_status") + } + } + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + return extra +} + func (self *SSnapshot) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return false } @@ -199,6 +225,11 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mccli snapshot := &SSnapshot{} snapshot.SetModelManager(self) snapshot.ProjectId = userCred.GetProjectId() + if manageId := disk.GetStorage().ManagerId; len(manageId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(manageId); provider != nil { + snapshot.ProjectId = provider.ProjectId + } + } snapshot.DiskId = disk.Id snapshot.StorageId = disk.StorageId snapshot.Size = disk.DiskSize @@ -392,6 +423,11 @@ func (manager *SSnapshotManager) newFromCloudSnapshot(userCred mcclient.TokenCre snapshot.CloudregionId = region.Id snapshot.ProjectId = userCred.GetProjectId() + if len(snapshot.ManagerId) > 0 { + if provider := CloudproviderManager.FetchCloudproviderById(snapshot.ManagerId); provider != nil { + snapshot.ProjectId = provider.ProjectId + } + } err := manager.TableSpec().Insert(&snapshot) if err != nil { log.Errorf("newFromCloudEip fail %s", err) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 060c266c62..acc6346d25 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -675,7 +675,7 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. regionStr, _ := query.GetString("region") if len(regionStr) > 0 { - regionObj, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr) + regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr) if err != nil { return nil, httperrors.NewNotFoundError("Region %s not found: %s", regionStr, err) } diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 345de3fd12..56883ff6bd 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -54,7 +54,7 @@ func (manager *SWireManager) ValidateCreateData(ctx context.Context, userCred mc } if len(vpcStr) > 0 { - vpcObj, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr) + vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("Vpc %s not found", vpcStr) @@ -534,7 +534,7 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu vpcStr := jsonutils.GetAnyString(query, []string{"vpc_id", "vpc"}) if len(vpcStr) > 0 { - vpc, err := VpcManager.FetchByIdOrName(userCred.GetProjectId(), vpcStr) + vpc, err := VpcManager.FetchByIdOrName(userCred, vpcStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("vpc %s not found", vpcStr) @@ -547,7 +547,7 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"}) if len(regionStr) > 0 { - region, err := CloudregionManager.FetchByIdOrName(userCred.GetProjectId(), regionStr) + region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("region %s not found", regionStr) @@ -605,4 +605,4 @@ func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict } } return extra -} \ No newline at end of file +} diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 8b1cc8c57e..8301f95680 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -523,7 +523,7 @@ func (manager *SZoneManager) ValidateCreateData(ctx context.Context, userCred mc regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id", "cloudregion", "cloudregion_id"}) var regionId string if len(regionStr) > 0 { - regionObj, err := CloudregionManager.FetchByIdOrName("", regionStr) + regionObj, err := CloudregionManager.FetchByIdOrName(nil, regionStr) if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewResourceNotFoundError("Region %s not found", regionStr) diff --git a/pkg/compute/sshkeys/doc.go b/pkg/compute/sshkeys/doc.go new file mode 100644 index 0000000000..ad943a49e4 --- /dev/null +++ b/pkg/compute/sshkeys/doc.go @@ -0,0 +1 @@ +package sshkeys // import "yunion.io/x/onecloud/pkg/compute/sshkeys" diff --git a/pkg/compute/sshkeys/handler.go b/pkg/compute/sshkeys/handler.go new file mode 100644 index 0000000000..f6bda5d2cf --- /dev/null +++ b/pkg/compute/sshkeys/handler.go @@ -0,0 +1,85 @@ +package sshkeys + +import ( + "context" + "database/sql" + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +func AddSshKeysHandler(prefix string, app *appsrv.Application) { + app.AddHandler2("GET", fmt.Sprintf("%s/sshkeypairs", prefix), auth.Authenticate(sshKeysHandler), nil, "get_sshkeys", nil) + app.AddHandler2("GET", fmt.Sprintf("%s/sshkeypairs/", prefix), auth.Authenticate(adminSshKeysHandler), nil, "get_sshkeys", nil) +} + +func adminSshKeysHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + publicOnly := false + userCred := auth.FetchUserCredential(ctx) + if !userCred.IsSystemAdmin() { + publicOnly = true + } + params := appctx.AppContextParams(ctx) + projectId := params[""] + if len(projectId) == 0 { + httperrors.InputParameterError(w, "empty project_id/tenant_id") + return + } + tenant, err := db.TenantCacheManager.FetchTenantByIdOrName(ctx, projectId) + if err != nil { + if err == sql.ErrNoRows { + httperrors.ResourceNotFoundError(w, "tenant/project %s not found", projectId) + return + } else { + httperrors.GeneralServerError(w, err) + return + } + } + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + isAdmin := jsonutils.QueryBoolean(query, "admin", false) + + sendSshKey(ctx, w, userCred, tenant.Id, isAdmin, publicOnly) +} + +func sshKeysHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + userCred := auth.FetchUserCredential(ctx) + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + isAdmin := jsonutils.QueryBoolean(query, "admin", false) + + sendSshKey(ctx, w, userCred, userCred.GetProjectId(), isAdmin, false) +} + +func sendSshKey(ctx context.Context, w http.ResponseWriter, userCred mcclient.TokenCredential, projectId string, isAdmin bool, publicOnly bool) { + var privKey, pubKey string + + if isAdmin && userCred.IsSystemAdmin() { + privKey, pubKey, _ = GetSshAdminKeypair(ctx) + } else { + privKey, pubKey, _ = GetSshProjectKeypair(ctx, projectId) + } + + ret := jsonutils.NewDict() + + if !publicOnly { + ret.Add(jsonutils.NewString(privKey), "private_key") + } + ret.Add(jsonutils.NewString(pubKey), "public_key") + body := jsonutils.NewDict() + body.Add(ret, "sshkeypair") + appsrv.SendJSON(w, body) +} diff --git a/pkg/compute/sshkeys/sshkeypairs.go b/pkg/compute/sshkeys/sshkeypairs.go new file mode 100644 index 0000000000..7f1e71b9fa --- /dev/null +++ b/pkg/compute/sshkeys/sshkeypairs.go @@ -0,0 +1,46 @@ +package sshkeys + +import ( + "context" + + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/util/seclib2" +) + +const ( + sshAdminPrivateKey = "admin-ssh-private-key" + sshAdminPublicKey = "admin-ssh-public-key" + + sshPrivateKey = "project-ssh-private-key" + sshPublicKey = "project-ssh-public-key" +) + +func _getKeys(ctx context.Context, tenantId string, privateKey, publicKey string) (string, string, error) { + tenant, err := db.TenantCacheManager.FetchTenantById(ctx, tenantId) + if err != nil { + return "", "", err + } + private := tenant.GetMetadata(privateKey, nil) + public := tenant.GetMetadata(publicKey, nil) + userCred := auth.AdminCredential() + if len(private) == 0 || len(public) == 0 { + private, public, _ = seclib2.GenerateRSASSHKeypair() + private, _ = utils.EncryptAESBase64(tenantId, private) + tenant.SetMetadata(ctx, privateKey, private, userCred) + tenant.SetMetadata(ctx, publicKey, public, userCred) + } + private, _ = utils.DescryptAESBase64(tenantId, private) + return private, public, nil +} + +func GetSshProjectKeypair(ctx context.Context, tenantId string) (string, string, error) { + return _getKeys(ctx, tenantId, sshPrivateKey, sshPublicKey) +} + +func GetSshAdminKeypair(ctx context.Context) (string, string, error) { + userCred := auth.AdminCredential() + return _getKeys(ctx, userCred.GetProjectId(), sshAdminPrivateKey, sshAdminPublicKey) +} diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go new file mode 100644 index 0000000000..434b6e5b3c --- /dev/null +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -0,0 +1,60 @@ +package tasks + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/utils" +) + +type CloudAccountSyncInfoTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(CloudAccountSyncInfoTask{}) +} + +func (self *CloudAccountSyncInfoTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + account := obj.(*models.SCloudaccount) + account.MarkStartSync(self.UserCred) + if _, err := account.GetSubAccounts(); err != nil { + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, err.Error()) + self.SetStageFailed(ctx, err.Error()) + return + } + + syncRange := models.SSyncRange{} + syncRangeJson, _ := self.Params.Get("sync_range") + if syncRangeJson != nil { + syncRangeJson.Unmarshal(&syncRange) + } + // do sync + exsitSubTask := false + self.SetStage("on_cloudaccount_sync_complete", nil) + for _, cloudprovider := range account.GetCloudproviders() { + if cloudprovider.Enabled { + exsitSubTask = true + cloudprovider.StartSyncCloudProviderInfoTask(ctx, self.UserCred, &syncRange, self.GetTaskId()) + } + } + if !exsitSubTask { + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") + self.SetStageComplete(ctx, nil) + } +} + +func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Context, account *models.SCloudaccount, data jsonutils.JSONObject) { + for _, cloudprovider := range account.GetCloudproviders() { + if cloudprovider.Enabled && + utils.IsInStringArray(account.Status, []string{models.CLOUD_PROVIDER_START_SYNC, models.CLOUD_PROVIDER_SYNCING}) && time.Now().Sub(cloudprovider.LastSync) < time.Minute*20 { + return + } + } + account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/disk_reset_task.go b/pkg/compute/tasks/disk_reset_task.go index 1d74fd1ae3..7f3ac4cc63 100644 --- a/pkg/compute/tasks/disk_reset_task.go +++ b/pkg/compute/tasks/disk_reset_task.go @@ -25,11 +25,13 @@ func (self *DiskResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, disk := obj.(*models.SDisk) storage := disk.GetStorage() if storage == nil { + disk.SetStatus(self.UserCred, models.DISK_READY, "") self.SetStageFailed(ctx, "Disk storage not found") return } host := storage.GetMasterHost() if host == nil { + disk.SetStatus(self.UserCred, models.DISK_READY, "") self.SetStageFailed(ctx, "Storage master host not found") return } @@ -39,6 +41,7 @@ func (self *DiskResetTask) OnInit(ctx context.Context, obj db.IStandaloneModel, func (self *DiskResetTask) RequestResetDisk(ctx context.Context, disk *models.SDisk, host *models.SHost) { snapshotId, err := self.Params.GetString("snapshot_id") if err != nil { + disk.SetStatus(self.UserCred, models.DISK_READY, "") self.SetStageFailed(ctx, fmt.Sprintf("Get snapshotId error %s", err.Error())) return } @@ -58,6 +61,7 @@ func (self *DiskResetTask) RequestResetDisk(ctx context.Context, disk *models.SD self.SetStage("OnRequestResetDisk", nil) err = host.GetHostDriver().RequestResetDisk(ctx, host, disk, params, self) if err != nil { + disk.SetStatus(self.UserCred, models.DISK_READY, "") self.SetStageFailed(ctx, err.Error()) } } @@ -83,6 +87,18 @@ func (self *DiskResetTask) OnRequestResetDisk(ctx context.Context, disk *models. return } } + if jsonutils.QueryBoolean(self.Params, "auto_start", false) { + guest := disk.GetGuests()[0] + self.SetStage("OnStartGuest", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + disk.SetStatus(self.UserCred, models.DISK_READY, "") + self.SetStageComplete(ctx, nil) + } +} + +func (self *DiskResetTask) OnStartGuest(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_READY, "") self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/usages/handler.go b/pkg/compute/usages/handler.go index 6d05a20e46..b103afba26 100644 --- a/pkg/compute/usages/handler.go +++ b/pkg/compute/usages/handler.go @@ -64,7 +64,7 @@ func getRangeObj(ctx context.Context, man db.IStandaloneModelManager, userCred m if err != nil { return nil, err } - return man.FetchByIdOrName(userCred.GetProjectId(), id) + return man.FetchByIdOrName(userCred, id) } func rangeObjHandler( diff --git a/pkg/httperrors/httperrors.go b/pkg/httperrors/httperrors.go index b9be4abbaf..55d0b1288e 100644 --- a/pkg/httperrors/httperrors.go +++ b/pkg/httperrors/httperrors.go @@ -71,6 +71,10 @@ func InvalidInputError(w http.ResponseWriter, msg string, params ...interface{}) JsonClientError(w, NewInputParameterError(msg, params...)) } +func InputParameterError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewInputParameterError(msg, params...)) +} + func MissingParameterError(w http.ResponseWriter, param string) { JsonClientError(w, NewMissingParameterError(param)) } @@ -94,3 +98,11 @@ func TenantNotFoundError(w http.ResponseWriter, msg string, params ...interface{ func OutOfQuotaError(w http.ResponseWriter, msg string, params ...interface{}) { JsonClientError(w, NewOutOfQuotaError(msg, params...)) } + +func NotSufficientPrivilegeError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewNotSufficientPrivilegeError(msg, params...)) +} + +func ResourceNotFoundError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewResourceNotFoundError(msg, params...)) +} diff --git a/pkg/mcclient/modules/mod_capabilities.go b/pkg/mcclient/modules/mod_capabilities.go index 76e058c8d8..4b6fa33124 100644 --- a/pkg/mcclient/modules/mod_capabilities.go +++ b/pkg/mcclient/modules/mod_capabilities.go @@ -26,5 +26,5 @@ func init() { Capabilities = SCapabilityManager{ ResourceManager: NewComputeManager("capability", "capabilities", []string{}, []string{}), } - registerCompute(&Capabilities) + registerComputeV2(&Capabilities) } diff --git a/pkg/mcclient/modules/mod_parameters.go b/pkg/mcclient/modules/mod_parameters.go index dad5bfe83d..cea308a80b 100644 --- a/pkg/mcclient/modules/mod_parameters.go +++ b/pkg/mcclient/modules/mod_parameters.go @@ -14,4 +14,4 @@ func init() { []string{"namespace", "namespace_id", "created_by", "updated_by"}, )} register(&Parameters) -} \ No newline at end of file +} diff --git a/pkg/mcclient/modules/mod_projecthosts.go b/pkg/mcclient/modules/mod_projecthosts.go index 3b93b2a7e7..8bcf66cadf 100644 --- a/pkg/mcclient/modules/mod_projecthosts.go +++ b/pkg/mcclient/modules/mod_projecthosts.go @@ -1,9 +1,5 @@ package modules -import ( - -) - type ProjectNodeManager struct { ResourceManager } diff --git a/pkg/mcclient/modules/mod_res_results.go b/pkg/mcclient/modules/mod_res_results.go index 35aaae8a6b..bfa28eb628 100644 --- a/pkg/mcclient/modules/mod_res_results.go +++ b/pkg/mcclient/modules/mod_res_results.go @@ -7,7 +7,7 @@ var ( func init() { ResResults = NewMeterManager("res_result", "res_results", []string{"res_id", "res_name", "cpu", "mem", "sys_disk", "data_disk", "ips", "res_type", "band_width", "os_distribution", "os_version", "platform", "region_id", - "project_name", "user_name", "start_time", "end_time", "time_length", "cpu_amount", "mem_amount", "disk_amount", "baremetal_amount", "gpu_amount", "res_fee"}, + "project_name", "user_name", "start_time", "end_time", "time_length", "cpu_amount", "mem_amount", "disk_amount", "baremetal_amount", "gpu_amount", "res_fee"}, []string{}, ) register(&ResResults) diff --git a/pkg/mcclient/modules/mod_servers.go b/pkg/mcclient/modules/mod_servers.go index 7b5c175f5b..95e75b4f0a 100644 --- a/pkg/mcclient/modules/mod_servers.go +++ b/pkg/mcclient/modules/mod_servers.go @@ -5,6 +5,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" ) @@ -18,24 +20,35 @@ func (this *ServerManager) GetLoginInfo(s *mcclient.ClientSession, id string, pa return nil, e } ret := jsonutils.NewDict() - login_key, e := data.GetString("login_key") + loginKey, e := data.GetString("login_key") if e != nil { return nil, fmt.Errorf("No login key: %s", e) - } else { - passwd, e := utils.DescryptAESBase64(id, login_key) - if e != nil { - return nil, e - } - ret.Add(jsonutils.NewString(passwd), "password") - v, e := data.Get("login_account") - if e == nil { - ret.Add(v, "username") - } - v, e = data.Get("login_key_timestamp") - if e == nil { - ret.Add(v, "updated") - } } + + var privateKey string + if params != nil && !gotypes.IsNil(params) { + privateKey, _ = params.GetString("private_key") + } + + var passwd string + if len(privateKey) > 0 { + passwd, e = seclib2.DecryptBase64(privateKey, loginKey) + } else { + passwd, e = utils.DescryptAESBase64(id, loginKey) + } + if e != nil { + return nil, e + } + ret.Add(jsonutils.NewString(passwd), "password") + v, e := data.Get("login_account") + if e == nil { + ret.Add(v, "username") + } + v, e = data.Get("login_key_timestamp") + if e == nil { + ret.Add(v, "updated") + } + return ret, nil } @@ -56,7 +69,8 @@ func init() { "Secgroup", "Secgrp_id", "vrouter", "vrouter_id", "Created_at", "Group_name", - "Group_id", "Hypervisor", "os_type"}, + "Group_id", "Hypervisor", "os_type", + "expired_at"}, []string{"Host", "Tenant", "is_system", "auto_delete_at"})} registerCompute(&Servers) diff --git a/pkg/mcclient/modules/mod_snapshots.go b/pkg/mcclient/modules/mod_snapshots.go index 8def8e7e3f..a06358d72e 100644 --- a/pkg/mcclient/modules/mod_snapshots.go +++ b/pkg/mcclient/modules/mod_snapshots.go @@ -8,7 +8,7 @@ func init() { Snapshots = NewComputeManager("snapshot", "snapshots", []string{"ID", "Name", "Size", "Status", "Disk_id", "Guest_id", "Created_at"}, - []string{"Storage_id", "Create_by", "Location", "Out_of_chain"}) + []string{"Storage_id", "Create_by", "Location", "Out_of_chain", "disk_type", "provider"}) - registerCompute(&Snapshots) + registerComputeV2(&Snapshots) } diff --git a/pkg/mcclient/modules/mod_sshkeypairs.go b/pkg/mcclient/modules/mod_sshkeypairs.go new file mode 100644 index 0000000000..335d59907e --- /dev/null +++ b/pkg/mcclient/modules/mod_sshkeypairs.go @@ -0,0 +1,37 @@ +package modules + +import ( + "fmt" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SSshkeypairManager struct { + ResourceManager +} + +func (this *SSshkeypairManager) List(s *mcclient.ClientSession, params jsonutils.JSONObject) (*ListResult, error) { + url := "/sshkeypairs" + queryStr := params.QueryString() + if len(queryStr) > 0 { + url = fmt.Sprintf("%s?%s", url, queryStr) + } + body, err := this._get(s, url, "sshkeypair") + if err != nil { + return nil, err + } + result := ListResult{Data: []jsonutils.JSONObject{body}} + return &result, nil +} + +var ( + Sshkeypairs SSshkeypairManager +) + +func init() { + Sshkeypairs = SSshkeypairManager{NewComputeManager("sshkeypair", "sshkeypairs", + []string{}, + []string{})} + + registerComputeV2(&Sshkeypairs) +} diff --git a/pkg/mcclient/modules/register.go b/pkg/mcclient/modules/register.go index de6072cb31..0c4a389a63 100644 --- a/pkg/mcclient/modules/register.go +++ b/pkg/mcclient/modules/register.go @@ -1,7 +1,15 @@ package modules func registerCompute(mod BaseManagerInterface) { + registerComputeV1(mod) + registerComputeV2(mod) +} + +func registerComputeV1(mod BaseManagerInterface) { _register("v1", mod) +} + +func registerComputeV2(mod BaseManagerInterface) { _register("v2", mod) } diff --git a/pkg/mcclient/options/k8s/cluster.go b/pkg/mcclient/options/k8s/cluster.go index 18e47522f0..b26974cc58 100644 --- a/pkg/mcclient/options/k8s/cluster.go +++ b/pkg/mcclient/options/k8s/cluster.go @@ -24,10 +24,14 @@ func (o ClusterListOptions) Params() *jsonutils.JSONDict { return params } +type K8sSupportVersion struct { + K8sVersion string `help:"Cluster kubernetes components version" choices:"v1.10.5|v1.11.3|v1.12.0"` +} + type ClusterCreateOptions struct { + K8sSupportVersion NAME string `help:"Name of cluster"` Mode string `help:"Cluster mode" choices:"internal"` - K8sVersion string `help:"Cluster kubernetes components version" choices:"v1.8.10|v1.9.5|v1.10.0"` InfraImage string `help:"Cluster kubelet infra container image"` Cidr string `help:"Cluster service CIDR, e.g. 10.43.0.0/16"` Domain string `help:"Cluster pod domain, e.g. cluster.local"` @@ -70,8 +74,8 @@ func (o ClusterImportOptions) Params() (*jsonutils.JSONDict, error) { } type ClusterUpdateOptions struct { - NAME string `help:"Name of cluster"` - K8sVersion string `help:"Cluster kubernetes components version" choices:"v1.8.10|v1.9.5|v1.10.0"` + NAME string `help:"Name of cluster"` + K8sSupportVersion } func (o ClusterUpdateOptions) Params() *jsonutils.JSONDict { diff --git a/pkg/mcclient/options/k8s/tiller.go b/pkg/mcclient/options/k8s/tiller.go index a1a6a64552..4af8fbbb24 100644 --- a/pkg/mcclient/options/k8s/tiller.go +++ b/pkg/mcclient/options/k8s/tiller.go @@ -16,7 +16,7 @@ type TillerCreateOptions struct { Canary bool `json:"canary_image"` // Override Tiller image - Image string `json:"tiller_image" default:"yunion/tiller:v2.9.0"` + Image string `json:"tiller_image" default:"yunion/tiller:v2.9.1"` // Limit the maximum number of revisions saved per release. Use 0 for no limit. MaxHistory int `json:"history_max"` } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 06110971d5..3d84e591c2 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -31,6 +31,11 @@ type ServerIdOptions struct { ID string `help:"ID or name of the server" json:"-"` } +type ServerLoginInfoOptions struct { + ID string `help:"ID or name of the server" json:"-"` + Key string `help:"File name of private key, if password is encrypted by key"` +} + type ServerIdsOptions struct { ID []string `help:"ID of servers to operate" metavar:"SERVER" json:"-"` } @@ -115,6 +120,7 @@ type ServerCreateOptions struct { Count *int `help:"Create multiple simultaneously" default:"1" json:"-"` DryRun *bool `help:"Dry run to test scheduler" json:"-"` RaidConfig []string `help:"Baremetal raid config" json:"-"` + UserDataFile string `help:"user_data file path" json:"-"` } func (opts *ServerCreateOptions) Params() (*jsonutils.JSONDict, error) { diff --git a/pkg/mcclient/token.go b/pkg/mcclient/token.go index 8c0e5e00ed..a7ca1b1afd 100644 --- a/pkg/mcclient/token.go +++ b/pkg/mcclient/token.go @@ -20,19 +20,24 @@ type Endpoint struct { Interface string } +type IIdentityProvider interface { + GetProjectId() string + GetUserId() string + GetTenantId() string +} + type TokenCredential interface { gotypes.ISerializable IServiceCatalog + IIdentityProvider + GetTokenString() string GetDomainId() string GetDomainName() string - GetTenantId() string GetTenantName() string - GetProjectId() string GetProjectName() string - GetUserId() string GetUserName() string GetRoles() []string GetExpires() time.Time diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index b20f154d89..21ed153df4 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -4,8 +4,6 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - "time" - "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -21,6 +19,8 @@ const ( ALIYUN_API_VERSION = "2014-05-26" ALIYUN_BSS_API_VERSION = "2017-12-14" + + ALIYUN_RAM_API_VERSION = "2015-05-01" ) type SAliyunClient struct { @@ -44,10 +44,6 @@ func jsonRequest(client *sdk.Client, apiName string, params map[string]string) ( return _jsonRequest(client, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params) } -func businessRequest(client *sdk.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) { - return _jsonRequest(client, "business.aliyuncs.com", ALIYUN_BSS_API_VERSION, apiName, params) -} - func _jsonRequest(client *sdk.Client, domain string, version string, apiName string, params map[string]string) (jsonutils.JSONObject, error) { req := requests.NewCommonRequest() req.Domain = domain @@ -58,6 +54,7 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str req.QueryParams[k] = v } } + req.Scheme = "https" resp, err := client.ProcessCommonRequest(req) if err != nil { @@ -94,14 +91,6 @@ func (self *SAliyunClient) jsonRequest(apiName string, params map[string]string) return jsonRequest(cli, apiName, params) } -func (self *SAliyunClient) businessRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { - cli, err := self.getDefaultClient() - if err != nil { - return nil, err - } - return businessRequest(cli, apiName, params) -} - func (self *SAliyunClient) fetchRegions() error { body, err := self.jsonRequest("DescribeRegions", map[string]string{"AcceptLanguage": "zh-CN"}) if err != nil { @@ -218,85 +207,3 @@ func (self *SAliyunClient) GetIStoragecacheById(id string) (cloudprovider.ICloud } return nil, cloudprovider.ErrNotFound } - -type SAccountBalance struct { - AvailableAmount float64 - AvailableCashAmount float64 - CreditAmount float64 - MybankCreditAmount float64 - Currency string -} - -type SCashCoupon struct { - ApplicableProducts string - ApplicableScenarios string - Balance float64 - CashCouponId string - CashCouponNo string - EffectiveTime time.Time - ExpiryTime time.Time - GrantedTime time.Time - NominalValue float64 - Status string -} - -type SPrepaidCard struct { - PrepaidCardId string - PrepaidCardNo string - GrantedTime time.Time - EffectiveTime time.Time - ExpiryTime time.Time - NominalValue float64 - Balance float64 - ApplicableProducts string - ApplicableScenarios string -} - -func (self *SAliyunClient) QueryAccountBalance() (*SAccountBalance, error) { - body, err := self.businessRequest("QueryAccountBalance", nil) - if err != nil { - log.Errorf("QueryAccountBalance fail %s", err) - return nil, err - } - balance := SAccountBalance{} - err = body.Unmarshal(&balance, "Data") - if err != nil { - log.Errorf("Unmarshal AccountBalance fail %s", err) - return nil, err - } - return &balance, nil -} - -func (self *SAliyunClient) QueryCashCoupons() ([]SCashCoupon, error) { - params := make(map[string]string) - params["EffectiveOrNot"] = "True" - body, err := self.businessRequest("QueryCashCoupons", params) - if err != nil { - log.Errorf("QueryCashCoupons fail %s", err) - return nil, err - } - coupons := make([]SCashCoupon, 0) - err = body.Unmarshal(&coupons, "Data", "CashCoupon") - if err != nil { - log.Errorf("Unmarshal fail %s", err) - return nil, err - } - return coupons, nil -} - -func (self *SAliyunClient) QueryPrepaidCards() ([]SPrepaidCard, error) { - params := make(map[string]string) - params["EffectiveOrNot"] = "True" - body, err := self.businessRequest("QueryPrepaidCards", params) - if err != nil { - log.Errorf("QueryPrepaidCards fail %s", err) - return nil, err - } - cards := make([]SPrepaidCard, 0) - err = body.Unmarshal(&cards, "Data", "PrepaidCard") - if err != nil { - log.Errorf("Unmarshal fail %s", err) - return nil, err - } - return cards, nil -} diff --git a/pkg/util/aliyun/business.go b/pkg/util/aliyun/business.go new file mode 100644 index 0000000000..544b0f81f0 --- /dev/null +++ b/pkg/util/aliyun/business.go @@ -0,0 +1,104 @@ +package aliyun + +import ( + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" +) + +func businessRequest(client *sdk.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + return _jsonRequest(client, "business.aliyuncs.com", ALIYUN_BSS_API_VERSION, apiName, params) +} + +func (self *SAliyunClient) businessRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient() + if err != nil { + return nil, err + } + return businessRequest(cli, apiName, params) +} + +type SAccountBalance struct { + AvailableAmount float64 + AvailableCashAmount float64 + CreditAmount float64 + MybankCreditAmount float64 + Currency string +} + +type SCashCoupon struct { + ApplicableProducts string + ApplicableScenarios string + Balance float64 + CashCouponId string + CashCouponNo string + EffectiveTime time.Time + ExpiryTime time.Time + GrantedTime time.Time + NominalValue float64 + Status string +} + +type SPrepaidCard struct { + PrepaidCardId string + PrepaidCardNo string + GrantedTime time.Time + EffectiveTime time.Time + ExpiryTime time.Time + NominalValue float64 + Balance float64 + ApplicableProducts string + ApplicableScenarios string +} + +func (self *SAliyunClient) QueryAccountBalance() (*SAccountBalance, error) { + body, err := self.businessRequest("QueryAccountBalance", nil) + if err != nil { + log.Errorf("QueryAccountBalance fail %s", err) + return nil, err + } + balance := SAccountBalance{} + err = body.Unmarshal(&balance, "Data") + if err != nil { + log.Errorf("Unmarshal AccountBalance fail %s", err) + return nil, err + } + return &balance, nil +} + +func (self *SAliyunClient) QueryCashCoupons() ([]SCashCoupon, error) { + params := make(map[string]string) + params["EffectiveOrNot"] = "True" + body, err := self.businessRequest("QueryCashCoupons", params) + if err != nil { + log.Errorf("QueryCashCoupons fail %s", err) + return nil, err + } + coupons := make([]SCashCoupon, 0) + err = body.Unmarshal(&coupons, "Data", "CashCoupon") + if err != nil { + log.Errorf("Unmarshal fail %s", err) + return nil, err + } + return coupons, nil +} + +func (self *SAliyunClient) QueryPrepaidCards() ([]SPrepaidCard, error) { + params := make(map[string]string) + params["EffectiveOrNot"] = "True" + body, err := self.businessRequest("QueryPrepaidCards", params) + if err != nil { + log.Errorf("QueryPrepaidCards fail %s", err) + return nil, err + } + cards := make([]SPrepaidCard, 0) + err = body.Unmarshal(&cards, "Data", "PrepaidCard") + if err != nil { + log.Errorf("Unmarshal fail %s", err) + return nil, err + } + return cards, nil +} diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index ab592b2c5e..d71f9f44b1 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -358,7 +358,6 @@ func (self *SDisk) Reset(snapshotId string) error { return self.storage.zone.region.resetDisk(self.DiskId, snapshotId) } - func (self *SDisk) GetBillingType() string { switch self.DiskChargeType { case PrePaidInstanceChargeType: @@ -372,4 +371,4 @@ func (self *SDisk) GetBillingType() string { func (self *SDisk) GetExpiredAt() time.Time { return self.ExpiredTime -} \ No newline at end of file +} diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 8f116d6746..c79b40d1bc 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -165,8 +165,8 @@ func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, - storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) { - vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, vswitchId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, secgroupId) + storageType string, diskSizes []int, publicKey string, secgroupId string, userData string) (cloudprovider.ICloudVM, error) { + vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, vswitchId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, secgroupId, userData) if err != nil { return nil, err } @@ -180,7 +180,8 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, - storageType string, diskSizes []int, publicKey string, secgroupId string) (string, error) { + storageType string, diskSizes []int, publicKey string, secgroupId string, + userData string) (string, error) { net := self.zone.getNetworkById(vswitchId) if net == nil { return "", fmt.Errorf("invalid switch ID %s", vswitchId) @@ -260,7 +261,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int for _, instType := range instanceTypes { instanceTypeId := instType.InstanceTypeId log.Debugf("Try instancetype : %s", instanceTypeId) - vmId, err := self.zone.region.CreateInstance(name, imgId, instanceTypeId, secgroupId, self.zone.ZoneId, desc, passwd, disks, vswitchId, ipAddr, keypair) + vmId, err := self.zone.region.CreateInstance(name, imgId, instanceTypeId, secgroupId, self.zone.ZoneId, desc, passwd, disks, vswitchId, ipAddr, keypair, userData) if err != nil { log.Errorf("Failed for %s: %s", instanceTypeId, err) } else { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index f564167f00..31e7c1b1b2 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -428,7 +428,7 @@ func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { func (self *SRegion) CreateInstance(name string, imageId string, instanceType string, securityGroupId string, zoneId string, desc string, passwd string, disks []SDisk, vSwitchId string, ipAddr string, - keypair string) (string, error) { + keypair string, userData string) (string, error) { params := make(map[string]string) params["RegionId"] = self.RegionId params["ImageId"] = imageId @@ -468,6 +468,11 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st if len(keypair) > 0 { params["KeyPairName"] = keypair } + + if len(userData) > 0 { + params["UserData"] = userData + } + params["ClientToken"] = utils.GenRequestId(20) body, err := self.ecsRequest("CreateInstance", params) @@ -788,3 +793,7 @@ func (self *SInstance) GetBillingType() string { func (self *SInstance) GetExpiredAt() time.Time { return self.ExpiredTime } + +func (self *SInstance) UpdateUserData(userData string) error { + return self.host.zone.region.updateInstance(self.InstanceId, "", "", "", "", userData) +} diff --git a/pkg/util/aliyun/ram.go b/pkg/util/aliyun/ram.go new file mode 100644 index 0000000000..8e0d2814fb --- /dev/null +++ b/pkg/util/aliyun/ram.go @@ -0,0 +1,222 @@ +package aliyun + +import ( + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +func ramRequest(client *sdk.Client, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + return _jsonRequest(client, "ram.aliyuncs.com", ALIYUN_RAM_API_VERSION, apiName, params) +} + +func (self *SAliyunClient) ramRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient() + if err != nil { + return nil, err + } + return ramRequest(cli, apiName, params) +} + +type SRole struct { + Arn string + CreateDate time.Time + Description string + RoleId string + RoleName string + + AssumeRolePolicyDocument string +} + +func (self *SAliyunClient) ListRoles() ([]SRole, error) { + body, err := self.ramRequest("ListRoles", nil) + if err != nil { + log.Errorf("listRoles fail %s", err) + return nil, err + } + + roles := make([]SRole, 0) + + err = body.Unmarshal(&roles, "Roles", "Role") + if err != nil { + return nil, err + } + + return roles, nil +} + +func (self *SAliyunClient) GetRole(roleName string) (*SRole, error) { + params := make(map[string]string) + params["RoleName"] = roleName + + body, err := self.ramRequest("GetRole", params) + if err != nil { + if isError(err, "EntityNotExist.Role") { + return nil, cloudprovider.ErrNotFound + } + return nil, err + } + + role := SRole{} + + err = body.Unmarshal(&role, "Role") + if err != nil { + return nil, err + } + + return &role, nil +} + +func (self *SAliyunClient) createRole(roleName string, document string, desc string) (*SRole, error) { + params := make(map[string]string) + params["RoleName"] = roleName + params["AssumeRolePolicyDocument"] = document + if len(desc) > 0 { + params["Description"] = desc + } + + body, err := self.ramRequest("CreateRole", params) + if err != nil { + return nil, err + } + + role := SRole{} + + err = body.Unmarshal(&role, "Role") + if err != nil { + return nil, err + } + + return &role, nil +} + +/** + {"AttachmentCount":0, +"CreateDate":"2018-10-12T05:05:16Z", +"DefaultVersion":"v1", +"Description":"只读访问Data Lake Analytics的权限", +"PolicyName":"AliyunDLAReadOnlyAccess", +"PolicyType":"System", +"UpdateDate":"2018-10-12T05:05:16Z"} +*/ + +type SPolicy struct { + AttachmentCount int + CreateDate time.Time + UpdateDate time.Time + DefaultVersion string + Description string + PolicyName string + PolicyType string +} + +func (self *SAliyunClient) ListPolicies(policyType string, role string) ([]SPolicy, error) { + var action string + params := make(map[string]string) + if len(role) > 0 { + params["RoleName"] = role + action = "ListPoliciesForRole" + } else { + params["MaxItems"] = "1000" + if len(policyType) > 0 { + params["PolicyType"] = policyType + } + action = "ListPolicies" + } + + body, err := self.ramRequest(action, params) + if err != nil { + log.Errorf("listPolicies fail %s", err) + return nil, err + } + + policies := make([]SPolicy, 0) + + err = body.Unmarshal(&policies, "Policies", "Policy") + if err != nil { + return nil, err + } + + return policies, nil +} + +func (self *SAliyunClient) GetPolicy(policyType string, policyName string) (*SPolicy, error) { + params := make(map[string]string) + params["PolicyType"] = policyType + params["PolicyName"] = policyName + + body, err := self.ramRequest("GetPolicy", params) + if err != nil { + if isError(err, "EntityNotExist.Role") { + return nil, cloudprovider.ErrNotFound + } + return nil, err + } + + policy := SPolicy{} + + err = body.Unmarshal(&policy, "Policy") + if err != nil { + return nil, err + } + + return &policy, nil +} + +func (self *SAliyunClient) createPolicy(name string, document string, desc string) (*SPolicy, error) { + params := make(map[string]string) + params["PolicyName"] = name + params["PolicyDocument"] = document + if len(desc) > 0 { + params["Description"] = desc + } + + body, err := self.ramRequest("CreatePolicy", params) + if err != nil { + return nil, err + } + + policy := SPolicy{} + + err = body.Unmarshal(&policy, "Policy") + if err != nil { + return nil, err + } + + return &policy, nil +} + +func (self *SAliyunClient) DeletePolicy(policyType string, policyName string) error { + params := make(map[string]string) + params["PolicyName"] = policyName + params["PolicyType"] = policyType + + _, err := self.ramRequest("DeletePolicy", params) + return err +} + +func (self *SAliyunClient) DeleteRole(roleName string) error { + params := make(map[string]string) + params["RoleName"] = roleName + + _, err := self.ramRequest("DeleteRole", params) + return err +} + +func (self *SAliyunClient) attachPolicy2Role(policyType string, policyName string, roleName string) error { + params := make(map[string]string) + params["PolicyType"] = policyType + params["PolicyName"] = policyName + params["RoleName"] = roleName + + _, err := self.ramRequest("AttachPolicyToRole", params) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/util/aliyun/ramimage.go b/pkg/util/aliyun/ramimage.go new file mode 100644 index 0000000000..4236b6b03d --- /dev/null +++ b/pkg/util/aliyun/ramimage.go @@ -0,0 +1,172 @@ +package aliyun + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +const ( + AliyunECSImageImportRole = "AliyunECSImageImportDefaultRole" + AliyunECSImageImportRoleDocument = `{ +"Statement": [ +{ +"Action": "sts:AssumeRole", +"Effect": "Allow", +"Principal": { + "Service": [ + "ecs.aliyuncs.com" + ] +} +} +], +"Version": "1" +}` + + AliyunECSImageImportRolePolicyType = "System" + AliyunECSImageImportRolePolicy = "AliyunECSImageImportRolePolicy" + AliyunECSImageImportRolePolicyDocument = `{ +"Version": "1", +"Statement": [ +{ +"Action": [ + "oss:GetObject", + "oss:GetBucketLocation" +], +"Resource": "*", +"Effect": "Allow" +} +] +}` +) + +func (self *SAliyunClient) EnableImageImport() error { + _, err := self.GetRole(AliyunECSImageImportRole) + if err != nil { + if err != cloudprovider.ErrNotFound { + return err + } + _, err = self.createRole(AliyunECSImageImportRole, + AliyunECSImageImportRoleDocument, + "Allow Import External Image from OSS") + if err != nil { + return err + } + } + + _, err = self.GetPolicy(AliyunECSImageImportRolePolicyType, AliyunECSImageImportRolePolicy) + if err != nil { + /*if err != cloudprovider.ErrNotFound { + return err + } + _, err = self.createPolicy(AliyunECSImageImportRolePolicy, + AliyunECSImageImportRolePolicyDocument, + "Allow Import External Image policy") + if err != nil { + return err + }*/ + return err + } + + policies, err := self.ListPolicies("", AliyunECSImageImportRole) + if err != nil { + return err + } + for i := 0; i < len(policies); i += 1 { + if policies[i].PolicyType == AliyunECSImageImportRolePolicyType && + policies[i].PolicyName == AliyunECSImageImportRolePolicy { + return nil // find policy + } + } + + err = self.attachPolicy2Role(AliyunECSImageImportRolePolicyType, AliyunECSImageImportRolePolicy, AliyunECSImageImportRole) + if err != nil { + return err + } + + return nil +} + +const ( + AliyunECSImageExportRole = "AliyunECSImageExportDefaultRole" + AliyunECSImageExportRoleDocument = `{ + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": [ + "ecs.aliyuncs.com" + ] + } + } + ], + "Version": "1" +}` + + AliyunECSImageExportRolePolicyType = "System" + AliyunECSImageExportRolePolicy = "AliyunECSImageExportRolePolicy" + AliyunECSImageExportRolePolicyDocument = `{ + "Version": "1", + "Statement": [ + { + "Action": [ + "oss:GetObject", + "oss:PutObject", + "oss:DeleteObject", + "oss:GetBucketLocation", + "oss:AbortMultipartUpload", + "oss:ListMultipartUploads", + "oss:ListParts" + ], + "Resource": "*", + "Effect": "Allow" + } + ] + }` +) + +func (self *SAliyunClient) EnableImageExport() error { + _, err := self.GetRole(AliyunECSImageExportRole) + if err != nil { + if err != cloudprovider.ErrNotFound { + return err + } + _, err = self.createRole(AliyunECSImageExportRole, + AliyunECSImageExportRoleDocument, + "Allow Export Import to OSS") + if err != nil { + return err + } + } + + _, err = self.GetPolicy(AliyunECSImageExportRolePolicyType, AliyunECSImageExportRolePolicy) + if err != nil { + /*if err != cloudprovider.ErrNotFound { + return err + } + _, err = self.createPolicy(AliyunECSImageImportRolePolicy, + AliyunECSImageImportRolePolicyDocument, + "Allow Import External Image policy") + if err != nil { + return err + }*/ + return err + } + + policies, err := self.ListPolicies("", AliyunECSImageExportRole) + if err != nil { + return err + } + for i := 0; i < len(policies); i += 1 { + if policies[i].PolicyType == AliyunECSImageExportRolePolicyType && + policies[i].PolicyName == AliyunECSImageExportRolePolicy { + return nil // find policy + } + } + + err = self.attachPolicy2Role(AliyunECSImageExportRolePolicyType, AliyunECSImageExportRolePolicy, AliyunECSImageExportRole) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index 8064ca3829..5c001b6a80 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -475,7 +475,7 @@ func (self *SRegion) CreateInstanceSimple(name string, imgId string, cpu int, me log.Debugf("Search in zone %s", z.LocalName) net := z.getNetworkById(vswitchId) if net != nil { - inst, err := z.getHost().CreateVM(name, imgId, 0, cpu, memGB*1024, vswitchId, "", "", passwd, storageType, dataDiskSizesGB, publicKey, "") + inst, err := z.getHost().CreateVM(name, imgId, 0, cpu, memGB*1024, vswitchId, "", "", passwd, storageType, dataDiskSizesGB, publicKey, "", "") if err != nil { return nil, err } @@ -601,7 +601,7 @@ func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStorag return nil, cloudprovider.ErrNotFound } -func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname string) error { +func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname, userData string) error { params := make(map[string]string) params["InstanceId"] = instId if len(name) > 0 { @@ -616,12 +616,15 @@ func (self *SRegion) updateInstance(instId string, name, desc, passwd, hostname if len(hostname) > 0 { params["HostName"] = hostname } + if len(userData) > 0 { + params["UserData"] = userData + } _, err := self.ecsRequest("ModifyInstanceAttribute", params) return err } func (self *SRegion) UpdateInstancePassword(instId string, passwd string) error { - return self.updateInstance(instId, "", "", passwd, "") + return self.updateInstance(instId, "", "", passwd, "", "") } // func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { diff --git a/pkg/util/aliyun/shell/ram.go b/pkg/util/aliyun/shell/ram.go new file mode 100644 index 0000000000..86cc18a953 --- /dev/null +++ b/pkg/util/aliyun/shell/ram.go @@ -0,0 +1,80 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/aliyun" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ListRolesOptions struct { + } + shellutils.R(&ListRolesOptions{}, "role-list", "List ram roles", func(cli *aliyun.SRegion, args *ListRolesOptions) error { + roles, err := cli.GetClient().ListRoles() + if err != nil { + return err + } + printList(roles, 0, 0, 0, []string{}) + return nil + }) + + type GetRoleOptions struct { + ROLENAME string + } + shellutils.R(&GetRoleOptions{}, "role-show", "Show ram role", func(cli *aliyun.SRegion, args *GetRoleOptions) error { + role, err := cli.GetClient().GetRole(args.ROLENAME) + if err != nil { + return err + } + printObject(role) + return nil + }) + + type ListPoliciesOptions struct { + PolicyType string + Role string + } + shellutils.R(&ListPoliciesOptions{}, "policy-list", "List ram policies", func(cli *aliyun.SRegion, args *ListPoliciesOptions) error { + policies, err := cli.GetClient().ListPolicies(args.PolicyType, args.Role) + if err != nil { + return err + } + printList(policies, 0, 0, 0, []string{}) + return nil + }) + + type GetPolicyOptions struct { + POLICYTYPE string + POLICYNAME string + } + shellutils.R(&GetPolicyOptions{}, "policy-show", "Show ram policy", func(cli *aliyun.SRegion, args *GetPolicyOptions) error { + policy, err := cli.GetClient().GetPolicy(args.POLICYTYPE, args.POLICYNAME) + if err != nil { + return err + } + printObject(policy) + return nil + }) + + type DeletePolicyOptions struct { + POLICYTYPE string + POLICYNAME string + } + shellutils.R(&DeletePolicyOptions{}, "policy-delete", "Delete policy", func(cli *aliyun.SRegion, args *DeletePolicyOptions) error { + return cli.GetClient().DeletePolicy(args.POLICYTYPE, args.POLICYNAME) + }) + + type DeleteRoleOptions struct { + NAME string + } + shellutils.R(&DeleteRoleOptions{}, "role-delete", "Delete role", func(cli *aliyun.SRegion, args *DeleteRoleOptions) error { + return cli.GetClient().DeleteRole(args.NAME) + }) + + shellutils.R(&ListRolesOptions{}, "enable-image-import", "Enable image import privilege", func(cli *aliyun.SRegion, args *ListRolesOptions) error { + return cli.GetClient().EnableImageImport() + }) + + shellutils.R(&ListRolesOptions{}, "enable-image-export", "Enable image export privilege", func(cli *aliyun.SRegion, args *ListRolesOptions) error { + return cli.GetClient().EnableImageExport() + }) +} diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go index f5ce2908d0..30e9e7fb6c 100644 --- a/pkg/util/aliyun/snapshot.go +++ b/pkg/util/aliyun/snapshot.go @@ -107,10 +107,7 @@ func (self *SSnapshot) Delete() error { if self.region == nil { return fmt.Errorf("not init region for snapshot %s", self.SnapshotId) } - params := make(map[string]string) - params["SnapshotId"] = self.SnapshotId - _, err := self.region.ecsRequest("DeleteSnapshot", params) - return err + return self.region.DeleteSnapshot(self.SnapshotId) } func (self *SSnapshot) GetMetadata() *jsonutils.JSONDict { @@ -166,3 +163,10 @@ func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSn return &snapshots[0], nil } } + +func (self *SRegion) DeleteSnapshot(snapshotId string) error { + params := make(map[string]string) + params["SnapshotId"] = snapshotId + _, err := self.ecsRequest("DeleteSnapshot", params) + return err +} diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 595907a14e..960e9cab88 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -120,7 +120,7 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI log.Errorf("GetOssClient err %s", err) return "", err } - bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), self.region.client.providerId)) + bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), imageId)) exist, err := oss.IsBucketExist(bucketName) if err != nil { log.Errorf("IsBucketExist err %s", err) @@ -136,6 +136,9 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI } else { log.Debugf("Bucket %s exists", bucketName) } + + defer oss.DeleteBucket(bucketName) // remove bucket + bucket, err := oss.Bucket(bucketName) if err != nil { log.Errorf("Bucket error %s %s", bucketName, err) @@ -148,6 +151,8 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI return "", err } + defer bucket.DeleteObject(imageId) // remove object + imageBaseName := imageId if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { imageBaseName = fmt.Sprintf("img%s", imageId) @@ -171,6 +176,13 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI log.Debugf("Import image %s", imageName) + // ensure privileges + err = self.region.GetClient().EnableImageImport() + if err != nil { + log.Errorf("fail to enable import privileges: %s", err) + return "", err + } + task, err := self.region.ImportImage(imageName, osArch, osType, osDist, bucketName, imageId) if err != nil { @@ -282,6 +294,12 @@ func (listener *OssProgressListener) ProgressChanged(event *oss.ProgressEvent) { } func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { + err := self.region.GetClient().EnableImageExport() + if err != nil { + log.Errorf("fail to enable export privileges: %s", err) + return nil, err + } + tmpImageFile, err := ioutil.TempFile(path, extId) if err != nil { return nil, err diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go index 02a1cf1039..9b93a16603 100644 --- a/pkg/util/aliyun/vswitch.go +++ b/pkg/util/aliyun/vswitch.go @@ -155,4 +155,4 @@ func (self *SVSwitch) Delete() error { func (self *SVSwitch) GetAllocTimeoutSeconds() int { return 120 // 2 minutes -} \ No newline at end of file +} diff --git a/pkg/util/ansible/const.go b/pkg/util/ansible/const.go new file mode 100644 index 0000000000..06ac34ff47 --- /dev/null +++ b/pkg/util/ansible/const.go @@ -0,0 +1,5 @@ +package ansible + +const ( + PUBLIC_CLOUD_ANSIBLE_USER = "yunionroot" +) diff --git a/pkg/util/ansible/doc.go b/pkg/util/ansible/doc.go new file mode 100644 index 0000000000..611028f5ee --- /dev/null +++ b/pkg/util/ansible/doc.go @@ -0,0 +1 @@ +package ansible // import "yunion.io/x/onecloud/pkg/util/ansible" diff --git a/pkg/util/azure/doc.go b/pkg/util/azure/doc.go new file mode 100644 index 0000000000..34145ae03c --- /dev/null +++ b/pkg/util/azure/doc.go @@ -0,0 +1 @@ +package azure // import "yunion.io/x/onecloud/pkg/util/azure" diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go index a41d838232..cfcf396fe9 100644 --- a/pkg/util/azure/host.go +++ b/pkg/util/azure/host.go @@ -48,7 +48,7 @@ func (self *SHost) Refresh() error { return nil } -func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string) (cloudprovider.ICloudVM, error) { +func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secgroupId string, userData string) (cloudprovider.ICloudVM, error) { nicId := "" if net := self.zone.getNetworkById(networkId); net == nil { return nil, fmt.Errorf("invalid network ID %s", networkId) @@ -57,7 +57,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, } else { nicId = nic.ID } - vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey) + vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, nicId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, userData) if err != nil { self.zone.region.DeleteNetworkInterface(nicId) return nil, err @@ -70,7 +70,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, } } -func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, nicId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string) (string, error) { +func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, nicId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, userData string) (string, error) { computeClient := compute.NewVirtualMachinesClientWithBaseURI(self.zone.region.client.baseUrl, self.zone.region.client.subscriptionId) computeClient.Authorizer = self.zone.region.client.authorizer @@ -119,7 +119,7 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int // //StorageURI: // }, // } - sshKeys := []compute.SSHPublicKey{compute.SSHPublicKey{KeyData: &publicKey}} + sshKeys := []compute.SSHPublicKey{{KeyData: &publicKey}} properties := compute.VirtualMachineProperties{ HardwareProfile: &compute.HardwareProfile{}, StorageProfile: &compute.StorageProfile{ @@ -150,6 +150,10 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int properties.OsProfile.LinuxConfiguration.SSH = &compute.SSHConfiguration{PublicKeys: &sshKeys} } + if len(userData) > 0 { + properties.OsProfile.CustomData = &userData + } + params := compute.VirtualMachine{Location: &self.zone.region.Name, Name: &name, VirtualMachineProperties: &properties} //log.Debugf("Create instance params: %s", jsonutils.Marshal(params).PrettyString()) for _, profile := range self.zone.region.getHardwareProfile(cpu, memMB) { diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index fccae31651..58d8a6babb 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -566,7 +566,7 @@ func (region *SRegion) ReplaceSystemDisk(instanceId, imageId, passwd, publicKey } else { osType := compute.OperatingSystemTypes(image.GetOsType()) disk, _ := region.GetDisk(diskId) - sshKeys := []compute.SSHPublicKey{compute.SSHPublicKey{KeyData: &publicKey}} + sshKeys := []compute.SSHPublicKey{{KeyData: &publicKey}} params := compute.VirtualMachineUpdate{ VirtualMachineProperties: &compute.VirtualMachineProperties{ StorageProfile: &compute.StorageProfile{ @@ -859,3 +859,11 @@ func (self *SInstance) GetBillingType() string { func (self *SInstance) GetExpiredAt() time.Time { return time.Now() } + +func (self *SInstance) UpdateUserData(userData string) error { + params := compute.VirtualMachineUpdate{} + params.OsProfile = &compute.OSProfile{ + CustomData: &userData, + } + return self.host.zone.region.UpdateInstance(self.ID, params) +} diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 38527b827d..c68d1941d4 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -340,7 +340,7 @@ func (self *SRegion) CreateInstanceSimple(name string, imgId string, cpu int, me net := z.getNetworkById(networkId) if net != nil { passwd := seclib2.RandomPassword2(12) - inst, err := z.getHost().CreateVM(name, imgId, 30, cpu, memGB*1024, networkId, "", "", passwd, storageType, dataDiskSizesGB, publicKey, "") + inst, err := z.getHost().CreateVM(name, imgId, 30, cpu, memGB*1024, networkId, "", "", passwd, storageType, dataDiskSizesGB, publicKey, "", "") if err != nil { return nil, err } diff --git a/pkg/util/cloudinit/cloudconfig.go b/pkg/util/cloudinit/cloudconfig.go new file mode 100644 index 0000000000..b622846ca7 --- /dev/null +++ b/pkg/util/cloudinit/cloudconfig.go @@ -0,0 +1,227 @@ +package cloudinit + +import ( + "bytes" + "encoding/base64" + + "fmt" + "strings" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/util/seclib2" + "yunion.io/x/pkg/utils" +) + +/* + * cloudconfig + * Reference: https://cloudinit.readthedocs.io/en/latest/topics/examples.html + * + */ + +type TSudoPolicy string + +const ( + CLOUD_CONFIG_HEADER = "#cloud-config\n" + + USER_SUDO_NOPASSWD = TSudoPolicy("sudo_nopasswd") + USER_SUDO = TSudoPolicy("sudo") + USER_SUDO_DENY = TSudoPolicy("sudo_deny") + USER_SUDO_NONE = TSudoPolicy("") +) + +type SWriteFile struct { + Path string + Permissions string + Owner string + Encoding string + Content string +} + +type SUser struct { + Name string + Passwd string + LockPassword string + SshAuthorizedKeys []string + Sudo string +} + +type SPhoneHome struct { + Url string +} + +type SCloudConfig struct { + Users []SUser + WriteFiles []SWriteFile + Runcmd []string + Bootcmd []string + Packages []string + PhoneHome *SPhoneHome + DisableRoot int + SshPwauth int +} + +func NewWriteFile(path string, content string, perm string, owner string, isBase64 bool) SWriteFile { + f := SWriteFile{} + + f.Path = path + f.Permissions = perm + f.Owner = owner + if isBase64 { + f.Encoding = "b64" + f.Content = base64.StdEncoding.EncodeToString([]byte(content)) + } else { + f.Content = content + } + + return f +} + +func NewUser(name string) SUser { + u := SUser{Name: name} + return u +} + +func (u *SUser) SudoPolicy(policy TSudoPolicy) *SUser { + switch policy { + case USER_SUDO_NOPASSWD: + u.Sudo = "ALL=(ALL) NOPASSWD:ALL" + case USER_SUDO: + u.Sudo = "ALL=(ALL) ALL" + case USER_SUDO_DENY: + u.Sudo = "False" + default: + u.Sudo = "" + } + return u +} + +func (u *SUser) SshKey(key string) *SUser { + if u.SshAuthorizedKeys == nil { + u.SshAuthorizedKeys = make([]string, 0) + } + u.SshAuthorizedKeys = append(u.SshAuthorizedKeys, key) + return u +} + +func (u *SUser) Password(passwd string) *SUser { + if len(passwd) > 0 { + hash, err := seclib2.GeneratePassword(passwd) + if err != nil { + log.Errorf("GeneratePassword error %s", err) + } else { + u.Passwd = hash + } + u.LockPassword = "false" + } + return u +} + +func (conf *SCloudConfig) UserData() string { + var buf bytes.Buffer + jsonConf := jsonutils.Marshal(conf) + buf.WriteString(CLOUD_CONFIG_HEADER) + buf.WriteString(jsonConf.YAMLString()) + return buf.String() +} + +func (conf *SCloudConfig) UserDataBase64() string { + data := conf.UserData() + return base64.StdEncoding.EncodeToString([]byte(data)) +} + +func ParseUserDataBase64(b64data string) (*SCloudConfig, error) { + data, err := base64.StdEncoding.DecodeString(b64data) + if err != nil { + return nil, err + } + return ParseUserData(string(data)) +} + +func ParseUserData(data string) (*SCloudConfig, error) { + if !strings.HasPrefix(data, CLOUD_CONFIG_HEADER) { + msg := "invalid userdata, not starting with #cloud-config" + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + jsonConf, err := jsonutils.ParseYAML(data) + if err != nil { + log.Errorf("parse userdata yaml error %s", err) + return nil, err + } + config := SCloudConfig{} + err = jsonConf.Unmarshal(&config) + if err != nil { + log.Errorf("unable to unmarchal userdata %s", err) + return nil, err + } + return &config, nil +} + +func (conf *SCloudConfig) MergeUser(u SUser) { + for i := 0; i < len(conf.Users); i += 1 { + if u.Name == conf.Users[i].Name { + // find user, merge keys + for j := 0; j < len(u.SshAuthorizedKeys); j += 1 { + if !utils.IsInStringArray(u.SshAuthorizedKeys[j], conf.Users[i].SshAuthorizedKeys) { + conf.Users[i].SshAuthorizedKeys = append(conf.Users[i].SshAuthorizedKeys, u.SshAuthorizedKeys[j]) + } + } + return + } + } + // no such user + conf.Users = append(conf.Users, u) +} + +func (conf *SCloudConfig) MergeWriteFile(f SWriteFile, replace bool) { + for i := 0; i < len(conf.WriteFiles); i += 1 { + if conf.WriteFiles[i].Path == f.Path { + // find file + if replace { + conf.WriteFiles[i].Content = f.Content + conf.WriteFiles[i].Encoding = f.Encoding + conf.WriteFiles[i].Owner = f.Owner + conf.WriteFiles[i].Permissions = f.Permissions + } + return + } + } + // no such file + conf.WriteFiles = append(conf.WriteFiles, f) +} + +func (conf *SCloudConfig) MergeRuncmd(cmd string) { + if !utils.IsInStringArray(cmd, conf.Runcmd) { + conf.Runcmd = append(conf.Runcmd, cmd) + } +} + +func (conf *SCloudConfig) MergeBootcmd(cmd string) { + if !utils.IsInStringArray(cmd, conf.Bootcmd) { + conf.Bootcmd = append(conf.Bootcmd, cmd) + } +} + +func (conf *SCloudConfig) MergePackage(pkg string) { + if !utils.IsInStringArray(pkg, conf.Packages) { + conf.Packages = append(conf.Packages, pkg) + } +} + +func (conf *SCloudConfig) Merge(conf2 *SCloudConfig) { + for _, u := range conf2.Users { + conf.MergeUser(u) + } + for _, f := range conf2.WriteFiles { + conf.MergeWriteFile(f, false) + } + for _, c := range conf2.Runcmd { + conf.MergeRuncmd(c) + } + for _, c := range conf2.Bootcmd { + conf.MergeBootcmd(c) + } + for _, p := range conf2.Packages { + conf.MergePackage(p) + } +} diff --git a/pkg/util/cloudinit/cloudconfig_test.go b/pkg/util/cloudinit/cloudconfig_test.go new file mode 100644 index 0000000000..f7589c4c69 --- /dev/null +++ b/pkg/util/cloudinit/cloudconfig_test.go @@ -0,0 +1,49 @@ +package cloudinit + +import ( + "testing" +) + +func TestSCloudConfig_UserData(t *testing.T) { + usr1 := NewUser("root") + usr1.SshKey("ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa4E8wmIOlmh1G8ZRcU2zpnl2frD2lLKdXpbTeUUZEKYFFlYM8TM5UrKrqrMCd3rFjaYGTKWiQwOiWroXlAXausbbVEI29KY+1Vd26qNyejj+CZO9MCj0naIrqa1V0of3TQY5I2U+ToIkyLqVFWhWVa57v/GUxsV2aNTmUS/qz0OPSCFPbGWWB35rsjwnFwq2jF6E8yJgTGDTYZcsghRi3IWfyfeHbSuWdvn6N8XrPBDmNg7h+GSvO6FJlp6MUw1hscECi13GwqXYgJnLG5RMiFH6s0vhozyHkue1vOTcryPHRQD0Jz/INUSaggH8L1HnYSUavOf4Cw25W9HfzgUBf") + + usr2 := NewUser("yunion") + usr2.Password("123@yunion").SudoPolicy(USER_SUDO_NOPASSWD) + + file1 := NewWriteFile("/etc/ansible/hosts", "gobuild\ncloudev\n", "", "", true) + file2 := NewWriteFile("/etc/hosts", "127.0.0.1 localhost\n", "", "", false) + config := SCloudConfig{ + Users: []SUser{ + usr1, + usr2, + }, + WriteFiles: []SWriteFile{ + file1, + file2, + }, + Runcmd: []string{ + "mkdir /var/run/httpd", + }, + PhoneHome: &SPhoneHome{ + Url: "http://www.yunion.io/$INSTANCE_ID", + }, + DisableRoot: 0, + SshPwauth: 1, + } + userData := config.UserData() + + t.Logf("%s", userData) + + config2, err := ParseUserData(userData) + if err != nil { + t.Errorf("%s", err) + } else { + userData2 := config2.UserData() + t.Logf("%s", userData2) + + if userData != userData2 { + t.Errorf("userData not equal to userData2") + } + } +} diff --git a/pkg/util/cloudinit/doc.go b/pkg/util/cloudinit/doc.go new file mode 100644 index 0000000000..7157b9c1db --- /dev/null +++ b/pkg/util/cloudinit/doc.go @@ -0,0 +1 @@ +package cloudinit // import "yunion.io/x/onecloud/pkg/util/cloudinit" diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index fa098fa474..2ab2bd5350 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -358,7 +358,7 @@ func (self *SHost) GetManagerId() string { } func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, - passwd string, storageType string, diskSizes []int, publicKey string, secGrpId string) (cloudprovider.ICloudVM, error) { + passwd string, storageType string, diskSizes []int, publicKey string, secGrpId string, userData string) (cloudprovider.ICloudVM, error) { log.Debugf("CreateVM") return nil, cloudprovider.ErrNotImplemented -} \ No newline at end of file +} diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 680a28c654..623a4da458 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -244,4 +244,8 @@ func (self *SVirtualMachine) GetBillingType() string { func (self *SVirtualMachine) GetExpiredAt() time.Time { return time.Time{} -} \ No newline at end of file +} + +func (self *SVirtualMachine) UpdateUserData(userData string) error { + return nil +} diff --git a/pkg/util/excelutils/doc.go b/pkg/util/excelutils/doc.go new file mode 100644 index 0000000000..ec5b477f9a --- /dev/null +++ b/pkg/util/excelutils/doc.go @@ -0,0 +1 @@ +package excelutils // import "yunion.io/x/onecloud/pkg/util/excelutils" diff --git a/pkg/util/excelutils/excelutils.go b/pkg/util/excelutils/excelutils.go index f5c1f487d6..3e8cd0f65b 100644 --- a/pkg/util/excelutils/excelutils.go +++ b/pkg/util/excelutils/excelutils.go @@ -1,9 +1,9 @@ package excelutils import ( - "io" "bytes" "fmt" + "io" "os" "github.com/360EntSecGroup-Skylar/excelize" @@ -21,7 +21,7 @@ func decimalBaseMaxWidth(decNum int, base int) int { } width := 0 for decNum > 0 { - decNum = decNum/base + decNum = decNum / base width += 1 } return width @@ -29,9 +29,9 @@ func decimalBaseMaxWidth(decNum int, base int) int { func decimalBaseN(decNum int, base int, width int) (int, int) { b := 1 - for i := 0; i < width - 1; i += 1 { - decNum = decNum/base - b = b*base + for i := 0; i < width-1; i += 1 { + decNum = decNum / base + b = b * base } return decNum, b } @@ -41,8 +41,8 @@ func decimal2Base(decNum int, base int) []int { ret := make([]int, width) for i := width; i > 0; i -= 1 { ith, divider := decimalBaseN(decNum, base, i) - decNum -= ith*divider - ret[width - i] = ith + decNum -= ith * divider + ret[width-i] = ith } return ret } @@ -84,18 +84,18 @@ func Export(data []jsonutils.JSONObject, keys []string, texts []string, writer i exportHeader(xlsx, texts, 1) for i := 0; i < len(data); i += 1 { - exportRow(xlsx, data[i], keys, i + 2) + exportRow(xlsx, data[i], keys, i+2) } return xlsx.Write(writer) } func ExportFile(data []jsonutils.JSONObject, keys []string, texts []string, filename string) error { - writer, err:= os.Create(filename) + writer, err := os.Create(filename) if err != nil { return err } defer writer.Close() return Export(data, keys, texts, writer) -} \ No newline at end of file +} diff --git a/pkg/util/excelutils/excelutils_test.go b/pkg/util/excelutils/excelutils_test.go index 548c9d5eff..c03e408a87 100644 --- a/pkg/util/excelutils/excelutils_test.go +++ b/pkg/util/excelutils/excelutils_test.go @@ -2,7 +2,6 @@ package excelutils import "testing" - func arrayEqual(a1, a2 []int) bool { if len(a1) != len(a2) { return false @@ -17,10 +16,10 @@ func arrayEqual(a1, a2 []int) bool { func TestDecimalBaseMaxWidth(t *testing.T) { cases := []struct { - decIn int + decIn int baseIn int - want int - } { + want int + }{ {100, 10, 3}, {16, 16, 2}, {15, 16, 1}, @@ -33,12 +32,12 @@ func TestDecimalBaseMaxWidth(t *testing.T) { } cases2 := []struct { - decIn int + decIn int baseIn int - width int - want int - want2 int - } { + width int + want int + want2 int + }{ {100, 10, 3, 1, 100}, {16, 16, 2, 1, 16}, {15, 16, 1, 15, 1}, @@ -52,10 +51,10 @@ func TestDecimalBaseMaxWidth(t *testing.T) { } cases3 := []struct { - decIn int + decIn int baseIn int - want []int - } { + want []int + }{ {100, 10, []int{1, 0, 0}}, {16, 16, []int{1, 0}}, {0, 16, []int{0}}, @@ -76,8 +75,8 @@ func TestDecimalBaseMaxWidth(t *testing.T) { cases4 := []struct { decIn int - want string - } { + want string + }{ {0, "A"}, {1, "B"}, {25, "Z"}, diff --git a/pkg/util/imagetools/doc.go b/pkg/util/imagetools/doc.go new file mode 100644 index 0000000000..a19b49421b --- /dev/null +++ b/pkg/util/imagetools/doc.go @@ -0,0 +1 @@ +package imagetools // import "yunion.io/x/onecloud/pkg/util/imagetools" diff --git a/pkg/util/seclib2/aes.go b/pkg/util/seclib2/aes.go new file mode 100644 index 0000000000..dccffa452a --- /dev/null +++ b/pkg/util/seclib2/aes.go @@ -0,0 +1,108 @@ +package seclib2 + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "fmt" + "io" +) + +// https://stackoverflow.com/questions/23897809/different-results-in-go-and-pycrypto-when-using-aes-cfb +// CFB stream with 8 bit segment size +// See http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf +type cfb8 struct { + b cipher.Block + blockSize int + in []byte + out []byte + + decrypt bool +} + +func (x *cfb8) XORKeyStream(dst, src []byte) { + for i := range src { + x.b.Encrypt(x.out, x.in) + copy(x.in[:x.blockSize-1], x.in[1:]) + if x.decrypt { + x.in[x.blockSize-1] = src[i] + } + dst[i] = src[i] ^ x.out[0] + if !x.decrypt { + x.in[x.blockSize-1] = dst[i] + } + } +} + +// NewCFB8Encrypter returns a Stream which encrypts with cipher feedback mode +// (segment size = 8), using the given Block. The iv must be the same length as +// the Block's block size. +func newCFB8Encrypter(block cipher.Block, iv []byte) cipher.Stream { + return newCFB8(block, iv, false) +} + +// NewCFB8Decrypter returns a Stream which decrypts with cipher feedback mode +// (segment size = 8), using the given Block. The iv must be the same length as +// the Block's block size. +func newCFB8Decrypter(block cipher.Block, iv []byte) cipher.Stream { + return newCFB8(block, iv, true) +} + +func newCFB8(block cipher.Block, iv []byte, decrypt bool) cipher.Stream { + blockSize := block.BlockSize() + if len(iv) != blockSize { + // stack trace will indicate whether it was de or encryption + panic("cipher.newCFB: IV length must equal block size") + } + x := &cfb8{ + b: block, + blockSize: blockSize, + out: make([]byte, blockSize), + in: make([]byte, blockSize), + decrypt: decrypt, + } + copy(x.in, iv) + + return x +} + +func toAESKey(k []byte) []byte { + if len(k) > 32 { + return k[0:32] + } else { + for len(k) < 32 { + k = append(k, '$') + } + return k + } +} + +func decryptAES(k, secret []byte) ([]byte, error) { + block, err := aes.NewCipher(toAESKey(k)) + if err != nil { + return nil, err + } + if len(secret) < aes.BlockSize { + return nil, fmt.Errorf("ciphertext too short") + } + iv := secret[:aes.BlockSize] + ciphertext := secret[aes.BlockSize:] + stream := newCFB8Decrypter(block, iv) + stream.XORKeyStream(ciphertext, ciphertext) + return ciphertext, nil +} + +func encryptAES(k, msg []byte) ([]byte, error) { + block, err := aes.NewCipher(toAESKey(k)) + if err != nil { + return nil, err + } + cipherText := make([]byte, aes.BlockSize+len(msg)) + iv := cipherText[:aes.BlockSize] + if _, err = io.ReadFull(rand.Reader, iv); err != nil { + return nil, err + } + stream := newCFB8Encrypter(block, iv) + stream.XORKeyStream(cipherText[aes.BlockSize:], msg) + return cipherText, nil +} diff --git a/pkg/util/seclib2/aes_test.go b/pkg/util/seclib2/aes_test.go new file mode 100644 index 0000000000..09a64b44b9 --- /dev/null +++ b/pkg/util/seclib2/aes_test.go @@ -0,0 +1,24 @@ +package seclib2 + +import "testing" + +func TestAes(t *testing.T) { + secret := "This is a secret for AES!!!" + key := "This is AES key" + + code, err := encryptAES([]byte(key), []byte(secret)) + if err != nil { + t.Errorf("encrypt error %s", err) + return + } + + secret2, err := decryptAES([]byte(key), code) + if err != nil { + t.Errorf("decrypt error %s", err) + return + } + + if secret != string(secret2) { + t.Errorf("aes encrypt/decrypt mismatch! %s != %s", secret, string(secret2)) + } +} diff --git a/pkg/util/seclib2/crypto.go b/pkg/util/seclib2/crypto.go new file mode 100644 index 0000000000..7e15b58509 --- /dev/null +++ b/pkg/util/seclib2/crypto.go @@ -0,0 +1,117 @@ +package seclib2 + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + + "golang.org/x/crypto/ssh" + + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "encoding/base64" + "fmt" + "yunion.io/x/log" +) + +func exportSshPublicKey(pubkey interface{}) ([]byte, error) { + pub, err := ssh.NewPublicKey(pubkey) + if err != nil { + return nil, err + } + return ssh.MarshalAuthorizedKey(pub), nil +} + +func ssh2CryptoPublicKey(key ssh.PublicKey) crypto.PublicKey { + cryptoPub := key.(ssh.CryptoPublicKey) + return cryptoPub.CryptoPublicKey() +} + +func ssh2rsaPublicKey(key ssh.PublicKey) *rsa.PublicKey { + cryptoKey := ssh2CryptoPublicKey(key) + return cryptoKey.(*rsa.PublicKey) +} + +func ssh2dsaPublicKey(key ssh.PublicKey) *dsa.PublicKey { + cryptoKey := ssh2CryptoPublicKey(key) + return cryptoKey.(*dsa.PublicKey) +} + +func ssh2ecdsaPublicKey(key ssh.PublicKey) *ecdsa.PublicKey { + cryptoKey := ssh2CryptoPublicKey(key) + return cryptoKey.(*ecdsa.PublicKey) +} + +func Encrypt(publicKey, origData []byte) ([]byte, error) { + pub, _, _, _, err := ssh.ParseAuthorizedKey(publicKey) + if err != nil { + log.Errorf("parse authorized key error %s", err) + return nil, err + } + if pub.Type() == ssh.KeyAlgoRSA { + return rsa.EncryptOAEP(sha1.New(), rand.Reader, ssh2rsaPublicKey(pub), origData, nil) + } else { + var pubInf interface{} + switch pub.Type() { + case ssh.KeyAlgoDSA: + pubInf = ssh2dsaPublicKey(pub) + case ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521: + pubInf = ssh2ecdsaPublicKey(pub) + default: + return nil, fmt.Errorf("unsupported key type %s", pub.Type()) + } + pubStr, err := exportSshPublicKey(pubInf) + if err != nil { + return nil, err + } + return encryptAES(pubStr, origData) + } +} + +func Decrypt(privateKey, secret []byte) ([]byte, error) { + priv, err := ssh.ParseRawPrivateKey(privateKey) + if err != nil { + return nil, err + } + switch priv.(type) { + case *rsa.PrivateKey: + rsaPriv := priv.(*rsa.PrivateKey) + return rsa.DecryptOAEP(sha1.New(), rand.Reader, rsaPriv, secret, nil) + case *dsa.PrivateKey: + dsaPriv := priv.(*dsa.PrivateKey) + dsaPub, err := exportSshPublicKey(&dsaPriv.PublicKey) + if err != nil { + return nil, err + } + return decryptAES(dsaPub, secret) + case *ecdsa.PrivateKey: + ecdsaPriv := priv.(*ecdsa.PrivateKey) + ecdsaPub, err := exportSshPublicKey(&ecdsaPriv.PublicKey) + if err != nil { + return nil, err + } + return decryptAES(ecdsaPub, secret) + } + return nil, fmt.Errorf("unsupported") +} + +func EncryptBase64(publicKey string, message string) (string, error) { + secretBytes, err := Encrypt([]byte(publicKey), []byte(message)) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(secretBytes), nil +} + +func DecryptBase64(privateKey string, secret string) (string, error) { + secretBytes, err := base64.StdEncoding.DecodeString(secret) + if err != nil { + return "", err + } + msgBytes, err := Decrypt([]byte(privateKey), secretBytes) + if err != nil { + return "", err + } + return string(msgBytes), nil +} diff --git a/pkg/util/seclib2/passwd.go b/pkg/util/seclib2/passwd.go new file mode 100644 index 0000000000..46179b5b8a --- /dev/null +++ b/pkg/util/seclib2/passwd.go @@ -0,0 +1,20 @@ +package seclib2 + +import ( + "fmt" + + "github.com/tredoe/osutil/user/crypt/sha512_crypt" + + "yunion.io/x/pkg/util/seclib" +) + +func GeneratePassword(passwd string) (string, error) { + salt := seclib.RandomPassword(8) + sha512Crypt := sha512_crypt.New() + return sha512Crypt.Generate([]byte(passwd), []byte(fmt.Sprintf("$6$%s", salt))) +} + +func VerifyPassword(passwd string, hash string) error { + sha512Crypt := sha512_crypt.New() + return sha512Crypt.Verify(hash, []byte(passwd)) +} diff --git a/pkg/util/seclib2/passwd_test.go b/pkg/util/seclib2/passwd_test.go new file mode 100644 index 0000000000..4c0c17b5c8 --- /dev/null +++ b/pkg/util/seclib2/passwd_test.go @@ -0,0 +1,18 @@ +package seclib2 + +import "testing" + +func TestGeneratePassword(t *testing.T) { + passwd := "Hello world!" + dk, err := GeneratePassword(passwd) + if err != nil { + t.Errorf("%s", err) + return + } + t.Logf("%s", dk) + + err = VerifyPassword(passwd, dk) + if err != nil { + t.Errorf("fail to verify %s", err) + } +} diff --git a/pkg/util/seclib2/seclib.go b/pkg/util/seclib2/seclib.go index 017cc307aa..c83a9aea00 100644 --- a/pkg/util/seclib2/seclib.go +++ b/pkg/util/seclib2/seclib.go @@ -13,17 +13,17 @@ const ( UPPERS = "ABCDEFGHJKMNPRSTUVWXYZ" PUNC = "()~@#$%^&*-+={}[]:;<>,.?/" - ALL_DIGITS = "0123456789" + ALL_DIGITS = "0123456789" ALL_LETTERS = "abcdefghijklmnopqrstuvwxyz" ALL_UPPERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - ALL_PUNC = "~`!@#$%^&*()-_=+[]{}|:';\",./<>?" + ALL_PUNC = "~`!@#$%^&*()-_=+[]{}|:';\",./<>?" ) type PasswordStrength struct { - Digits int + Digits int Lowercases int Uppercases int - Punctuats int + Punctuats int } var CHARS = fmt.Sprintf("%s%s%s%s", DIGITS, LETTERS, UPPERS, PUNC) diff --git a/pkg/util/seclib2/seclib_test.go b/pkg/util/seclib2/seclib_test.go index 9921aa09fb..cab70f211b 100644 --- a/pkg/util/seclib2/seclib_test.go +++ b/pkg/util/seclib2/seclib_test.go @@ -1,8 +1,8 @@ package seclib2 import ( - "testing" "math/rand" + "testing" "time" ) @@ -12,10 +12,10 @@ func TestRandomPassword2(t *testing.T) { } func TestMeetComplxity(t *testing.T) { - cases := [] struct { - in string + cases := []struct { + in string want bool - } { + }{ {"123456", false}, {"123abcABC!@#", true}, } diff --git a/pkg/util/seclib2/ssh.go b/pkg/util/seclib2/ssh.go new file mode 100644 index 0000000000..0019c0d6f7 --- /dev/null +++ b/pkg/util/seclib2/ssh.go @@ -0,0 +1,97 @@ +package seclib2 + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + + "crypto/dsa" + "golang.org/x/crypto/ssh" + + "encoding/asn1" + "math/big" + "yunion.io/x/log" +) + +func GenerateRSASSHKeypair() (string, string, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + log.Errorf("generate rsa key error %s", err) + return "", "", err + } + + privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} + privateStr := string(pem.EncodeToMemory(privateKeyPEM)) + + pub, err := exportSshPublicKey(&privateKey.PublicKey) + if err != nil { + return "", "", err + } + publicStr := string(pub) + + return privateStr, publicStr, nil +} + +func GenerateDSASSHKeypair() (string, string, error) { + var privateKey dsa.PrivateKey + + params := &privateKey.Parameters + err := dsa.GenerateParameters(params, rand.Reader, dsa.L1024N160) + if err != nil { + log.Errorf("generateParameter error %s", err) + return "", "", err + } + err = dsa.GenerateKey(&privateKey, rand.Reader) + if err != nil { + log.Errorf("generate key error %s", err) + return "", "", err + } + + type DsaASN1 struct { + Version int + P *big.Int + Q *big.Int + G *big.Int + Pub *big.Int + Priv *big.Int + } + + k := DsaASN1{} + k.P = privateKey.P + k.Q = privateKey.Q + k.G = privateKey.G + k.Pub = privateKey.Y + k.Priv = privateKey.X + + privBytes, err := asn1.Marshal(k) + if err != nil { + log.Errorf("asn1 marshal error %s", err) + return "", "", err + } + + privateKeyPEM := &pem.Block{Type: "DSA PRIVATE KEY", Bytes: privBytes} + privateStr := string(pem.EncodeToMemory(privateKeyPEM)) + + pub, err := exportSshPublicKey(&privateKey.PublicKey) + if err != nil { + return "", "", err + } + publicStr := string(pub) + + return privateStr, publicStr, nil +} + +func GetPublicKeyScheme(pubkey ssh.PublicKey) string { + switch pubkey.Type() { + case ssh.KeyAlgoRSA: + return "RSA" + case ssh.KeyAlgoDSA: + return "DSA" + case ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521: + return "ECDSA" + // case ssh.KeyAlgoED25519: + // return "ED" + } + return "UNKNOWN" +} diff --git a/pkg/util/seclib2/ssh_test.go b/pkg/util/seclib2/ssh_test.go new file mode 100644 index 0000000000..0672b7ce41 --- /dev/null +++ b/pkg/util/seclib2/ssh_test.go @@ -0,0 +1,152 @@ +package seclib2 + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "golang.org/x/crypto/ssh" + "testing" +) + +func TestGenerateRSASSHKeypair(t *testing.T) { + priv, pub, _ := GenerateRSASSHKeypair() + t.Logf("%s", priv) + t.Logf("%s", pub) +} + +func TestGenerateDSASSHKeypair(t *testing.T) { + priv, pub, _ := GenerateDSASSHKeypair() + t.Logf("%s", priv) + t.Logf("%s", pub) +} + +func getPublicKeyPem(privateKey string) ([]byte, error) { + block, _ := pem.Decode([]byte(privateKey)) + if block == nil { + return nil, fmt.Errorf("invalid private key") + } + priv, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + + derPkix, err := x509.MarshalPKIXPublicKey(&priv.PublicKey) + if err != nil { + return nil, err + } + + block = &pem.Block{Type: "PUBLIC KEY", Bytes: derPkix} + return pem.EncodeToMemory(block), nil +} + +func getRSAPublicKeySsh(privateKey string) ([]byte, error) { + block, _ := pem.Decode([]byte(privateKey)) + if block == nil { + return nil, fmt.Errorf("invalid private key") + } + priv, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + + return exportSshPublicKey(&priv.PublicKey) +} + +func getDSAPublicKeySsh(privateKey string) ([]byte, error) { + block, _ := pem.Decode([]byte(privateKey)) + if block == nil { + return nil, fmt.Errorf("invalid private key") + } + priv, err := ssh.ParseDSAPrivateKey(block.Bytes) + if err != nil { + return nil, err + } + + return exportSshPublicKey(&priv.PublicKey) +} + +func TestRsaDecryptEncrypt(t *testing.T) { + privateKey, publicKey, err := GenerateRSASSHKeypair() + if err != nil { + t.Errorf("fail to generate keypair %s", err) + return + } + /* publicKey2, err := getPublicKeyPem(privateKey) + if err != nil { + t.Errorf("fail to get public key in pem format %s", err) + return + } */ + pub3, err := getRSAPublicKeySsh(privateKey) + if err != nil { + t.Errorf("fail to get public key in ssh format %s", err) + return + } + + if publicKey != string(pub3) { + t.Errorf("public key mismatch! %s != %s", publicKey, pub3) + return + } + + t.Logf("%s", string(pub3)) + // t.Logf("%s", string(publicKey2)) + + secret := "this is a secret string!!!" + code, err := EncryptBase64(publicKey, secret) + if err != nil { + t.Errorf("rsa encrypt error %s", err) + return + } + t.Logf("%s", code) + secret2, err := DecryptBase64(privateKey, code) + if err != nil { + t.Errorf("rsa decrypt error %s", err) + return + } + if secret != secret2 { + t.Errorf("rsa decrypt/encrypt error! %s != %s", secret2, secret) + return + } +} + +func TestDsaDecryptEncrypt(t *testing.T) { + privateKey, publicKey, err := GenerateDSASSHKeypair() + if err != nil { + t.Errorf("fail to generate keypair %s", err) + return + } + /* publicKey2, err := getPublicKeyPem(privateKey) + if err != nil { + t.Errorf("fail to get public key in pem format %s", err) + return + } */ + pub3, err := getDSAPublicKeySsh(privateKey) + if err != nil { + t.Errorf("fail to get public key in ssh format %s", err) + return + } + + if publicKey != string(pub3) { + t.Errorf("public key mismatch! %s != %s", publicKey, pub3) + return + } + + t.Logf("%s", string(pub3)) + // t.Logf("%s", string(publicKey2)) + + secret := "this is a secret string!!!" + code, err := EncryptBase64(publicKey, secret) + if err != nil { + t.Errorf("dsa encrypt error %s", err) + return + } + t.Logf("%s", code) + secret2, err := DecryptBase64(privateKey, code) + if err != nil { + t.Errorf("rsa decrypt error %s", err) + return + } + if secret != secret2 { + t.Errorf("rsa decrypt/encrypt error! %s != %s", secret2, secret) + return + } +} diff --git a/pkg/yunionconf/models/initdb.go b/pkg/yunionconf/models/initdb.go index 29e63b6006..caa5cad49c 100644 --- a/pkg/yunionconf/models/initdb.go +++ b/pkg/yunionconf/models/initdb.go @@ -16,4 +16,4 @@ func InitDB() error { } } return nil -} \ No newline at end of file +} diff --git a/pkg/yunionconf/options/options.go b/pkg/yunionconf/options/options.go index edf1773e82..009856d54e 100644 --- a/pkg/yunionconf/options/options.go +++ b/pkg/yunionconf/options/options.go @@ -8,4 +8,4 @@ type YunionConfOptions struct { var ( Options YunionConfOptions -) \ No newline at end of file +) diff --git a/pkg/yunionconf/service/service.go b/pkg/yunionconf/service/service.go index a83c89a6d5..ad43746fa1 100644 --- a/pkg/yunionconf/service/service.go +++ b/pkg/yunionconf/service/service.go @@ -37,4 +37,4 @@ func StartService() { log.Errorf("InitDB fail: %s", err) } } -} \ No newline at end of file +} diff --git a/vendor/github.com/tredoe/osutil/AUTHORS.md b/vendor/github.com/tredoe/osutil/AUTHORS.md new file mode 100644 index 0000000000..1a575d3f3b --- /dev/null +++ b/vendor/github.com/tredoe/osutil/AUTHORS.md @@ -0,0 +1,16 @@ +# Authors + +This is the official list of authors for copyright purposes. +This file is distinct from the 'CONTRIBUTORS' file. See the latter for an explanation. + +Names should be added to this file as: + + Name or Organization / (url address) + +(The email address is not required for organizations) + +Please keep the list sorted. + +## Code + +* Jonas mg (https://github.com/tredoe) \ No newline at end of file diff --git a/vendor/github.com/tredoe/osutil/CONTRIBUTORS.md b/vendor/github.com/tredoe/osutil/CONTRIBUTORS.md new file mode 100644 index 0000000000..879ecc2386 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/CONTRIBUTORS.md @@ -0,0 +1,18 @@ +# Contributors + +This is the official list of people who can contribute (and typically +have contributed) to the repository. + +The 'AUTHORS' file lists the copyright holders; this file lists people. For +example, the employees of an organization are listed here but not in 'AUTHORS', +because the organization holds the copyright. + +Names should be added to this file as: + + Name / (url address) + +Please keep the list sorted. + +## Code + +* Jonas mg (https://github.com/tredoe) \ No newline at end of file diff --git a/vendor/github.com/tredoe/osutil/LICENSE-MPL.txt b/vendor/github.com/tredoe/osutil/LICENSE-MPL.txt new file mode 100644 index 0000000000..52d135112e --- /dev/null +++ b/vendor/github.com/tredoe/osutil/LICENSE-MPL.txt @@ -0,0 +1,374 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/tredoe/osutil/user/crypt/AUTHORS.md b/vendor/github.com/tredoe/osutil/user/crypt/AUTHORS.md new file mode 100644 index 0000000000..8eb23dd0c6 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/AUTHORS.md @@ -0,0 +1,8 @@ +### Initial author + +[Jeramey Crawford](https://github.com/jeramey) + +### Other authors + +[Jonas mg](https://github.com/tredoe) + diff --git a/vendor/github.com/tredoe/osutil/user/crypt/LICENSE b/vendor/github.com/tredoe/osutil/user/crypt/LICENSE new file mode 100644 index 0000000000..c39e0de5d0 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012, Jeramey Crawford +Copyright (c) 2013, Jonas mg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/tredoe/osutil/user/crypt/README.md b/vendor/github.com/tredoe/osutil/user/crypt/README.md new file mode 100644 index 0000000000..cbcfb2e523 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/README.md @@ -0,0 +1,25 @@ +crypt +===== +A password hashing library. + +The goal of crypt is to bring a library of many common and popular password +hashing algorithms to Go and to provide a simple and consistent interface to +each of them. As every hashing method is implemented in pure Go, this library +should be as portable as Go itself. + +All hashing methods come with a test suite which verifies their operation +against itself as well as the output of other password hashing implementations +to ensure compatibility with them. + +I hope you find this library to be useful and easy to use! + +Note: forked from + +## Installation + + go get github.com/tredoe/osutil/user/crypt + +## License + +The source files are distributed under a BSD-style license that can be found +in the LICENSE file. diff --git a/vendor/github.com/tredoe/osutil/user/crypt/common/base64.go b/vendor/github.com/tredoe/osutil/user/crypt/common/base64.go new file mode 100644 index 0000000000..ed057d5c93 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/common/base64.go @@ -0,0 +1,60 @@ +// Copyright 2012, Jeramey Crawford +// Copyright 2013, Jonas mg +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file. + +package common + +const alphabet = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +// Base64_24Bit is a variant of Base64 encoding, commonly used with password +// hashing algorithms to encode the result of their checksum output. +// +// The algorithm operates on up to 3 bytes at a time, encoding the following +// 6-bit sequences into up to 4 hash64 ASCII bytes. +// +// 1. Bottom 6 bits of the first byte +// 2. Top 2 bits of the first byte, and bottom 4 bits of the second byte. +// 3. Top 4 bits of the second byte, and bottom 2 bits of the third byte. +// 4. Top 6 bits of the third byte. +// +// This encoding method does not emit padding bytes as Base64 does. +func Base64_24Bit(src []byte) (hash []byte) { + if len(src) == 0 { + return []byte{} // TODO: return nil + } + + hashSize := (len(src) * 8) / 6 + if (len(src) % 6) != 0 { + hashSize += 1 + } + hash = make([]byte, hashSize) + + dst := hash + for len(src) > 0 { + switch len(src) { + default: + dst[0] = alphabet[src[0]&0x3f] + dst[1] = alphabet[((src[0]>>6)|(src[1]<<2))&0x3f] + dst[2] = alphabet[((src[1]>>4)|(src[2]<<4))&0x3f] + dst[3] = alphabet[(src[2]>>2)&0x3f] + src = src[3:] + dst = dst[4:] + case 2: + dst[0] = alphabet[src[0]&0x3f] + dst[1] = alphabet[((src[0]>>6)|(src[1]<<2))&0x3f] + dst[2] = alphabet[(src[1]>>4)&0x3f] + src = src[2:] + dst = dst[3:] + case 1: + dst[0] = alphabet[src[0]&0x3f] + dst[1] = alphabet[(src[0]>>6)&0x3f] + src = src[1:] + dst = dst[2:] + } + } + + return +} diff --git a/vendor/github.com/tredoe/osutil/user/crypt/common/doc.go b/vendor/github.com/tredoe/osutil/user/crypt/common/doc.go new file mode 100644 index 0000000000..8eb6aff2d3 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/common/doc.go @@ -0,0 +1,13 @@ +// Copyright 2012, Jeramey Crawford +// Copyright 2013, Jonas mg +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file. + +// Package common contains routines used by multiple password hashing +// algorithms. +// +// Generally, you will never import this package directly. Many of the +// *_crypt packages will import this package if they require it. +package common diff --git a/vendor/github.com/tredoe/osutil/user/crypt/common/salt.go b/vendor/github.com/tredoe/osutil/user/crypt/common/salt.go new file mode 100644 index 0000000000..22f51cc677 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/common/salt.go @@ -0,0 +1,105 @@ +// Copyright 2012, Jeramey Crawford +// Copyright 2013, Jonas mg +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file. + +package common + +import ( + "crypto/rand" + "errors" + "strconv" +) + +var ( + ErrSaltPrefix = errors.New("invalid magic prefix") + ErrSaltFormat = errors.New("invalid salt format") + ErrSaltRounds = errors.New("invalid rounds") +) + +// Salt represents a salt. +type Salt struct { + MagicPrefix []byte + + SaltLenMin int + SaltLenMax int + + RoundsMin int + RoundsMax int + RoundsDefault int +} + +// Generate generates a random salt of a given length. +// +// The length is set thus: +// +// length > SaltLenMax: length = SaltLenMax +// length < SaltLenMin: length = SaltLenMin +func (s *Salt) Generate(length int) []byte { + if length > s.SaltLenMax { + length = s.SaltLenMax + } else if length < s.SaltLenMin { + length = s.SaltLenMin + } + + saltLen := (length * 6 / 8) + if (length*6)%8 != 0 { + saltLen += 1 + } + salt := make([]byte, saltLen) + rand.Read(salt) + + out := make([]byte, len(s.MagicPrefix)+length) + copy(out, s.MagicPrefix) + copy(out[len(s.MagicPrefix):], Base64_24Bit(salt)) + return out +} + +// GenerateWRounds creates a random salt with the random bytes being of the +// length provided, and the rounds parameter set as specified. +// +// The parameters are set thus: +// +// length > SaltLenMax: length = SaltLenMax +// length < SaltLenMin: length = SaltLenMin +// +// rounds < 0: rounds = RoundsDefault +// rounds < RoundsMin: rounds = RoundsMin +// rounds > RoundsMax: rounds = RoundsMax +// +// If rounds is equal to RoundsDefault, then the "rounds=" part of the salt is +// removed. +func (s *Salt) GenerateWRounds(length, rounds int) []byte { + if length > s.SaltLenMax { + length = s.SaltLenMax + } else if length < s.SaltLenMin { + length = s.SaltLenMin + } + if rounds < 0 { + rounds = s.RoundsDefault + } else if rounds < s.RoundsMin { + rounds = s.RoundsMin + } else if rounds > s.RoundsMax { + rounds = s.RoundsMax + } + + saltLen := (length * 6 / 8) + if (length*6)%8 != 0 { + saltLen += 1 + } + salt := make([]byte, saltLen) + rand.Read(salt) + + roundsText := "" + if rounds != s.RoundsDefault { + roundsText = "rounds=" + strconv.Itoa(rounds) + } + + out := make([]byte, len(s.MagicPrefix)+len(roundsText)+length) + copy(out, s.MagicPrefix) + copy(out[len(s.MagicPrefix):], []byte(roundsText)) + copy(out[len(s.MagicPrefix)+len(roundsText):], Base64_24Bit(salt)) + return out +} diff --git a/vendor/github.com/tredoe/osutil/user/crypt/crypt.go b/vendor/github.com/tredoe/osutil/user/crypt/crypt.go new file mode 100644 index 0000000000..5ded2da8c6 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/crypt.go @@ -0,0 +1,108 @@ +// Copyright 2013, Jonas mg +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file. + +// Package crypt provides interface for password crypt functions and collects +// common constants. +package crypt + +import ( + "errors" + "strings" + + "github.com/tredoe/osutil/user/crypt/common" +) + +var ErrKeyMismatch = errors.New("hashed value is not the hash of the given password") + +// Crypter is the common interface implemented by all crypt functions. +type Crypter interface { + // Generate performs the hashing algorithm, returning a full hash suitable + // for storage and later password verification. + // + // If the salt is empty, a randomly-generated salt will be generated with a + // length of SaltLenMax and number RoundsDefault of rounds. + // + // Any error only can be got when the salt argument is not empty. + Generate(key, salt []byte) (string, error) + + // Verify compares a hashed key with its possible key equivalent. + // Returns nil on success, or an error on failure; if the hashed key is + // diffrent, the error is "ErrKeyMismatch". + Verify(hashedKey string, key []byte) error + + // Cost returns the hashing cost (in rounds) used to create the given hashed + // key. + // + // When, in the future, the hashing cost of a key needs to be increased in + // order to adjust for greater computational power, this function allows one + // to establish which keys need to be updated. + // + // The algorithms based in MD5-crypt use a fixed value of rounds. + Cost(hashedKey string) (int, error) + + // SetSalt sets a different salt. It is used to easily create derivated + // algorithms, i.e. "apr1_crypt" from "md5_crypt". + SetSalt(salt common.Salt) +} + +// Crypt identifies a crypt function that is implemented in another package. +type Crypt uint + +const ( + APR1 Crypt = iota + 1 // import "github.com/tredoe/osutil/user/crypt/apr1_crypt" + MD5 // import "github.com/tredoe/osutil/user/crypt/md5_crypt" + SHA256 // import "github.com/tredoe/osutil/user/crypt/sha256_crypt" + SHA512 // import "github.com/tredoe/osutil/user/crypt/sha512_crypt" + maxCrypt +) + +var cryptPrefixes = make([]string, maxCrypt) + +var crypts = make([]func() Crypter, maxCrypt) + +// RegisterCrypt registers a function that returns a new instance of the given +// crypt function. This is intended to be called from the init function in +// packages that implement crypt functions. +func RegisterCrypt(c Crypt, f func() Crypter, prefix string) { + if c >= maxCrypt { + panic("crypt: RegisterHash of unknown crypt function") + } + crypts[c] = f + cryptPrefixes[c] = prefix +} + +// New returns a new crypter. +func New(c Crypt) Crypter { + f := crypts[c] + if f != nil { + return f() + } + panic("crypt: requested crypt function is unavailable") +} + +// NewFromHash returns a new Crypter using the prefix in the given hashed key. +func NewFromHash(hashedKey string) Crypter { + var f func() Crypter + + if strings.HasPrefix(hashedKey, cryptPrefixes[SHA512]) { + f = crypts[SHA512] + } else if strings.HasPrefix(hashedKey, cryptPrefixes[SHA256]) { + f = crypts[SHA256] + } else if strings.HasPrefix(hashedKey, cryptPrefixes[MD5]) { + f = crypts[MD5] + } else if strings.HasPrefix(hashedKey, cryptPrefixes[APR1]) { + f = crypts[APR1] + } else { + toks := strings.SplitN(hashedKey, "$", 3) + prefix := "$" + toks[1] + "$" + panic("crypt: unknown cryp function from prefix: " + prefix) + } + + if f != nil { + return f() + } + panic("crypt: requested cryp function is unavailable") +} diff --git a/vendor/github.com/tredoe/osutil/user/crypt/sha512_crypt/sha512_crypt.go b/vendor/github.com/tredoe/osutil/user/crypt/sha512_crypt/sha512_crypt.go new file mode 100644 index 0000000000..fd55e88124 --- /dev/null +++ b/vendor/github.com/tredoe/osutil/user/crypt/sha512_crypt/sha512_crypt.go @@ -0,0 +1,254 @@ +// Copyright 2012, Jeramey Crawford +// Copyright 2013, Jonas mg +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file. + +// Package sha512_crypt implements Ulrich Drepper's SHA512-crypt password +// hashing algorithm. +// +// The specification for this algorithm can be found here: +// http://www.akkadia.org/drepper/SHA-crypt.txt +package sha512_crypt + +import ( + "bytes" + "crypto/sha512" + "strconv" + + "github.com/tredoe/osutil/user/crypt" + "github.com/tredoe/osutil/user/crypt/common" +) + +func init() { + crypt.RegisterCrypt(crypt.SHA512, New, MagicPrefix) +} + +const ( + MagicPrefix = "$6$" + SaltLenMin = 1 + SaltLenMax = 16 + RoundsMin = 1000 + RoundsMax = 999999999 + RoundsDefault = 5000 +) + +var _rounds = []byte("rounds=") + +type crypter struct{ Salt common.Salt } + +// New returns a new crypt.Crypter computing the SHA512-crypt password hashing. +func New() crypt.Crypter { + return &crypter{GetSalt()} +} + +func (c *crypter) Generate(key, salt []byte) (string, error) { + var rounds int + var isRoundsDef bool + + if len(salt) == 0 { + salt = c.Salt.GenerateWRounds(SaltLenMax, RoundsDefault) + } + if !bytes.HasPrefix(salt, c.Salt.MagicPrefix) { + return "", common.ErrSaltPrefix + } + + saltToks := bytes.Split(salt, []byte{'$'}) + if len(saltToks) < 3 { + return "", common.ErrSaltFormat + } + + if bytes.HasPrefix(saltToks[2], _rounds) { + isRoundsDef = true + pr, err := strconv.ParseInt(string(saltToks[2][7:]), 10, 32) + if err != nil { + return "", common.ErrSaltRounds + } + rounds = int(pr) + if rounds < RoundsMin { + rounds = RoundsMin + } else if rounds > RoundsMax { + rounds = RoundsMax + } + salt = saltToks[3] + } else { + rounds = RoundsDefault + salt = saltToks[2] + } + + if len(salt) > SaltLenMax { + salt = salt[0:SaltLenMax] + } + + // Compute alternate SHA512 sum with input KEY, SALT, and KEY. + Alternate := sha512.New() + Alternate.Write(key) + Alternate.Write(salt) + Alternate.Write(key) + AlternateSum := Alternate.Sum(nil) // 64 bytes + + A := sha512.New() + A.Write(key) + A.Write(salt) + // Add for any character in the key one byte of the alternate sum. + i := len(key) + for ; i > 64; i -= 64 { + A.Write(AlternateSum) + } + A.Write(AlternateSum[0:i]) + + // Take the binary representation of the length of the key and for every add + // the alternate sum, for every 0 the key. + for i = len(key); i > 0; i >>= 1 { + if (i & 1) != 0 { + A.Write(AlternateSum) + } else { + A.Write(key) + } + } + Asum := A.Sum(nil) + + // Start computation of P byte sequence. + P := sha512.New() + // For every character in the password add the entire password. + for i = 0; i < len(key); i++ { + P.Write(key) + } + Psum := P.Sum(nil) + // Create byte sequence P. + Pseq := make([]byte, 0, len(key)) + for i = len(key); i > 64; i -= 64 { + Pseq = append(Pseq, Psum...) + } + Pseq = append(Pseq, Psum[0:i]...) + + // Start computation of S byte sequence. + S := sha512.New() + for i = 0; i < (16 + int(Asum[0])); i++ { + S.Write(salt) + } + Ssum := S.Sum(nil) + // Create byte sequence S. + Sseq := make([]byte, 0, len(salt)) + for i = len(salt); i > 64; i -= 64 { + Sseq = append(Sseq, Ssum...) + } + Sseq = append(Sseq, Ssum[0:i]...) + + Csum := Asum + + // Repeatedly run the collected hash value through SHA512 to burn CPU cycles. + for i = 0; i < rounds; i++ { + C := sha512.New() + + // Add key or last result. + if (i & 1) != 0 { + C.Write(Pseq) + } else { + C.Write(Csum) + } + // Add salt for numbers not divisible by 3. + if (i % 3) != 0 { + C.Write(Sseq) + } + // Add key for numbers not divisible by 7. + if (i % 7) != 0 { + C.Write(Pseq) + } + // Add key or last result. + if (i & 1) != 0 { + C.Write(Csum) + } else { + C.Write(Pseq) + } + + Csum = C.Sum(nil) + } + + out := make([]byte, 0, 123) + out = append(out, c.Salt.MagicPrefix...) + if isRoundsDef { + out = append(out, []byte("rounds="+strconv.Itoa(rounds)+"$")...) + } + out = append(out, salt...) + out = append(out, '$') + out = append(out, common.Base64_24Bit([]byte{ + Csum[42], Csum[21], Csum[0], + Csum[1], Csum[43], Csum[22], + Csum[23], Csum[2], Csum[44], + Csum[45], Csum[24], Csum[3], + Csum[4], Csum[46], Csum[25], + Csum[26], Csum[5], Csum[47], + Csum[48], Csum[27], Csum[6], + Csum[7], Csum[49], Csum[28], + Csum[29], Csum[8], Csum[50], + Csum[51], Csum[30], Csum[9], + Csum[10], Csum[52], Csum[31], + Csum[32], Csum[11], Csum[53], + Csum[54], Csum[33], Csum[12], + Csum[13], Csum[55], Csum[34], + Csum[35], Csum[14], Csum[56], + Csum[57], Csum[36], Csum[15], + Csum[16], Csum[58], Csum[37], + Csum[38], Csum[17], Csum[59], + Csum[60], Csum[39], Csum[18], + Csum[19], Csum[61], Csum[40], + Csum[41], Csum[20], Csum[62], + Csum[63], + })...) + + // Clean sensitive data. + A.Reset() + Alternate.Reset() + P.Reset() + for i = 0; i < len(Asum); i++ { + Asum[i] = 0 + } + for i = 0; i < len(AlternateSum); i++ { + AlternateSum[i] = 0 + } + for i = 0; i < len(Pseq); i++ { + Pseq[i] = 0 + } + + return string(out), nil +} + +func (c *crypter) Verify(hashedKey string, key []byte) error { + newHash, err := c.Generate(key, []byte(hashedKey)) + if err != nil { + return err + } + if newHash != hashedKey { + return crypt.ErrKeyMismatch + } + return nil +} + +func (c *crypter) Cost(hashedKey string) (int, error) { + saltToks := bytes.Split([]byte(hashedKey), []byte{'$'}) + if len(saltToks) < 3 { + return 0, common.ErrSaltFormat + } + + if !bytes.HasPrefix(saltToks[2], _rounds) { + return RoundsDefault, nil + } + roundToks := bytes.Split(saltToks[2], []byte{'='}) + cost, err := strconv.ParseInt(string(roundToks[1]), 10, 0) + return int(cost), err +} + +func (c *crypter) SetSalt(salt common.Salt) { c.Salt = salt } + +func GetSalt() common.Salt { + return common.Salt{ + MagicPrefix: []byte(MagicPrefix), + SaltLenMin: SaltLenMin, + SaltLenMax: SaltLenMax, + RoundsDefault: RoundsDefault, + RoundsMin: RoundsMin, + RoundsMax: RoundsMax, + } +} diff --git a/vendor/yunion.io/x/jsonutils/compond.go b/vendor/yunion.io/x/jsonutils/compond.go new file mode 100644 index 0000000000..1ffc930c35 --- /dev/null +++ b/vendor/yunion.io/x/jsonutils/compond.go @@ -0,0 +1,13 @@ +package jsonutils + +func (val *JSONValue) isCompond() bool { + return false +} + +func (val *JSONDict) isCompond() bool { + return true +} + +func (val *JSONArray) isCompond() bool { + return true +} diff --git a/vendor/yunion.io/x/jsonutils/interface.go b/vendor/yunion.io/x/jsonutils/interface.go new file mode 100644 index 0000000000..3ee19a10f6 --- /dev/null +++ b/vendor/yunion.io/x/jsonutils/interface.go @@ -0,0 +1,39 @@ +package jsonutils + +func (self *JSONValue) Interface() interface{} { + return nil +} + +func (self *JSONBool) Interface() interface{} { + return self.data +} + +func (self *JSONInt) Interface() interface{} { + return self.data +} + +func (self *JSONFloat) Interface() interface{} { + return self.data +} + +func (self *JSONString) Interface() interface{} { + return self.data +} + +func (self *JSONArray) Interface() interface{} { + ret := make([]interface{}, len(self.data)) + for i := 0; i < len(self.data); i += 1 { + ret[i] = self.data[i].Interface() + } + return ret +} + +func (self *JSONDict) Interface() interface{} { + mapping := make(map[string]interface{}) + + for k, v := range self.data { + mapping[k] = v.Interface() + } + + return mapping +} diff --git a/vendor/yunion.io/x/jsonutils/jsonutils.go b/vendor/yunion.io/x/jsonutils/jsonutils.go index d3321d1caf..2b9f289cfe 100644 --- a/vendor/yunion.io/x/jsonutils/jsonutils.go +++ b/vendor/yunion.io/x/jsonutils/jsonutils.go @@ -65,6 +65,8 @@ type JSONObject interface { Equals(obj JSONObject) bool unmarshalValue(val reflect.Value) error // IsZero() bool + Interface() interface{} + isCompond() bool } type JSONValue struct { diff --git a/vendor/yunion.io/x/jsonutils/yamlutils.go b/vendor/yunion.io/x/jsonutils/yamlutils.go index 1daf0a63a3..fd2b978882 100644 --- a/vendor/yunion.io/x/jsonutils/yamlutils.go +++ b/vendor/yunion.io/x/jsonutils/yamlutils.go @@ -61,43 +61,49 @@ func parseYAMLDict(lines []string) (map[string]JSONObject, error) { } else { key := lines[i][0:keypos] val := strings.Trim(lines[i][keypos+1:], " ") + if len(val) > 0 && val != "|" { - o, e := Parse([]byte(val)) - if e != nil { - return dict, e - } else { - dict[key] = o - } + dict[key] = NewString(val) i++ } else { + sublines := make([]string, 0) j := i + 1 for j < len(lines) && len(strings.Trim(lines[j], " ")) == 0 { + sublines = append(sublines, "") j++ } - if j >= len(lines) || lines[j][0] != ' ' { - return dict, fmt.Errorf("Illformat") - } - indent := 0 - for indent < len(lines[j]) && lines[j][indent] == ' ' { - indent++ - } - sublines := make([]string, 0) - for j < len(lines) { - if indent >= len(lines[j]) && len(strings.Trim(lines[j], " ")) == 0 { - j++ - } else if indent < len(lines[j]) && len(strings.Trim(lines[j][:indent], " ")) == 0 { - sublines = append(sublines, lines[j][indent:]) - j++ - } else { - break + if j < len(lines) { + if lines[j][0] != ' ' { + return dict, fmt.Errorf("Illformat") + } + + indent := 0 + for indent < len(lines[j]) && lines[j][indent] == ' ' { + indent++ + } + + for j < len(lines) { + if indent >= len(lines[j]) && len(strings.Trim(lines[j], " ")) == 0 { + sublines = append(sublines, "") + j++ + } else if indent < len(lines[j]) && len(strings.Trim(lines[j][:indent], " ")) == 0 { + sublines = append(sublines, lines[j][indent:]) + j++ + } else { + break + } } } - o, e := parseYAMLLines(sublines) - if e != nil { - return dict, e + if val == "|" { + dict[key] = NewString(strings.Join(sublines, "\n")) } else { + o, e := parseYAMLLines(sublines) + if e != nil { + return dict, e + } dict[key] = o } + i = j } } @@ -192,8 +198,14 @@ func (this *JSONDict) yamlLines() []string { var ret = make([]string, 0) for _, key := range this.SortedKeys() { val := this.data[key] + if val.IsZero() { + switch val.(type) { + case *JSONString, *JSONDict, *JSONArray, *JSONValue: + continue + } + } lines := val.yamlLines() - if len(lines) == 1 { + if !val.isCompond() && len(lines) == 1 { ret = append(ret, fmt.Sprintf("%s: %s", key, lines[0])) } else { switch val.(type) {