From 6ccdd25f77d9dc46f8d1f666ede909a580998fa0 Mon Sep 17 00:00:00 2001 From: TangBin Date: Tue, 10 Dec 2019 11:02:59 +0800 Subject: [PATCH 1/3] add user & project batch import template --- pkg/apigateway/handler/misc.go | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/pkg/apigateway/handler/misc.go b/pkg/apigateway/handler/misc.go index dc128bcf17..0b02387b8d 100644 --- a/pkg/apigateway/handler/misc.go +++ b/pkg/apigateway/handler/misc.go @@ -22,6 +22,7 @@ import ( "net/http" "strings" "time" + "yunion.io/x/onecloud/pkg/apigateway/options" "github.com/360EntSecGroup-Skylar/excelize" @@ -168,23 +169,46 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW return } + var err error + var content bytes.Buffer switch template { case "BatchHostRegister": records := [][]string{{"MAC地址", "名称", "IPMI地址", "IPMI用户名", "IPMI密码"}} - content, err := writeXlsx("hosts", records) + content, err = writeXlsx("hosts", records) if err != nil { httperrors.InternalServerError(w, "internal server error") return } + case "BatchUserRegister": + records := [][]string{{"用户名（user）", "部门/域（domain）", "是否登录控制台（allow_web_console：true、false）"}} + content, err = writeXlsx("users", records) + if err != nil { + httperrors.InternalServerError(w, "internal server error") + return + } + case "BatchProjectRegister": + var titles []string + if options.Options.NonDefaultDomainProjects { + titles = []string{"项目名称", "域", "配额"} + } else { + titles = []string{"项目名称", "域"} + } - w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - w.Header().Set("Content-Disposition", "Attachment; filename=hosts_template.xlsx") - w.Write(content.Bytes()) - return + records := [][]string{titles} + content, err = writeXlsx("projects", records) + if err != nil { + httperrors.InternalServerError(w, "internal server error") + return + } default: httperrors.InputParameterError(w, "template not found %s", template) return } + + w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + w.Header().Set("Content-Disposition", "Attachment; filename=hosts_template.xlsx") + w.Write(content.Bytes()) + return } func (mh *MiscHandler) postPIUploads(ctx context.Context, w http.ResponseWriter, req *http.Request) { From 083c8465cd16cb9a70d54fc55fcd5684ef10b883 Mon Sep 17 00:00:00 2001 From: TangBin Date: Tue, 10 Dec 2019 15:57:57 +0800 Subject: [PATCH 2/3] user batch create --- pkg/apigateway/handler/misc.go | 98 ++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/pkg/apigateway/handler/misc.go b/pkg/apigateway/handler/misc.go index 0b02387b8d..c1b80b7f57 100644 --- a/pkg/apigateway/handler/misc.go +++ b/pkg/apigateway/handler/misc.go @@ -22,6 +22,8 @@ import ( "net/http" "strings" "time" + + "golang.org/x/sync/errgroup" "yunion.io/x/onecloud/pkg/apigateway/options" "github.com/360EntSecGroup-Skylar/excelize" @@ -97,6 +99,10 @@ func (mh *MiscHandler) PostUploads(ctx context.Context, w http.ResponseWriter, r case "BatchHostRegister": mh.DoBatchHostRegister(ctx, w, req) return + // 用户批量注册 + case "BatchUserRegister": + mh.DoBatchUserRegister(ctx, w, req) + return default: err := httperrors.NewInputParameterError("Unsupported action %s", actions[0]) httperrors.JsonClientError(w, err) @@ -161,6 +167,98 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW appsrv.SendJSON(w, resp) } +func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseWriter, req *http.Request) { + s := FetchSession(ctx, req, "") + files := req.MultipartForm.File + + userfiles, ok := files["users"] + if !ok || len(userfiles) == 0 || userfiles[0] == nil { + e := httperrors.NewInputParameterError("Missing parameter %s", "users") + httperrors.JsonClientError(w, e) + return + } + + fileHeader := userfiles[0].Header + contentType := fileHeader.Get("Content-Type") + if contentType != "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" { + e := httperrors.NewInputParameterError("Wrong content type %s, required application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", contentType) + httperrors.JsonClientError(w, e) + return + } + + file, err := userfiles[0].Open() + defer file.Close() + if err != nil { + log.Errorf(err.Error()) + e := httperrors.NewInternalServerError("can't open file") + httperrors.JsonClientError(w, e) + return + } + + xlsx, err := excelize.OpenReader(file) + if err != nil { + log.Errorf(err.Error()) + e := httperrors.NewInternalServerError("can't parse file") + httperrors.JsonClientError(w, e) + } + + // skipped header row + names := map[string]string{} + domains := map[string]string{} + users := []jsonutils.JSONDict{} + for _, row := range xlsx.GetRows("users")[1:] { + if len(row) < 3 { + log.Debugf("batchHostRegister row length too short (less than 3) %s", row) + } + + if _, ok := names[row[0]]; ok { + e := httperrors.NewClientError("duplicate name %s", row[0]) + httperrors.JsonClientError(w, e) + return + } + + domainId, ok := domains[row[1]] + if !ok { + id, err := modules.Domains.GetId(s, row[1], nil) + if err != nil { + httperrors.JsonClientError(w, httperrors.NewGeneralError(err)) + return + } + + domainId = id + domains[row[1]] = id + } + + user := jsonutils.NewDict() + user.Add(jsonutils.NewString(row[0]), "name") + user.Add(jsonutils.NewString(domainId), "domain_id") + user.Add(jsonutils.NewString("OneCloud@2019"), "password") + if strings.ToLower(row[2]) == "true" { + user.Add(jsonutils.JSONTrue, "allow_web_console") + } else { + user.Add(jsonutils.JSONFalse, "allow_web_console") + } + } + + // batch create + var userG errgroup.Group + for i := range users { + user := users[i] + userG.Go(func() error { + _, err := modules.Users.Create(s, &user) + return err + }) + } + + if err := userG.Wait(); err != nil { + e := httperrors.NewGeneralError(err) + httperrors.GeneralServerError(w, e) + return + } + + appsrv.SendJSON(w, jsonutils.NewDict()) +} + func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) { params := appctx.AppContextParams(ctx) template, ok := params[""] From 256b62997a16bb0819a76929397490da4e0c048d Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 13 Dec 2019 19:20:22 +0800 Subject: [PATCH 3/3] add host iso&pxe register --- pkg/apigateway/handler/misc.go | 109 ++++++++++++++++++++++++------ pkg/mcclient/modules/mod_hosts.go | 39 +++-------- 2 files changed, 98 insertions(+), 50 deletions(-) diff --git a/pkg/apigateway/handler/misc.go b/pkg/apigateway/handler/misc.go index c1b80b7f57..99f1e34bd7 100644 --- a/pkg/apigateway/handler/misc.go +++ b/pkg/apigateway/handler/misc.go @@ -23,14 +23,13 @@ import ( "strings" "time" - "golang.org/x/sync/errgroup" - "yunion.io/x/onecloud/pkg/apigateway/options" - "github.com/360EntSecGroup-Skylar/excelize" + "golang.org/x/sync/errgroup" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/apigateway/options" "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/httperrors" @@ -39,6 +38,19 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" ) +const ( + HOST_MAC = "*MAC地址" + HOST_NAME = "*名称" + HOST_IPMI_ADDR = "*IPMI地址" + HOST_IPMI_USERNAME = "*IPMI用户名" + HOST_IPMI_PASSWORD = "*IPMI密码" + HOST_MNG_IP_ADDR = "*管理口IP地址" + HOST_IPMI_ADDR_OPTIONAL = "IPMI地址" + HOST_IPMI_USERNAME_OPTIONAL = "IPMI用户名" + HOST_IPMI_PASSWORD_OPTIONAL = "IPMI密码" + HOST_MNG_IP_ADDR_OPTIONAL = "管理口IP地址" +) + func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession { token := AppContextToken(ctx) session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion) @@ -142,22 +154,48 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW log.Errorf(err.Error()) e := httperrors.NewInternalServerError("can't parse file") httperrors.JsonClientError(w, e) + return + } + + rows := xlsx.GetRows("hosts") + if len(rows) == 0 { + e := httperrors.NewGeneralError(fmt.Errorf("empty file content")) + httperrors.JsonClientError(w, e) + return } h := "" + paramKeys := []string{} + for _, title := range rows[0] { + switch title { + case HOST_MAC: + paramKeys = append(paramKeys, "access_mac") + case HOST_NAME: + paramKeys = append(paramKeys, "name") + case HOST_IPMI_ADDR, HOST_IPMI_ADDR_OPTIONAL: + paramKeys = append(paramKeys, "ipmi_ip_addr") + case HOST_IPMI_USERNAME, HOST_IPMI_USERNAME_OPTIONAL: + paramKeys = append(paramKeys, "ipmi_username") + case HOST_IPMI_PASSWORD, HOST_IPMI_PASSWORD_OPTIONAL: + paramKeys = append(paramKeys, "ipmi_password") + case HOST_MNG_IP_ADDR, HOST_MNG_IP_ADDR_OPTIONAL: + paramKeys = append(paramKeys, "access_ip") + default: + e := httperrors.NewInternalServerError("empty file content") + httperrors.JsonClientError(w, e) + return + } + } + // skipped header row for _, row := range xlsx.GetRows("hosts")[1:] { - if len(row) < 4 { - log.Debugf("batchHostRegister row length too short (less than 4) %s", row) - } - h = h + strings.Join(row, ",") + "\n" } s := FetchSession(ctx, req, "") params := jsonutils.NewDict() params.Set("hosts", jsonutils.NewString(h)) - resp, err := modules.Hosts.DoBatchRegister(s, params) + resp, err := modules.Hosts.DoBatchRegister(s, paramKeys, params) if err != nil { e := httperrors.NewGeneralError(err) httperrors.JsonClientError(w, e) @@ -200,21 +238,38 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW log.Errorf(err.Error()) e := httperrors.NewInternalServerError("can't parse file") httperrors.JsonClientError(w, e) + return } // skipped header row - names := map[string]string{} + rows := xlsx.GetRows("users") + if len(rows) <= 1 { + e := httperrors.NewInputParameterError("empty file") + httperrors.JsonClientError(w, e) + return + } + + users := []jsonutils.JSONObject{} + names := map[string]bool{} domains := map[string]string{} - users := []jsonutils.JSONDict{} - for _, row := range xlsx.GetRows("users")[1:] { - if len(row) < 3 { - log.Debugf("batchHostRegister row length too short (less than 3) %s", row) + for i, row := range rows[1:] { + name := row[0] + if len(name) == 0 { + e := httperrors.NewClientError("row %d name is empty", i+2) + httperrors.JsonClientError(w, e) + return } - if _, ok := names[row[0]]; ok { + if _, ok := names[name]; ok { e := httperrors.NewClientError("duplicate name %s", row[0]) httperrors.JsonClientError(w, e) return + } else { + names[name] = true + _, err := modules.UsersV3.Get(s, name, nil) + if err == nil { + continue + } } domainId, ok := domains[row[1]] @@ -230,14 +285,16 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW } user := jsonutils.NewDict() - user.Add(jsonutils.NewString(row[0]), "name") + user.Add(jsonutils.NewString(name), "name") user.Add(jsonutils.NewString(domainId), "domain_id") user.Add(jsonutils.NewString("OneCloud@2019"), "password") - if strings.ToLower(row[2]) == "true" { + if strings.ToLower(row[2]) == "true" || strings.ToLower(row[2]) == "1" { user.Add(jsonutils.JSONTrue, "allow_web_console") } else { user.Add(jsonutils.JSONFalse, "allow_web_console") } + + users = append(users, user) } // batch create @@ -245,7 +302,7 @@ func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseW for i := range users { user := users[i] userG.Go(func() error { - _, err := modules.Users.Create(s, &user) + _, err := modules.UsersV3.Create(s, user) return err }) } @@ -271,7 +328,21 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW var content bytes.Buffer switch template { case "BatchHostRegister": - records := [][]string{{"MAC地址", "名称", "IPMI地址", "IPMI用户名", "IPMI密码"}} + records := [][]string{{HOST_MAC, HOST_NAME, HOST_IPMI_ADDR_OPTIONAL, HOST_IPMI_USERNAME_OPTIONAL, HOST_IPMI_PASSWORD_OPTIONAL}} + content, err = writeXlsx("hosts", records) + if err != nil { + httperrors.InternalServerError(w, "internal server error") + return + } + case "BatchHostISORegister": + records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR}} + content, err = writeXlsx("hosts", records) + if err != nil { + httperrors.InternalServerError(w, "internal server error") + return + } + case "BatchHostPXERegister": + records := [][]string{{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_IP_ADDR_OPTIONAL}} content, err = writeXlsx("hosts", records) if err != nil { httperrors.InternalServerError(w, "internal server error") @@ -304,7 +375,7 @@ func (mh *MiscHandler) getDownloadsHandler(ctx context.Context, w http.ResponseW } w.Header().Set("Content-Type", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - w.Header().Set("Content-Disposition", "Attachment; filename=hosts_template.xlsx") + w.Header().Set("Content-Disposition", "Attachment; filename=template.xlsx") w.Write(content.Bytes()) return } diff --git a/pkg/mcclient/modules/mod_hosts.go b/pkg/mcclient/modules/mod_hosts.go index 3e0c97b3bc..79f455f55f 100644 --- a/pkg/mcclient/modules/mod_hosts.go +++ b/pkg/mcclient/modules/mod_hosts.go @@ -16,7 +16,6 @@ package modules import ( "fmt" - "regexp" "strings" "yunion.io/x/jsonutils" @@ -89,7 +88,7 @@ func (this *HostManager) GetIpmiInfo(s *mcclient.ClientSession, id string, param return ret, nil } -func parseHosts(data string) ([]jsonutils.JSONObject, string) { +func parseHosts(titles []string, data string) ([]jsonutils.JSONObject, string) { msg := "" hosts := strings.Split(data, "\n") ret := []jsonutils.JSONObject{} @@ -101,37 +100,15 @@ func parseHosts(data string) ([]jsonutils.JSONObject, string) { } fields := strings.Split(host, ",") - if len(fields) != 5 { - msg += fmt.Sprintf("第%d行： %s (格式不正确)\n", i, host) - continue - } - - // mac address check - if match, err := regexp.MatchString(MACAddressPattern, fields[0]); err != nil || !match { - msg += fmt.Sprintf("第%d行： %s (Mac地址格式不正确)\n", i, host) - continue - } - - // name check - if len(fields[1]) == 0 { - msg += fmt.Sprintf("第%d行： %s (名称不能为空)\n", i, host) - } params := jsonutils.NewDict() - params.Add(jsonutils.NewString(fields[0]), "access_mac") - params.Add(jsonutils.NewString(fields[1]), "name") params.Add(jsonutils.NewString("baremetal"), "host_type") - if len(fields[2]) > 0 { - params.Add(jsonutils.NewString(fields[2]), "ipmi_ip_addr") - } - - if len(fields[3]) > 0 { - params.Add(jsonutils.NewString(fields[3]), "ipmi_username") - } - - if len(fields[4]) > 0 { - params.Add(jsonutils.NewString(fields[4]), "ipmi_password") + for i := range fields { + field := fields[i] + if len(field) > 0 { + params.Add(jsonutils.NewString(field), titles[i]) + } } ret = append(ret, params) @@ -140,13 +117,13 @@ func parseHosts(data string) ([]jsonutils.JSONObject, string) { return ret, msg } -func (this *HostManager) DoBatchRegister(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (this *HostManager) DoBatchRegister(s *mcclient.ClientSession, titles []string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { data, err := params.GetString("hosts") if err != nil { return nil, err } - hosts, msg := parseHosts(data) + hosts, msg := parseHosts(titles, data) if len(msg) > 0 { return nil, httperrors.NewInputParameterError(msg) }