diff --git a/cmd/climc/shell/compute/servers.go b/cmd/climc/shell/compute/servers.go index 507aec1b27..92caa7cebc 100644 --- a/cmd/climc/shell/compute/servers.go +++ b/cmd/climc/shell/compute/servers.go @@ -144,6 +144,8 @@ func init() { cmd.Get("sshport", new(options.ServerIdOptions)) cmd.Get("qemu-info", new(options.ServerIdOptions)) cmd.Get("hardware-info", new(options.ServerIdOptions)) + cmd.Get("screen-dump-show", new(options.ServerScreenDumpOptions)) + cmd.BatchPerform("screen-dump", new(options.ServerIdsOptions)) cmd.GetProperty(&options.ServerStatusStatisticsOptions{}) cmd.GetProperty(&options.ServerProjectStatisticsOptions{}) diff --git a/cmd/climc/shell/compute/serverscreendumps.go b/cmd/climc/shell/compute/serverscreendumps.go new file mode 100644 index 0000000000..dd187b5609 --- /dev/null +++ b/cmd/climc/shell/compute/serverscreendumps.go @@ -0,0 +1,23 @@ +package compute + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/cmd/climc/shell" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type ServerScreenDumpListOptions struct { + options.BaseListOptions + Server string `help:"Id or name of server"` +} + +func (o *ServerScreenDumpListOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + +func init() { + cmd := shell.NewResourceCmd(&modules.ServerScreenDumps) + cmd.List(new(ServerScreenDumpListOptions)) +} diff --git a/pkg/apis/compute/guest_image.go b/pkg/apis/compute/guest_image.go index cf87e4c218..6f8c5c605f 100644 --- a/pkg/apis/compute/guest_image.go +++ b/pkg/apis/compute/guest_image.go @@ -27,3 +27,26 @@ type SImagesInGuest struct { RootImage SSubImage DataImages []SSubImage } + +type SGuestScreenDump struct { + S3AccessKey string + S3SecretKey string + S3Endpoint string + S3BucketName string + S3ObjectName string + S3UseSSL bool +} + +type GuestScreenDumpListInput struct { + Server string +} + +type GetDetailsGuestScreenDumpInput struct { + ObjectName string +} + +type GetDetailsGuestScreenDumpOutput struct { + GuestId string + Name string + ScreenDump string +} diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index b42bd40186..c93a5bcdb5 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -1077,12 +1077,15 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri funcName := fmt.Sprintf("GetDetails%s", specCamel) funcValue := modelValue.MethodByName(funcName) + log.Errorf("MethodByName %s", funcName) if !funcValue.IsValid() || funcValue.IsNil() { + log.Errorf("MethodByName2 %s", funcName) return nil, httperrors.NewSpecNotFoundError("%s %s %s not found", dispatcher.Keyword(), idStr, spec) } outs, err := callFunc(funcValue, funcName, params...) if err != nil { + log.Errorf("MethodByName4 %s", funcName) return nil, err } if len(outs) != 2 { @@ -1092,6 +1095,7 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri resVal := outs[0] errVal := outs[1].Interface() if !gotypes.IsNil(errVal) { + log.Errorf("MethodByName3 %s", funcName) return nil, errVal.(error) } else { if gotypes.IsNil(resVal.Interface()) { diff --git a/pkg/cloudcommon/db/opslog_const.go b/pkg/cloudcommon/db/opslog_const.go index 4d762bcff1..5cf352cfe3 100644 --- a/pkg/cloudcommon/db/opslog_const.go +++ b/pkg/cloudcommon/db/opslog_const.go @@ -267,6 +267,7 @@ const ( ACT_GUEST_CREATE_FROM_IMPORT_SUCC = "guest_create_from_import_succ" ACT_GUEST_CREATE_FROM_IMPORT_FAIL = "guest_create_from_import_fail" ACT_GUEST_PANICKED = "guest_panicked" + ACT_GUEST_SCREEN_DUMP = "guest_screen_dump" ACT_HOST_MAINTENANCE = "host_maintenance" ACT_HOST_DOWN = "host_down" diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index bb6071000a..2fe6958560 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -163,6 +163,7 @@ type CommonOptions struct { type HostCommonOptions struct { CommonOptions + S3ScreenDumpOptions ExecutorSocketPath string `help:"Executor socket path" default:"/var/run/onecloud/exec.sock"` DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"` @@ -176,6 +177,15 @@ type HostCommonOptions struct { Qcow2Preallocation string `help:"Qcow2 image create preallocation" default:"metadata" choices:"disable|metadata|falloc|full"` } +type S3ScreenDumpOptions struct { + S3AccessKey string `help:"s3 access key"` + S3SecretKey string `help:"s3 secret key"` + S3Endpoint string `help:"s3 endpoint"` + S3UseSSL bool `help:"s3 access use ssl"` + S3BucketName string `help:"s3 bucket name" default:"onecloud-screendump"` + S3BucketLifecycleKeepDay int `help:"s3 bucket lifecycle keep day" default:"180"` +} + type DBOptions struct { SqlConnection string `help:"SQL connection string" alias:"connection"` diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 31dd27937e..12d74fd511 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -522,6 +522,10 @@ func (drv *SBaseGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcc return nil, httperrors.ErrNotImplemented } +func (self *SBaseGuestDriver) RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) { + return nil, httperrors.ErrNotImplemented +} + func (drv *SBaseGuestDriver) FetchMonitorUrl(ctx context.Context, guest *models.SGuest) string { s := auth.GetAdminSessionWithPublic(ctx, consts.GetRegion()) tsdbURL, err := tsdb.GetDefaultServiceSourceURL(s, options.Options.MonitorEndpointType) diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 625bd7cc64..ed62bdc4c6 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -1201,6 +1201,17 @@ func (self *SKVMGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcc return res, nil } +func (self *SKVMGuestDriver) RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) { + url := fmt.Sprintf("%s/servers/%s/guest-screen-dump", host.ManagerUri, guest.Id) + httpClient := httputils.GetDefaultClient() + header := mcclient.GetTokenHeaders(userCred) + _, res, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, nil, false) + if err != nil { + return nil, errors.Wrap(err, "host request") + } + return res, nil +} + func (self *SKVMGuestDriver) FetchMonitorUrl(ctx context.Context, guest *models.SGuest) string { if options.Options.KvmMonitorAgentUseMetadataService && !guest.IsSriov() { var metadataIp string diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 133b7b8ffb..43e3506000 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -148,15 +148,27 @@ func (self *SGuest) PerformEvent(ctx context.Context, userCred mcclient.TokenCre } if event == "GUEST_PANICKED" { kwargs := jsonutils.NewDict() - kwargs.Set("reason", data) + kwargs.Set("reason", jsonutils.NewString(event)) + if data.Contains("screen_dump_info") { + screenDumpInfo := api.SGuestScreenDump{} + if err := data.Unmarshal(&screenDumpInfo, "screen_dump_info"); err != nil { + log.Errorf("failed unmarshal screen_dump_info %s", err) + } else { + kwargs.Set("screen_dump_name", jsonutils.NewString(screenDumpInfo.S3ObjectName)) + if _, err := self.SaveGuestScreenDump(ctx, userCred, &screenDumpInfo); err != nil { + log.Errorf("SaveGuestScreenDump failed %s", err) + } + } + } - db.OpsLog.LogEvent(self, db.ACT_GUEST_PANICKED, data.String(), userCred) - logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_PANICKED, data.String(), userCred, true) + db.OpsLog.LogEvent(self, db.ACT_GUEST_PANICKED, kwargs.String(), userCred) + logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_PANICKED, kwargs.String(), userCred, true) notifyclient.EventNotify(ctx, userCred, notifyclient.SEventNotifyParam{ Obj: self, Action: notifyclient.ActionServerPanicked, IsFail: true, }) + } return nil, nil } diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 79773a96fa..436cb4b610 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -255,6 +255,8 @@ type IGuestDriver interface { BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error + + RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error) } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/guestscreendump.go b/pkg/compute/models/guestscreendump.go new file mode 100644 index 0000000000..e9775660c5 --- /dev/null +++ b/pkg/compute/models/guestscreendump.go @@ -0,0 +1,186 @@ +package models + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "strings" + + "yunion.io/x/cloudmux/pkg/multicloud/objectstore" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type SGuestScreenDumpManager struct { + db.SResourceBaseManager +} + +var GuestScreenDumpManager *SGuestScreenDumpManager + +func init() { + db.InitManager(func() { + GuestScreenDumpManager = &SGuestScreenDumpManager{ + SResourceBaseManager: db.NewResourceBaseManager( + SGuestScreenDump{}, + "guest_screen_dumps_tbl", + "guest_screen_dump", + "guest_screen_dumps", + ), + } + GuestScreenDumpManager.SetVirtualObject(GuestScreenDumpManager) + GuestScreenDumpManager.TableSpec().AddIndex(true, "guest_id") + }) +} + +type SGuestScreenDump struct { + db.SResourceBase + + RowId int64 `primary:"true" auto_increment:"true" list:"user"` + GuestId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` + Name string `width:"64" charset:"ascii" nullable:"true" list:"user"` + + // s3 config + S3AccessKey string `width:"64" charset:"ascii" nullable:"true"` + S3SecretKey string `width:"64" charset:"ascii" nullable:"true"` + S3Endpoint string `width:"64" charset:"ascii" nullable:"true" list:"user"` + S3BucketName string `width:"64" charset:"ascii" nullable:"true" list:"user"` + S3UseSsl bool `default:"false" list:"user" create:"optional"` +} + +func (manager *SGuestScreenDumpManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestScreenDumpListInput, +) (*sqlchemy.SQuery, error) { + if query.Server != "" { + iGuest, err := GuestManager.FetchByIdOrName(ctx, userCred, query.Server) + if err != nil { + return q, errors.Wrap(err, "fetch guest") + } + q = q.Equals("guest_id", iGuest.GetId()) + } + return q, nil +} + +func (self *SGuestScreenDump) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, self) +} + +func (self *SGuest) SaveGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, screenDumpInfo *api.SGuestScreenDump) (*SGuestScreenDump, error) { + sd := new(SGuestScreenDump) + sd.SetModelManager(GuestScreenDumpManager, sd) + sd.GuestId = self.GetId() + sd.S3SecretKey = base64.StdEncoding.EncodeToString([]byte(screenDumpInfo.S3SecretKey)) + sd.S3Endpoint = screenDumpInfo.S3Endpoint + sd.S3BucketName = screenDumpInfo.S3BucketName + sd.S3AccessKey = base64.StdEncoding.EncodeToString([]byte(screenDumpInfo.S3AccessKey)) + sd.Name = screenDumpInfo.S3ObjectName + + lockman.LockClass(ctx, GuestScreenDumpManager, self.ProjectId) + defer lockman.ReleaseClass(ctx, GuestScreenDumpManager, self.ProjectId) + + err := GuestScreenDumpManager.TableSpec().Insert(ctx, sd) + if err != nil { + return nil, errors.Wrap(err, "save guest screen_dump") + } + db.OpsLog.LogEvent(self, db.ACT_GUEST_SCREEN_DUMP, sd.Name, userCred) + logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_SCREEN_DUMP, sd.Name, userCred, true) + return sd, nil +} + +func (self *SGuest) GetDetailsScreenDumpShow(ctx context.Context, userCred mcclient.TokenCredential, input *api.GetDetailsGuestScreenDumpInput) (*api.GetDetailsGuestScreenDumpOutput, error) { + if input.ObjectName == "" { + return nil, httperrors.NewMissingParameterError("object_name") + } + q := GuestScreenDumpManager.Query() + q = q.Equals("guest_id", self.Id) + q = q.Equals("name", input.ObjectName) + + screenDump := new(SGuestScreenDump) + err := q.First(screenDump) + if err != nil { + return nil, errors.Wrap(err, "query screenDump") + } + + ak, _ := base64.StdEncoding.DecodeString(screenDump.S3AccessKey) + sk, _ := base64.StdEncoding.DecodeString(screenDump.S3SecretKey) + + url := screenDump.S3Endpoint + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + prefix := "http://" + if screenDump.S3UseSsl { + prefix = "https://" + } + url = prefix + url + } + cfg := objectstore.NewObjectStoreClientConfig(url, string(ak), string(sk)) + s3Client, err := objectstore.NewObjectStoreClient(cfg) + if err != nil { + return nil, errors.Wrap(err, "new minio client") + } + bucket, err := s3Client.GetIBucketByName(screenDump.S3BucketName) + if err != nil { + return nil, errors.Wrapf(err, "get bucket %s", screenDump.S3BucketName) + } + irc, err := bucket.GetObject(ctx, screenDump.Name, nil) + if err != nil { + return nil, errors.Wrapf(err, "get object %s", screenDump.Name) + } + defer irc.Close() + obj, err := io.ReadAll(irc) + if err != nil { + return nil, errors.Wrapf(err, "read object %s", screenDump.Name) + } + ret := new(api.GetDetailsGuestScreenDumpOutput) + + contentType := http.DetectContentType(obj) + base64Encoded := base64.StdEncoding.EncodeToString(obj) + ret.ScreenDump = fmt.Sprintf("data:%s;base64,%s", contentType, base64Encoded) + ret.ScreenDump = base64.StdEncoding.EncodeToString(obj) + ret.GuestId = self.Id + ret.Name = screenDump.Name + return ret, nil +} + +func (self *SGuest) PerformScreenDump(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, + data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.PowerStates != api.VM_POWER_STATES_ON { + return nil, httperrors.NewBadRequestError("can't use qga in vm status: %s", self.Status) + } + + host, _ := self.GetHost() + driver, err := self.GetDriver() + if err != nil { + return nil, err + } + res, err := driver.RequestGuestScreenDump(ctx, userCred, nil, host, self) + if err != nil { + return nil, err + } + screenDumpInfo := api.SGuestScreenDump{} + if err := res.Unmarshal(&screenDumpInfo); err != nil { + return nil, errors.Wrap(err, "unmarshal screen dump info") + } + if _, err := self.SaveGuestScreenDump(ctx, userCred, &screenDumpInfo); err != nil { + return nil, errors.Wrap(err, "failed save ") + } + input := &api.GetDetailsGuestScreenDumpInput{ + ObjectName: screenDumpInfo.S3ObjectName, + } + ret, err := self.GetDetailsScreenDumpShow(ctx, userCred, input) + if err != nil { + return nil, err + } + return jsonutils.Marshal(ret), err +} diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index a124d2a741..a3206f4f76 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -166,6 +166,7 @@ func InitHandlers(app *appsrv.Application) { models.RouteTableAssociationManager, models.RouteTableRouteSetManager, models.InterVpcNetworkRouteSetManager, + models.GuestScreenDumpManager, models.SchedpolicyManager, models.DynamicschedtagManager, diff --git a/pkg/hostman/guestman/guest-agent.go b/pkg/hostman/guestman/guest-agent.go index 2db7028ff7..267248ebe3 100644 --- a/pkg/hostman/guestman/guest-agent.go +++ b/pkg/hostman/guestman/guest-agent.go @@ -147,7 +147,7 @@ func (m *SGuestManager) QgaGetOsInfo(sid string) (jsonutils.JSONObject, error) { return jsonutils.Marshal(res), nil } -func (guest *SKVMGuestInstance) QgaAddNicsConfigure(addNics []*desc.SGuestNetwork) error { +func (guest *SKVMGuestInstance) QgaDeployNicsConfigure(nics []*desc.SGuestNetwork) error { if guest.guestAgent == nil { if err := guest.InitQga(); err != nil { return errors.Wrap(err, "init qga") @@ -156,5 +156,5 @@ func (guest *SKVMGuestInstance) QgaAddNicsConfigure(addNics []*desc.SGuestNetwor if err := guest.guestAgent.GuestPing(1); err != nil { return errors.Wrap(err, "Qga ping") } - return guest.guestAgent.QgaDeployNics(deployapi.GuestNicsToServerNics(addNics)) + return guest.guestAgent.QgaDeployNics(deployapi.GuestNicsToServerNics(nics)) } diff --git a/pkg/hostman/guestman/guesthandlers/guesthandler.go b/pkg/hostman/guestman/guesthandlers/guesthandler.go index 032e42ae70..78b83ab47f 100644 --- a/pkg/hostman/guestman/guesthandlers/guesthandler.go +++ b/pkg/hostman/guestman/guesthandlers/guesthandler.go @@ -112,6 +112,7 @@ func AddGuestTaskHandler(prefix string, app *appsrv.Application) { "qga-set-network": qgaSetNetwork, "qga-get-os-info": qgaGetOsInfo, "start-rescue": guestStartRescue, + "guest-screen-dump": guestScreenDump, } { app.AddHandler("POST", fmt.Sprintf("%s/%s//%s", prefix, keyWord, action), @@ -1019,6 +1020,11 @@ func qgaGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, sid st return gm.QgaGetOsInfo(sid) } +func guestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) { + gm := guestman.GetGuestManager() + return gm.RequestGuestScreenDump(sid) +} + // prepare rescue files func guestStartRescue(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) { return guestman.GetGuestManager().GuestStartRescue(ctx, userCred, sid, body) diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 2b7a4b7c7e..98820e9aa4 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -60,6 +60,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/image/drivers/s3" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/util/cgrouputils" @@ -1917,6 +1918,57 @@ func (m *SGuestManager) ProbeGuestInitStatus(sid string) *compute.HostUploadGues return resp } +func (m *SGuestManager) RequestGuestScreenDump(sid string) (jsonutils.JSONObject, error) { + guest, _ := m.GetServer(sid) + if guest == nil { + return nil, httperrors.NewNotFoundError("guest %s not found", sid) + } + kvmGuest, ok := guest.(*SKVMGuestInstance) + if !ok { + return nil, httperrors.NewBadRequestError("guest %s not kvm instance", sid) + } + + screenDumpPath := kvmGuest.generateScreenDumpPath() + screenDumpName := filepath.Base(screenDumpPath) + c := make(chan interface{}, 0) + kvmGuest.Monitor.ScreenDump(screenDumpPath, func(res string) { + log.Infof("qmp screendump res %s", res) + if len(res) > 0 { + c <- errors.Errorf("qmp screen dump failed: %s", res) + return + } + + if fileutils2.Exists(screenDumpPath) { + log.Infof("screendump success at %s", screenDumpPath) + _, err := s3.Put(context.Background(), screenDumpPath, screenDumpName, 50, 4, nil) + if err != nil { + log.Errorf("faild put screenDumpPath %s to s3 %s", screenDumpPath, err) + c <- err + } else { + screenDumpInfo := compute.SGuestScreenDump{ + S3AccessKey: options.HostOptions.S3AccessKey, + S3SecretKey: options.HostOptions.S3SecretKey, + S3Endpoint: options.HostOptions.S3Endpoint, + S3BucketName: options.HostOptions.S3BucketName, + S3ObjectName: screenDumpName, + S3UseSSL: options.HostOptions.S3UseSSL, + } + c <- jsonutils.Marshal(screenDumpInfo) + log.Infof("put screendump %s success", screenDumpName) + os.Remove(screenDumpPath) + } + } + }) + ret := <-c + switch ret.(type) { + case jsonutils.JSONObject: + return ret.(jsonutils.JSONObject), nil + case error: + return nil, ret.(error) + } + return nil, errors.Errorf("unknown ret of screendump") +} + func SyncGuestNicsTraffics(guestNicsTraffics map[string]map[string]compute.SNicTrafficRecord) { session := hostutils.GetComputeSession(context.Background()) hostId := guestManager.host.GetHostId() diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index bce0144c91..f3cf434db0 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -616,6 +616,7 @@ type SGuestNetworkSyncTask struct { addNics []*desc.SGuestNetwork errors []error + delNicCnt int addNicMacs []string addNicConfs []*monitor.NetworkModify @@ -625,30 +626,25 @@ type SGuestNetworkSyncTask struct { func (n *SGuestNetworkSyncTask) Start(callback func(...error)) { n.callback = callback if len(n.addNics) > 0 { - nics := make([]*desc.SGuestNetwork, 0) - nics = append(nics, n.guest.Desc.Nics...) - nics = append(nics, n.addNics...) - if err := n.guest.QgaAddNicsConfigure(nics); err != nil { - log.Errorf("QgaAddNicsConfigure failed %s", err) - } else { - addNicMacs := make([]string, 0) - addNicConfs := make([]*monitor.NetworkModify, 0) - for i := range n.addNics { - addNicMacs = append(addNicMacs, n.addNics[i].Mac) - netMod := &monitor.NetworkModify{} - if len(n.addNics[i].Ip) > 0 { - netMod.Ipmask = fmt.Sprintf("%s/%d", n.addNics[i].Ip, n.addNics[i].Masklen) - netMod.Gateway = n.addNics[i].Gateway - } - if len(n.addNics[i].Ip6) > 0 { - netMod.Ip6mask = fmt.Sprintf("%s/%d", n.addNics[i].Ip6, n.addNics[i].Masklen6) - } - addNicConfs = append(addNicConfs, netMod) + addNicMacs := make([]string, 0) + addNicConfs := make([]*monitor.NetworkModify, 0) + for i := range n.addNics { + addNicMacs = append(addNicMacs, n.addNics[i].Mac) + netMod := &monitor.NetworkModify{} + if len(n.addNics[i].Ip) > 0 { + netMod.Ipmask = fmt.Sprintf("%s/%d", n.addNics[i].Ip, n.addNics[i].Masklen) + netMod.Gateway = n.addNics[i].Gateway } - n.addNicMacs = addNicMacs - n.addNicConfs = addNicConfs + if len(n.addNics[i].Ip6) > 0 { + netMod.Ip6mask = fmt.Sprintf("%s/%d", n.addNics[i].Ip6, n.addNics[i].Masklen6) + } + addNicConfs = append(addNicConfs, netMod) } + n.addNicMacs = addNicMacs + n.addNicConfs = addNicConfs } + + n.delNicCnt = len(n.delNics) n.syncNetworkConf() } @@ -662,13 +658,22 @@ func (n *SGuestNetworkSyncTask) syncNetworkConf() { n.addNics = n.addNics[:len(n.addNics)-1] n.addNic(nic) } else { - if len(n.addNicMacs) > 0 { - // try restart added nics, wait for added nic ready - time.Sleep(3 * time.Second) - if err := n.qgaRestartAddedNics(); err != nil { - log.Errorf("failed qgaRestartAddedNics") + func() { + if len(n.addNicMacs) > 0 || n.delNicCnt > 0 { + if err := n.guest.QgaDeployNicsConfigure(n.guest.Desc.Nics); err != nil { + log.Errorf("failed do QgaDeployNicsConfigure %s", err) + return + } } - } + if len(n.addNicMacs) > 0 { + // try restart added nics, wait for added nic ready + time.Sleep(3 * time.Second) + if err := n.qgaRestartAddedNics(); err != nil { + log.Errorf("failed qgaRestartAddedNics %s", err) + return + } + } + }() n.callback(n.errors...) } @@ -872,7 +877,7 @@ func (n *SGuestNetworkSyncTask) onDeviceAdd(nic *desc.SGuestNetwork) { func NewGuestNetworkSyncTask( guest *SKVMGuestInstance, delNics, addNics []*desc.SGuestNetwork, ) *SGuestNetworkSyncTask { - return &SGuestNetworkSyncTask{guest, delNics, addNics, make([]error, 0), nil, nil, nil} + return &SGuestNetworkSyncTask{guest, delNics, addNics, make([]error, 0), 0, nil, nil, nil} } /** diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 211740901e..b5b0bce6c0 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -58,6 +58,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils" "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/image/drivers/s3" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" @@ -492,6 +493,10 @@ func (s *SKVMGuestInstance) getQemuLogPath() string { return path.Join(s.HomeDir(), "qemu.log") } +func (s *SKVMGuestInstance) generateScreenDumpPath() string { + return path.Join(s.HomeDir(), fmt.Sprintf("%s.%d", s.GetId(), time.Now().Unix())) +} + func (s *SKVMGuestInstance) RecycleDir() string { return path.Join(s.manager.ServersPath, "recycle") } @@ -1126,6 +1131,42 @@ func (s *SKVMGuestInstance) eventGuestPaniced(event *monitor.Event) { if info, ok := event.Data["info"]; ok { params.Set("info", jsonutils.Marshal(info)) } + + screenDumpPath := s.generateScreenDumpPath() + screenDumpName := filepath.Base(screenDumpPath) + + c := make(chan struct{}) + s.Monitor.ScreenDump(screenDumpPath, func(res string) { + log.Infof("qmp screendump res %s", res) + if fileutils2.Exists(screenDumpPath) { + log.Infof("screendump success at %s", screenDumpPath) + _, err := s3.Put(context.Background(), screenDumpPath, screenDumpName, 50, 4, nil) + if err != nil { + log.Errorf("faild put screenDumpPath %s to s3 %s", screenDumpPath, err) + } else { + screenDumpInfo := api.SGuestScreenDump{ + S3AccessKey: options.HostOptions.S3AccessKey, + S3SecretKey: options.HostOptions.S3SecretKey, + S3Endpoint: options.HostOptions.S3Endpoint, + S3BucketName: options.HostOptions.S3BucketName, + S3ObjectName: screenDumpName, + S3UseSSL: options.HostOptions.S3UseSSL, + } + params.Set("screen_dump_info", jsonutils.Marshal(screenDumpInfo)) + log.Infof("put screendump %s success", screenDumpName) + os.Remove(screenDumpPath) + } + } + c <- struct{}{} + }) + // wait screen dump + select { + case <-time.After(time.Second * 5): + log.Errorf("ScreenDump no response after 5 seconds") + case <-c: + break + } + params.Set("event", jsonutils.NewString(strings.Trim(event.Event, "\""))) _, err := modules.Servers.PerformAction( hostutils.GetComputeSession(context.Background()), diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index 94a2c6b73c..e218472c63 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -15,6 +15,8 @@ package hostman import ( + "fmt" + "strings" "time" execlient "yunion.io/x/executor/client" @@ -40,6 +42,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/hostman/storageman/diskhandlers" "yunion.io/x/onecloud/pkg/hostman/storageman/storagehandler" + "yunion.io/x/onecloud/pkg/image/drivers/s3" losetupman "yunion.io/x/onecloud/pkg/util/losetup/manager" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/qemuimg" @@ -102,6 +105,8 @@ func (host *SHostService) RunService() { log.Fatalf("Storage manager init error: %v", err) } + initS3() + var guestChan chan struct{} if err := guestman.Init(hostInstance, options.HostOptions.ServersPath, options.HostOptions.DeployConcurrent); err != nil { @@ -167,6 +172,41 @@ func (host *SHostService) initHandlers(app *appsrv.Application) { app_common.ExportOptionsHandler(app, &options.HostOptions) } +func initS3() { + url := options.HostOptions.S3Endpoint + if len(url) == 0 { + return + } + + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + prefix := "http://" + if options.HostOptions.S3UseSSL { + prefix = "https://" + } + url = prefix + url + } + err := s3.Init( + url, + options.HostOptions.S3AccessKey, + options.HostOptions.S3SecretKey, + options.HostOptions.S3BucketName, + options.HostOptions.S3UseSSL, + "", + ) + if err != nil { + log.Fatalf("failed init s3 client %s", err) + } + + lifecycle := fmt.Sprintf( + `%sEnabled%d`, + options.HostOptions.S3BucketName, options.HostOptions.S3BucketLifecycleKeepDay, + ) + err = s3.SetBucketLifecycle(lifecycle) + if err != nil { + log.Fatalf("failed set bucket lifecycle %s", lifecycle) + } +} + func StartService() { var srv = &SHostService{} srv.SServiceBase = &service.SServiceBase{ diff --git a/pkg/hostman/monitor/hmp.go b/pkg/hostman/monitor/hmp.go index 7b0347ba92..4611baa997 100644 --- a/pkg/hostman/monitor/hmp.go +++ b/pkg/hostman/monitor/hmp.go @@ -567,6 +567,10 @@ func (m *HmpMonitor) QueryMachines(callback QueryMachinesCallback) { go callback(nil, "unsupported query machines for hmp") } +func (m *HmpMonitor) ScreenDump(savePath string, callback StringCallback) { + m.HumanMonitorCommand(fmt.Sprintf("screendump %s", savePath), callback) +} + func (m *HmpMonitor) Quit(cb StringCallback) { m.Query("quit", cb) } diff --git a/pkg/hostman/monitor/monitor.go b/pkg/hostman/monitor/monitor.go index 126cb6a117..598ac149d0 100644 --- a/pkg/hostman/monitor/monitor.go +++ b/pkg/hostman/monitor/monitor.go @@ -252,6 +252,7 @@ type Monitor interface { NetdevAdd(id, netType string, params map[string]string, callback StringCallback) NetdevDel(id string, callback StringCallback) + ScreenDump(savePath string, callback StringCallback) SaveState(statFilePath string, callback StringCallback) QueryMachines(callback QueryMachinesCallback) Quit(StringCallback) diff --git a/pkg/hostman/monitor/qmp.go b/pkg/hostman/monitor/qmp.go index 4cfa432599..ce49de2a08 100644 --- a/pkg/hostman/monitor/qmp.go +++ b/pkg/hostman/monitor/qmp.go @@ -987,6 +987,10 @@ func (m *QmpMonitor) CancelBlockJob(driveName string, force bool, callback Strin m.HumanMonitorCommand(cmd, callback) } +func (m *QmpMonitor) ScreenDump(savePath string, callback StringCallback) { + m.HumanMonitorCommand(fmt.Sprintf("screendump %s", savePath), callback) +} + func (m *QmpMonitor) BlockJobComplete(drive string, callback StringCallback) { m.HumanMonitorCommand(fmt.Sprintf("block_job_complete %s", drive), callback) } diff --git a/pkg/image/drivers/s3/minio.go b/pkg/image/drivers/s3/minio.go index 6350f46762..6fe6e354e2 100644 --- a/pkg/image/drivers/s3/minio.go +++ b/pkg/image/drivers/s3/minio.go @@ -103,6 +103,13 @@ func ensureBucket() error { return nil } +func SetBucketLifecycle(lifecycle string) error { + if client == nil { + return ErrClientNotInit + } + return client.osc.SetBucketLifecycle(client.bucket, lifecycle) +} + func PutStream(ctx context.Context, file io.ReaderAt, fSize int64, objName string, partSizeMb int64, parallel int, progresser func(saved int64)) (string, error) { if client == nil { return "", ErrClientNotInit diff --git a/pkg/mcclient/modules/compute/mod_screendump.go b/pkg/mcclient/modules/compute/mod_screendump.go new file mode 100644 index 0000000000..0ce34370fb --- /dev/null +++ b/pkg/mcclient/modules/compute/mod_screendump.go @@ -0,0 +1,18 @@ +package compute + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + ServerScreenDumps modulebase.ResourceManager +) + +func init() { + ServerScreenDumps = modules.NewComputeManager("guest_screen_dump", "guest_screen_dumps", + []string{"Guest_id", "Name", "Created_at", "S3_endpoint", "S3_bucket_name"}, + []string{}) + + modules.RegisterCompute(&ServerScreenDumps) +} diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index 66735f4dee..7819459896 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -1520,6 +1520,15 @@ func (o *ServerVncOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(o), nil } +type ServerScreenDumpOptions struct { + ServerIdOptions + ObjectName string +} + +func (o *ServerScreenDumpOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + type ServerIsoOptions struct { ServerIdOptions Ordinal int `help:"server iso ordinal, default 0"` diff --git a/pkg/util/logclient/consts.go b/pkg/util/logclient/consts.go index 6d1b5019b5..9c785d96a2 100644 --- a/pkg/util/logclient/consts.go +++ b/pkg/util/logclient/consts.go @@ -144,6 +144,7 @@ const ( ACT_HOST_IMPORT_LIBVIRT_SERVERS = "host_import_libvirt_servers" ACT_GUEST_CREATE_FROM_IMPORT = "guest_create_from_import" ACT_GUEST_PANICKED = "guest_panicked" + ACT_GUEST_SCREEN_DUMP = "guest_screen_dump" ACT_HOST_MAINTAINING = "host_maintaining" ACT_HOST_UNMAINTENANCE = "host_unmaintenance"