diff --git a/cmd/climc/shell/pprof.go b/cmd/climc/shell/pprof.go new file mode 100644 index 0000000000..6222ea1af8 --- /dev/null +++ b/cmd/climc/shell/pprof.go @@ -0,0 +1,87 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "syscall" + + "yunion.io/x/pkg/util/signalutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + type TraceOptions struct { + Second int `help:"pprof seconds" short-token:"s"` + SERVICE string `help:"Service type"` + } + + downloadToTemp := func(input io.Reader, pattern string) (string, error) { + tmpfile, err := ioutil.TempFile("", pattern) + if err != nil { + return "", err + } + defer tmpfile.Close() + if _, err := io.Copy(tmpfile, input); err != nil { + return "", err + } + return tmpfile.Name(), nil + } + + pprofRun := func(s *mcclient.ClientSession, svcType, pType string, second int, args ...string) error { + src, err := modules.GetPProfByType(s, svcType, pType, second) + if err != nil { + return err + } + tempfile, err := downloadToTemp(src, pType) + if err != nil { + return err + } + defer func() { os.Remove(tempfile) }() + + signalutils.RegisterSignal(func() { + os.Remove(tempfile) + os.Exit(0) + }, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) + signalutils.StartTrap() + + cmd := procutils.NewCommand("go", "tool") + cmd.Args = append(cmd.Args, args...) + cmd.Args = append(cmd.Args, tempfile) + if _, err := cmd.Run(); err != nil { + return err + } + return nil + } + + R(&TraceOptions{}, "pprof-trace", "pprof trace of backend service", func(s *mcclient.ClientSession, args *TraceOptions) error { + return pprofRun(s, args.SERVICE, "trace", args.Second, "trace") + }) + + R(&TraceOptions{}, "pprof-profile", "pprof profile of backend service", func(s *mcclient.ClientSession, args *TraceOptions) error { + port, err := netutils2.GetFreePort() + if err != nil { + return err + } + return pprofRun(s, args.SERVICE, "profile", args.Second, "pprof", fmt.Sprintf("-http=:%d", port)) + }) +} diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index d9865a6d61..ee2f16bf02 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -249,8 +249,16 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri if to == 0 { to = app.processTimeout } - ctx, cancel := context.WithTimeout(app.context, to) - defer cancel() + var ( + ctx context.Context = app.context + cancel context.CancelFunc = nil + ) + if to > 0 { + ctx, cancel = context.WithTimeout(app.context, to) + } + if cancel != nil { + defer cancel() + } session := hand.workerMan if session == nil { if r.Method == "GET" || r.Method == "HEAD" { @@ -434,6 +442,7 @@ func (app *Application) ListenAndServeWithoutCleanup(addr, certFile, keyFile str func (app *Application) ListenAndServeTLSWithCleanup2(addr string, certFile, keyFile string, onStop func(), isMaster bool) { if isMaster { app.addDefaultHandlers() + AddPProfHandler(app) } s := app.initServer(addr) if isMaster { diff --git a/pkg/appsrv/handlerinfo.go b/pkg/appsrv/handlerinfo.go index 224b0b3e74..dde6d05581 100644 --- a/pkg/appsrv/handlerinfo.go +++ b/pkg/appsrv/handlerinfo.go @@ -115,6 +115,11 @@ func (hi *SHandlerInfo) SetProcessTimeout(to time.Duration) *SHandlerInfo { return hi } +func (hi *SHandlerInfo) SetProcessNoTimeout() *SHandlerInfo { + hi.processTimeout = -1 + return hi +} + func (hi *SHandlerInfo) SetWorkerManager(workerMan *SWorkerManager) *SHandlerInfo { hi.workerMan = workerMan return hi diff --git a/pkg/appsrv/handlers.go b/pkg/appsrv/handlers.go index 78e0e72c51..40a5592e0a 100644 --- a/pkg/appsrv/handlers.go +++ b/pkg/appsrv/handlers.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "net/http/pprof" "yunion.io/x/pkg/util/version" ) @@ -47,3 +48,33 @@ func CORSHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Max-Age", "86400") } }*/ + +func AddPProfHandler(app *Application) { + prefix := "/debug/pprof" + app.AddHandler("GET", fmt.Sprintf("%s/", prefix), profIndex).SetProcessNoTimeout() + app.AddHandler("GET", fmt.Sprintf("%s/cmdline", prefix), profCmdline).SetProcessNoTimeout() + app.AddHandler("GET", fmt.Sprintf("%s/profile", prefix), profProfile).SetProcessNoTimeout() + app.AddHandler("GET", fmt.Sprintf("%s/symbol", prefix), profSymbol).SetProcessNoTimeout() + app.AddHandler("POST", fmt.Sprintf("%s/symbol", prefix), profSymbol).SetProcessNoTimeout() + app.AddHandler("GET", fmt.Sprintf("%s/trace", prefix), profTrace).SetProcessNoTimeout() +} + +func profIndex(_ context.Context, w http.ResponseWriter, r *http.Request) { + pprof.Index(w, r) +} + +func profCmdline(_ context.Context, w http.ResponseWriter, r *http.Request) { + pprof.Cmdline(w, r) +} + +func profProfile(_ context.Context, w http.ResponseWriter, r *http.Request) { + pprof.Profile(w, r) +} + +func profSymbol(_ context.Context, w http.ResponseWriter, r *http.Request) { + pprof.Symbol(w, r) +} + +func profTrace(_ context.Context, w http.ResponseWriter, r *http.Request) { + pprof.Trace(w, r) +} diff --git a/pkg/mcclient/modules/base.go b/pkg/mcclient/modules/base.go index 39e85c6f6b..70f8c998b1 100644 --- a/pkg/mcclient/modules/base.go +++ b/pkg/mcclient/modules/base.go @@ -97,6 +97,20 @@ func (this *BaseManager) rawRequest(session *mcclient.ClientSession, header, body, this.GetApiVersion()) } +func (this *BaseManager) rawBaseUrlRequest(s *mcclient.ClientSession, + method httputils.THttpMethod, path string, + header http.Header, body io.Reader) (*http.Response, error) { + baseUrlF := func(baseurl string) string { + obj, _ := url.Parse(baseurl) + obj.Path = "" + return obj.String() + } + return s.RawBaseUrlRequest( + this.serviceType, this.endpointType, + method, this.versionedURL(path), + header, body, this.GetApiVersion(), baseUrlF) +} + type ListResult struct { Data []jsonutils.JSONObject Total int diff --git a/pkg/mcclient/modules/pprof.go b/pkg/mcclient/modules/pprof.go new file mode 100644 index 0000000000..4bc7e90242 --- /dev/null +++ b/pkg/mcclient/modules/pprof.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "fmt" + "io" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +func GetPProfByType(s *mcclient.ClientSession, serviceType string, profileType string, seconds int) (io.Reader, error) { + man := &BaseManager{serviceType: serviceType} + if seconds <= 0 { + seconds = 15 + } + resp, err := man.rawBaseUrlRequest(s, "GET", fmt.Sprintf("/debug/pprof/%s?seconds=%d", profileType, seconds), nil, nil) + if err != nil { + return nil, err + } + return resp.Body, nil +} diff --git a/pkg/mcclient/modules/version.go b/pkg/mcclient/modules/version.go index 3cff617ced..b209142cb2 100644 --- a/pkg/mcclient/modules/version.go +++ b/pkg/mcclient/modules/version.go @@ -22,7 +22,7 @@ import ( func GetVersion(s *mcclient.ClientSession, serviceType string) (string, error) { man := &BaseManager{serviceType: serviceType} - resp, err := man.rawRequest(s, "GET", "/version", nil, nil) + resp, err := man.rawBaseUrlRequest(s, "GET", "/version", nil, nil) if err != nil { return "", err } diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index 3e8bd6fd00..c9ea4068de 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -182,15 +182,20 @@ func (this *ClientSession) getBaseUrl(service, endpointType, apiVersion string) } } -func (this *ClientSession) RawVersionRequest( - service, endpointType string, method httputils.THttpMethod, url string, +func (this *ClientSession) RawBaseUrlRequest( + service, endpointType string, + method httputils.THttpMethod, url string, headers http.Header, body io.Reader, apiVersion string, + baseurlFactory func(string) string, ) (*http.Response, error) { baseurl, err := this.getBaseUrl(service, endpointType, apiVersion) if err != nil { return nil, err } + if baseurlFactory != nil { + baseurl = baseurlFactory(baseurl) + } tmpHeader := http.Header{} if headers != nil { populateHeader(&tmpHeader, headers) @@ -205,6 +210,14 @@ func (this *ClientSession) RawVersionRequest( method, url, tmpHeader, body) } +func (this *ClientSession) RawVersionRequest( + service, endpointType string, method httputils.THttpMethod, url string, + headers http.Header, body io.Reader, + apiVersion string, +) (*http.Response, error) { + return this.RawBaseUrlRequest(service, endpointType, method, url, headers, body, apiVersion, nil) +} + func (this *ClientSession) RawRequest(service, endpointType string, method httputils.THttpMethod, url string, headers http.Header, body io.Reader) (*http.Response, error) { return this.RawVersionRequest(service, endpointType, method, url, headers, body, "") } diff --git a/pkg/util/netutils2/netutils.go b/pkg/util/netutils2/netutils.go index ca14105452..6078dd3e94 100644 --- a/pkg/util/netutils2/netutils.go +++ b/pkg/util/netutils2/netutils.go @@ -44,6 +44,19 @@ var PRIVATE_PREFIXES = []string{ "192.168.0.0/16", } +func GetFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + func IsTcpPortUsed(addr string, port int) bool { conn, _ := net.Dial("tcp", fmt.Sprintf("%s:%d", addr, port)) if conn != nil {