From 65220a6f5a50572798b5f3ab3c587c872698d941 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Tue, 1 Dec 2020 19:16:19 +0800 Subject: [PATCH 1/6] feat(vpcagent): export subnet mac gen funcs --- pkg/vpcagent/ovn/keeper.go | 6 +++--- pkg/vpcagent/ovn/mac/mac.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/vpcagent/ovn/keeper.go b/pkg/vpcagent/ovn/keeper.go index 3617b0be64..92912800dd 100644 --- a/pkg/vpcagent/ovn/keeper.go +++ b/pkg/vpcagent/ovn/keeper.go @@ -266,9 +266,9 @@ func (keeper *OVNNorthboundKeeper) ClaimVpc(ctx context.Context, vpc *agentmodel func (keeper *OVNNorthboundKeeper) ClaimNetwork(ctx context.Context, network *agentmodels.Network, mtu int) error { var ( - rpMac = mac.HashMac(network.Id, "rp") - dhcpMac = mac.HashMac(network.Id, "dhcp") - mdMac = mac.HashMac(network.Id, "md") + rpMac = mac.HashSubnetRouterPortMac(network.Id) + dhcpMac = mac.HashSubnetDhcpMac(network.Id) + mdMac = mac.HashSubnetMetadataMac(network.Id) mdIp = "169.254.169.254" ) netLs := &ovn_nb.LogicalSwitch{ diff --git a/pkg/vpcagent/ovn/mac/mac.go b/pkg/vpcagent/ovn/mac/mac.go index 5015932f60..573b222482 100644 --- a/pkg/vpcagent/ovn/mac/mac.go +++ b/pkg/vpcagent/ovn/mac/mac.go @@ -24,3 +24,15 @@ func HashMac(in ...string) string { func HashVpcHostDistgwMac(hostId string) string { return HashMac(hostId) } + +func HashSubnetRouterPortMac(netId string) string { + return HashMac(netId, "rp") +} + +func HashSubnetDhcpMac(netId string) string { + return HashMac(netId, "dhcp") +} + +func HashSubnetMetadataMac(netId string) string { + return HashMac(netId, "md") +} From 43215c2ad2cd38e6a86dba2d256b086b4324b4d0 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 2 Dec 2020 14:43:58 +0800 Subject: [PATCH 2/6] feat(host): metadata: make vpc aware metadata handler --- pkg/hostman/metadata/metadatahandler.go | 94 +++++++++++++++++-------- 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/pkg/hostman/metadata/metadatahandler.go b/pkg/hostman/metadata/metadatahandler.go index a8f941e488..91fdcbaaaf 100644 --- a/pkg/hostman/metadata/metadatahandler.go +++ b/pkg/hostman/metadata/metadatahandler.go @@ -23,7 +23,9 @@ import ( "strconv" "strings" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/netutils" "yunion.io/x/onecloud/pkg/appsrv" @@ -32,32 +34,79 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" ) -func addMetadataHandler(prefix string, app *appsrv.Application) { +func StartService(app *appsrv.Application, address string, port int) { + s := &Service{ + Address: address, + Port: port, + + DescGetter: classicDescGetter{}, + } + Start(app, s) +} + +func Start(app *appsrv.Application, s *Service) { + s.addHandler(app) + addr := net.JoinHostPort(s.Address, strconv.Itoa(s.Port)) + log.Infof("Start metadata service on http://%s", addr) + app.ListenAndServeWithoutCleanup(addr, "", "") +} + +type DescGetter interface { + Get(ip string) (guestDesc, guestNic jsonutils.JSONObject) +} + +type classicDescGetter struct{} + +func (g classicDescGetter) Get(ip string) (guestDesc, guestNic jsonutils.JSONObject) { + guestDesc, guestNic = guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) + return +} + +type DescGetterFunc func(ip string) (guestDesc jsonutils.JSONObject) + +func (f DescGetterFunc) Get(ip string) (guestDesc jsonutils.JSONObject) { + return f(ip) +} + +type Service struct { + Address string + Port int + + DescGetter DescGetter +} + +func (s *Service) getGuestNicDesc(r *http.Request) (guestDesc, guestNic jsonutils.JSONObject) { + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + panic(errors.Wrapf(err, "SplitHostPort %s", r.RemoteAddr)) + } + guestDesc, guestNic = s.DescGetter.Get(ip) + return +} + +func (s *Service) addHandler(app *appsrv.Application) { + prefix := "" + for _, method := range []string{"GET", "HEAD"} { app.AddHandler(method, fmt.Sprintf("%s/", - prefix, `(latest|\d{4}-\d{2}-\d{2})`), versionOnly) + prefix, `(latest|\d{4}-\d{2}-\d{2})`), s.versionOnly) } for _, method := range []string{"GET", "HEAD"} { app.AddHandler(method, fmt.Sprintf("%s//user-data", - prefix, `(latest|\d{4}-\d{2}-\d{2})`), userData) + prefix, `(latest|\d{4}-\d{2}-\d{2})`), s.userData) app.AddHandler(method, fmt.Sprintf("%s//meta-data", - prefix, `(latest|\d{4}-\d{2}-\d{2})`), metaData) + prefix, `(latest|\d{4}-\d{2}-\d{2})`), s.metaData) } } -func versionOnly(ctx context.Context, w http.ResponseWriter, r *http.Request) { +func (s *Service) versionOnly(ctx context.Context, w http.ResponseWriter, r *http.Request) { hostutils.Response(ctx, w, strings.Join([]string{"meta-data", "user-data"}, "\n")) } -func userData(ctx context.Context, w http.ResponseWriter, r *http.Request) { - ip, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - hostutils.Response(ctx, w, httperrors.NewBadRequestError("Parse Remoteaddr %s error %s", r.RemoteAddr, err.Error())) - return - } - guestDesc, gusetNic := guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) - if guestDesc == nil || gusetNic == nil { +func (s *Service) userData(ctx context.Context, w http.ResponseWriter, r *http.Request) { + guestDesc, guestNic := s.getGuestNicDesc(r) + if guestDesc == nil || guestNic == nil { hostutils.Response(ctx, w, "") return } @@ -78,15 +127,9 @@ func userData(ctx context.Context, w http.ResponseWriter, r *http.Request) { hostutils.Response(ctx, w, string(userDataDecoded)) } -func metaData(ctx context.Context, w http.ResponseWriter, r *http.Request) { - ip, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - hostutils.Response(ctx, w, httperrors.NewBadRequestError("Parse Remoteaddr %s error %s", r.RemoteAddr, err.Error())) - return - } - - guestDesc, gusetNic := guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) - if guestDesc == nil || gusetNic == nil { +func (s *Service) metaData(ctx context.Context, w http.ResponseWriter, r *http.Request) { + guestDesc, guestNic := s.getGuestNicDesc(r) + if guestDesc == nil || guestNic == nil { hostutils.Response(ctx, w, "") return } @@ -278,10 +321,3 @@ func metaData(ctx context.Context, w http.ResponseWriter, r *http.Request) { } hostutils.Response(ctx, w, httperrors.NewNotFoundError("Resource not handled")) } - -func StartService(app *appsrv.Application, address string, port int) { - addMetadataHandler("", app) - addr := net.JoinHostPort(address, strconv.Itoa(port)) - log.Infof("Host Metadata Start listen on %s://%s", "http", addr) - app.ListenAndServeWithoutCleanup(addr, "", "") -} From de2f1ec3b4136ce3b2b067dfa23012ac29c4e2ec Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 2 Dec 2020 15:21:23 +0800 Subject: [PATCH 3/6] refactor(host): metadata: eliminate ref of guestNic --- pkg/hostman/metadata/metadatahandler.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/hostman/metadata/metadatahandler.go b/pkg/hostman/metadata/metadatahandler.go index 91fdcbaaaf..2561d14eb3 100644 --- a/pkg/hostman/metadata/metadatahandler.go +++ b/pkg/hostman/metadata/metadatahandler.go @@ -52,13 +52,13 @@ func Start(app *appsrv.Application, s *Service) { } type DescGetter interface { - Get(ip string) (guestDesc, guestNic jsonutils.JSONObject) + Get(ip string) (guestDesc jsonutils.JSONObject) } type classicDescGetter struct{} -func (g classicDescGetter) Get(ip string) (guestDesc, guestNic jsonutils.JSONObject) { - guestDesc, guestNic = guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) +func (g classicDescGetter) Get(ip string) (guestDesc jsonutils.JSONObject) { + guestDesc, _ = guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) return } @@ -75,12 +75,12 @@ type Service struct { DescGetter DescGetter } -func (s *Service) getGuestNicDesc(r *http.Request) (guestDesc, guestNic jsonutils.JSONObject) { +func (s *Service) getGuestNicDesc(r *http.Request) (guestDesc jsonutils.JSONObject) { ip, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { panic(errors.Wrapf(err, "SplitHostPort %s", r.RemoteAddr)) } - guestDesc, guestNic = s.DescGetter.Get(ip) + guestDesc = s.DescGetter.Get(ip) return } @@ -105,8 +105,8 @@ func (s *Service) versionOnly(ctx context.Context, w http.ResponseWriter, r *htt } func (s *Service) userData(ctx context.Context, w http.ResponseWriter, r *http.Request) { - guestDesc, guestNic := s.getGuestNicDesc(r) - if guestDesc == nil || guestNic == nil { + guestDesc := s.getGuestNicDesc(r) + if guestDesc == nil { hostutils.Response(ctx, w, "") return } @@ -128,8 +128,8 @@ func (s *Service) userData(ctx context.Context, w http.ResponseWriter, r *http.R } func (s *Service) metaData(ctx context.Context, w http.ResponseWriter, r *http.Request) { - guestDesc, guestNic := s.getGuestNicDesc(r) - if guestDesc == nil || guestNic == nil { + guestDesc := s.getGuestNicDesc(r) + if guestDesc == nil { hostutils.Response(ctx, w, "") return } From 38b6e011ef0bc2a6924cb3d361be711b5eb9c3c4 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 2 Dec 2020 16:22:55 +0800 Subject: [PATCH 4/6] refactor(hostman): use metadata.DescGetterFunc --- pkg/hostman/host_services.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index e429a80650..9e9d491a88 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -19,6 +19,7 @@ import ( "path/filepath" execlient "yunion.io/x/executor/client" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -109,9 +110,17 @@ func (host *SHostService) RunService() { host.initHandlers(app) // Init Metadata handler - go metadata.StartService( + go metadata.Start( app_common.InitApp(&options.HostOptions.BaseOptions, false), - options.HostOptions.Address, options.HostOptions.Port+1000) + &metadata.Service{ + Address: options.HostOptions.Address, + Port: options.HostOptions.Port + 1000, + DescGetter: metadata.DescGetterFunc(func(ip string) jsonutils.JSONObject { + guestDesc, _ := guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) + return guestDesc + }), + }, + ) cronManager.AddJobEveryFewDays( "CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false) From 6dce7d8ebc7b35ff58b378e868450fca609cebc6 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 2 Dec 2020 16:23:18 +0800 Subject: [PATCH 5/6] refactor(hostman): metadata: eliminate import of guestman --- pkg/hostman/metadata/metadatahandler.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pkg/hostman/metadata/metadatahandler.go b/pkg/hostman/metadata/metadatahandler.go index 2561d14eb3..98ff87bfe0 100644 --- a/pkg/hostman/metadata/metadatahandler.go +++ b/pkg/hostman/metadata/metadatahandler.go @@ -14,6 +14,7 @@ package metadata +// NOTE keep imports minimal. DO NOT IMPORT guestman import ( "context" "encoding/base64" @@ -29,21 +30,10 @@ import ( "yunion.io/x/pkg/util/netutils" "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/hostman/guestman" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/httperrors" ) -func StartService(app *appsrv.Application, address string, port int) { - s := &Service{ - Address: address, - Port: port, - - DescGetter: classicDescGetter{}, - } - Start(app, s) -} - func Start(app *appsrv.Application, s *Service) { s.addHandler(app) addr := net.JoinHostPort(s.Address, strconv.Itoa(s.Port)) @@ -55,13 +45,6 @@ type DescGetter interface { Get(ip string) (guestDesc jsonutils.JSONObject) } -type classicDescGetter struct{} - -func (g classicDescGetter) Get(ip string) (guestDesc jsonutils.JSONObject) { - guestDesc, _ = guestman.GetGuestManager().GetGuestNicDesc("", ip, "", "", false) - return -} - type DescGetterFunc func(ip string) (guestDesc jsonutils.JSONObject) func (f DescGetterFunc) Get(ip string) (guestDesc jsonutils.JSONObject) { From 771824d30799d861fb51b9522b19d44980da6319 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 2 Dec 2020 17:22:14 +0800 Subject: [PATCH 6/6] feat(appsrv): Application: add Stop method for shutdown behaviour --- pkg/appsrv/appsrv.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index 525c2293a0..81092eb4fe 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -64,6 +64,7 @@ type Application struct { isExiting bool idleConnsClosed chan struct{} + httpServer *http.Server } const ( @@ -467,16 +468,23 @@ func (app *Application) ListenAndServeTLSWithCleanup2(addr string, certFile, key app.addDefaultHandlers() AddPProfHandler(app) } - s := app.initServer(addr) + app.httpServer = app.initServer(addr) if isMaster { - app.registerCleanShutdown(s, onStop) + app.registerCleanShutdown(app.httpServer, onStop) } - app.listenAndServeInternal(s, certFile, keyFile) + app.listenAndServeInternal(app.httpServer, certFile, keyFile) if isMaster { app.waitCleanShutdown() } } +func (app *Application) Stop(ctx context.Context) error { + if app.httpServer != nil { + return app.httpServer.Shutdown(ctx) + } + return nil +} + func (app *Application) listenAndServeInternal(s *http.Server, certFile, keyFile string) { var err error if len(certFile) == 0 && len(keyFile) == 0 {