feature: support apigatewa type of endpoint

pkg/apigateway/app/app.go

@@ -15,7 +15,10 @@
 package app
 
 import (
+	"yunion.io/x/log"
+
 	"yunion.io/x/onecloud/pkg/apigateway/handler"
+	"yunion.io/x/onecloud/pkg/apigateway/options"
 	"yunion.io/x/onecloud/pkg/apis/cloudid"
 	"yunion.io/x/onecloud/pkg/appsrv"
 )
@@ -29,8 +32,10 @@ type Application struct {
 	ResourceHandler     handler.IHandler
 	CSRFResourceHandler handler.IHandler
 	RPCHandler          handler.IHandler
-	// InfluxdbProxyHandler handler.IHandler
+
 	CloudIdSAMLHandler handler.IHandler
+
+	BackendServiceProxyHandler handler.IHandler
 }
 
 func NewApp(app *appsrv.Application) *Application {
@@ -66,7 +71,11 @@ func (app *Application) InitHandlers() *Application {
 		AddGet(handler.FetchAuthToken).
 		AddPost(handler.FetchAuthToken)
 
-	// app.InfluxdbProxyHandler = handler.NewInfluxdbProxyHandler("/query")
+	if options.Options.EnableBackendServiceProxy {
+		// bind backend service API proxy
+		log.Infof("enable backend service proxy")
+		app.BackendServiceProxyHandler = handler.NewBackendServiceProxyHandler("/api/s/<service>")
+	}
 
 	app.CloudIdSAMLHandler = handler.NewProxyHandlerWithService(cloudid.SAML_IDP_PREFIX, cloudid.SERVICE_TYPE)
 
@@ -75,8 +84,8 @@ func (app *Application) InitHandlers() *Application {
 
 func (app *Application) Bind() {
 	for _, h := range []handler.IHandler{
+		app.BackendServiceProxyHandler,
 		app.CloudIdSAMLHandler,
-		// app.InfluxdbProxyHandler,
 		app.MiscHandler,
 		app.AuthHandler,
 		app.K8sHandler,
@@ -84,6 +93,8 @@ func (app *Application) Bind() {
 		app.ResourceHandler,
 		app.CSRFResourceHandler,
 	} {
-		h.Bind(app.Application)
+		if h != nil {
+			h.Bind(app.Application)
+		}
 	}
 }

pkg/apigateway/handler/backendproxy.go

@@ -0,0 +1,79 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/proxy"
+)
+
+type SBackendServiceProxyHandler struct {
+	prefix string
+}
+
+func NewBackendServiceProxyHandler(prefix string) *SBackendServiceProxyHandler {
+	return &SBackendServiceProxyHandler{
+		prefix: prefix,
+	}
+}
+
+func (h *SBackendServiceProxyHandler) requestManipulator(ctx context.Context, r *http.Request) (*http.Request, error) {
+	// remove leading prefixes /api/s/<service>
+	path := r.URL.Path[len("/api/s/"):]
+	slashPos := strings.Index(path, "/")
+	if slashPos <= 0 {
+		return r, httperrors.NewBadRequestError("invalid request URL %s", r.URL.Path)
+	}
+	path = path[slashPos:]
+	log.Debugf("Path: %s => %s", r.URL.Path, path)
+	r.URL = &url.URL{
+		Path:     path,
+		RawQuery: r.URL.RawQuery,
+		Fragment: r.URL.Fragment,
+	}
+	return r, nil
+}
+
+func (h *SBackendServiceProxyHandler) Bind(app *appsrv.Application) {
+	app.AddReverseProxyHandler(h.prefix, h.fetchReverseEndpoint(), h.requestManipulator)
+}
+
+func (h *SBackendServiceProxyHandler) fetchReverseEndpoint() *proxy.SEndpointFactory {
+	f := func(ctx context.Context, r *http.Request) (string, error) {
+		params := appctx.AppContextParams(ctx)
+		serviceName := params["<service>"]
+		if len(serviceName) == 0 {
+			return "", httperrors.NewBadRequestError("no service")
+		}
+		endpointType := "internalURL"
+		session := auth.GetAdminSession(ctx, FetchRegion(r), "")
+		ep, err := session.GetServiceURL(serviceName, endpointType)
+		if err != nil {
+			return "", httperrors.NewBadRequestError("invalid service %s: %s", serviceName, err)
+		}
+		return getEndpointSchemeHost(ep)
+	}
+	return proxy.NewEndpointFactory(f, "backendService")
+}

pkg/apigateway/handler/misc.go

@@ -98,9 +98,9 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	imageDownloadByUrl := uploadHandlerInfo(GET, prefix+"imageutils/image/<image_name>", imageDownloadByUrlHandler)
 	app.AddHandler3(imageDownloadByUrl)
 	// fetch vm image download url or download large file directly with query parameter <direct=true>
-	imageDownloader := uploadHandlerInfo("GET", prefix+"/imageutils/download/<image_id>", FetchAuthToken(imageDownloadHandler))
+	imageDownloader := uploadHandlerInfo("GET", prefix+"imageutils/download/<image_id>", FetchAuthToken(imageDownloadHandler))
 	app.AddHandler3(imageDownloader)
-	imageUploader := uploadHandlerInfo("POST", prefix+"/imageutils/upload", FetchAuthToken(imageUploadHandler))
+	imageUploader := uploadHandlerInfo("POST", prefix+"imageutils/upload", FetchAuthToken(imageUploadHandler))
 	app.AddHandler3(imageUploader)
 	s3upload := uploadHandlerInfo(POST, prefix+"s3uploads", FetchAuthToken(h.postS3UploadHandler))
 	app.AddHandler3(s3upload)

pkg/apigateway/options/options.go

@@ -43,6 +43,9 @@ type GatewayOptions struct {
 	// 上报非敏感基础信息，帮助软件更加完善
 	DisableReporting bool `default:"false" help:"Reporting data every 24 hours, report data incloud version, os, platform and usages"`
 
+	// 启用后端服务反向代理网关
+	EnableBackendServiceProxy bool `default:"false" help:"Proxy API request to backend services"`
+
 	common_options.CommonOptions `"request_worker_count->default":"32"`
 }
 
@@ -59,5 +62,9 @@ func OnOptionsChange(oldO, newO interface{}) bool {
 		changed = true
 	}
 
+	if oldOpts.EnableBackendServiceProxy != newOpts.EnableBackendServiceProxy {
+		changed = true
+	}
+
 	return changed
 }

pkg/apis/identity/consts.go

@@ -54,6 +54,8 @@ const (
 	EndpointInterfaceAdmin    = "admin"
 	EndpointInterfaceConsole  = "console"
 
+	EndpointInterfaceApigateway = "apigateway"
+
 	KeystoneDomainRoot = "<<keystone.domain.root>>"
 
 	IdMappingEntityUser   = "user"

pkg/mcclient/session.go

@@ -27,6 +27,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/utils"
 
@@ -143,25 +144,11 @@ func (this *ClientSession) GetServiceCatalog() IServiceCatalog {
 }
 
 func (this *ClientSession) GetServiceVersionURL(service, endpointType, apiVersion string) (string, error) {
-	if len(this.endpointType) > 0 {
-		// session specific endpoint type should override the input endpointType, which is supplied by manager
-		endpointType = this.endpointType
+	urls, err := this.GetServiceVersionURLs(service, endpointType, apiVersion)
+	if err != nil {
+		return "", errors.Wrap(err, "GetServiceVersionURLs")
 	}
-	service = this.getServiceName(service, apiVersion)
-	catalog := this.GetServiceCatalog()
-	if gotypes.IsNil(catalog) {
-		return this.client.authUrl, nil
-	}
-	url, err := catalog.GetServiceURL(service, this.region, this.zone, endpointType)
-	if err != nil && service == api.SERVICE_TYPE {
-		return this.client.authUrl, nil
-	}
-	// HACK! in case schema of keystone changed, always trust authUrl
-	if service == api.SERVICE_TYPE && this.client.authUrl[:5] != url[:5] {
-		log.Warningf("Schema of keystone authUrl and endpoint mismatch: %s!=%s", this.client.authUrl, url)
-		return this.client.authUrl, nil
-	}
-	return url, err
+	return urls[rand.Intn(len(urls))], nil
 }
 
 func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]string, error) {
@@ -174,10 +161,66 @@ func (this *ClientSession) GetServiceVersionURLs(service, endpointType, apiVersi
 		endpointType = this.endpointType
 	}
 	service = this.getServiceName(service, apiVersion)
-	urls, err := this.GetServiceCatalog().GetServiceURLs(service, this.region, this.zone, endpointType)
-	if err != nil && service == api.SERVICE_TYPE {
+	if endpointType == api.EndpointInterfaceApigateway {
+		return this.getApigatewayServiceURLs(service, endpointType)
+	} else {
+		return this.getServiceVersionURLs(service, endpointType)
+	}
+}
+
+func (this *ClientSession) getApigatewayServiceURLs(service, endpointType string) ([]string, error) {
+	urls, err := this.getServiceVersionURLs(service, "")
+	if err != nil {
+		return nil, errors.Wrap(err, "getServiceVersionURLs")
+	}
+	// replace URLs with authUrl prefix
+	// find the common prefix
+	prefix := this.client.authUrl
+	lastSlashPos := strings.LastIndex(prefix, "/api/s/identity")
+	if lastSlashPos <= 0 {
+		return nil, errors.Wrapf(err, "invalue auth_url %s", prefix)
+	}
+	prefix = httputils.JoinPath(prefix[:lastSlashPos], "api/s", service)
+	rets := make([]string, len(urls))
+	for i, url := range urls {
+		if len(url) < 9 {
+			// len("https://") == 8
+			log.Errorf("invalid url %s: shorter than 9 bytes", url)
+			continue
+		}
+		slashPos := strings.IndexByte(url[9:], '/')
+		if slashPos > 0 {
+			url = url[9+slashPos:]
+			rets[i] = httputils.JoinPath(prefix, url)
+		} else {
+			rets[i] = prefix
+		}
+	}
+	return rets, nil
+}
+
+func (this *ClientSession) getServiceVersionURLs(service, endpointType string) ([]string, error) {
+	catalog := this.GetServiceCatalog()
+	if gotypes.IsNil(catalog) {
 		return []string{this.client.authUrl}, nil
 	}
+	urls, err := catalog.GetServiceURLs(service, this.region, this.zone, endpointType)
+	// HACK! in case of fail to get kestone url or schema of keystone changed, always trust authUrl
+	if service == api.SERVICE_TYPE && (err != nil || len(urls) == 0 || (len(this.client.authUrl) != 0 && this.client.authUrl[:5] != urls[0][:5])) {
+		var msg string
+		if err != nil {
+			msg = fmt.Sprintf("fail to retrieve keystone urls: %s", err)
+		} else if len(urls) == 0 {
+			msg = fmt.Sprintf("empty keystone url")
+		} else {
+			msg = fmt.Sprintf("Schema of keystone authUrl and endpoint mismatch: %s!=%s", this.client.authUrl, urls)
+		}
+		log.Warningln(msg)
+		return []string{this.client.authUrl}, nil
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "catalog.GetServiceURLs")
+	}
 	return urls, err
 }