cloudproxy: initial version

pkg/cloudproxy/models/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models // import "yunion.io/x/onecloud/pkg/cloudproxy/models"

pkg/cloudproxy/models/forwards.go

@@ -0,0 +1,470 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/sqlchemy"
+
+	cloudproxy_api "yunion.io/x/onecloud/pkg/apis/cloudproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+type SForward struct {
+	db.SVirtualResourceBase
+
+	ProxyEndpointId string `width:"36" charset:"ascii" nullable:"false" list:"user" update:"user" create:"required"`
+	ProxyAgentId    string `width:"36" charset:"ascii" nullable:"true" list:"user" update:"user" create:"optional"`
+
+	Type        string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	RemoteAddr  string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	RemotePort  int    `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	BindPortReq int    `width:"16" charset:"ascii" nullable:"false" list:"user" update:"user" create:"optional"`
+
+	Opaque string `width:"36" charset:"ascii" nullable:"true" list:"user" update:"user" create:"optional"`
+
+	BindPort        int       `width:"16" charset:"ascii" nullable:"false" list:"user" update:"user" create:"optional"`
+	LastSeen        time.Time `nullable:"true" get:"user" list:"user"`
+	LastSeenTimeout int       `width:"16" charset:"ascii" nullable:"false" list:"user" update:"user" create:"optional" default:"117"`
+}
+
+type SForwardManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var ForwardManager *SForwardManager
+
+func init() {
+	ForwardManager = &SForwardManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SForward{},
+			"forwards_tbl",
+			"forward",
+			"forwards",
+		),
+	}
+	ForwardManager.SetVirtualObject(ForwardManager)
+}
+
+func (man *SForwardManager) validateLocalSetPort(ctx context.Context, data *jsonutils.JSONDict, agentId string, portReq int) (*jsonutils.JSONDict, error) {
+	var (
+		fwds []SForward
+		q    = man.Query().
+			Equals("proxy_agent_id", agentId).
+			Equals("bind_port_req", portReq)
+	)
+	if err := db.FetchModelObjects(man, q, &fwds); err != nil {
+		return nil, httperrors.NewServerError("query forwards by agent failed: %v", err)
+	}
+	if len(fwds) > 0 {
+		return nil, httperrors.NewConflictError("port %d on agent %s was already occupied",
+			portReq, agentId)
+	}
+	data.Set("bind_port", jsonutils.NewInt(int64(portReq)))
+	return data, nil
+}
+
+func (man *SForwardManager) validateRemoteSetPort(ctx context.Context, data *jsonutils.JSONDict, epId string, portReq int) (*jsonutils.JSONDict, error) {
+	var (
+		fwds []SForward
+		q    = man.Query().
+			Equals("proxy_endpoint_id", epId).
+			Equals("bind_port_req", portReq)
+	)
+	if err := db.FetchModelObjects(man, q, &fwds); err != nil {
+		return nil, httperrors.NewServerError("query forwards by endpoint id failed: %v", err)
+	}
+	if len(fwds) > 0 {
+		return nil, httperrors.NewConflictError("port %d on proxy endpoint %s was already occupied",
+			portReq, epId)
+	}
+	data.Set("bind_port", jsonutils.NewInt(int64(portReq)))
+	return data, nil
+}
+
+func (man *SForwardManager) validateLocalSelectAgent(ctx context.Context, data *jsonutils.JSONDict, portReq int) (*jsonutils.JSONDict, error) {
+	agents, err := ProxyAgentManager.allAgents(ctx)
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+	agentsNum := len(agents)
+	if agentsNum == 0 {
+		return nil, httperrors.NewResourceNotFoundError("empty proxy agents set")
+	}
+	s := rand.Intn(agentsNum)
+	for i := s; ; {
+		agent := &agents[i]
+
+		var err error
+		data, err = man.validateLocalSetPort(ctx, data, agent.Id, portReq)
+		if err == nil {
+			data.Set("proxy_agent_id", jsonutils.NewString(agent.Id))
+			return data, nil
+		}
+
+		i += 1
+		if i == agentsNum {
+			i = 0
+		}
+		if i == s {
+			break
+		}
+	}
+	return nil, httperrors.NewResourceNotFoundError("no proxy agent accepts request for port %d", portReq)
+}
+
+func (man *SForwardManager) validateRemoteSelectAgent(ctx context.Context, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	agents, err := ProxyAgentManager.allAgents(ctx)
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+	agentsNum := len(agents)
+	if agentsNum == 0 {
+		return nil, httperrors.NewResourceNotFoundError("empty proxy agents set")
+	}
+	i := rand.Intn(agentsNum)
+	agent := agents[i]
+	data.Set("proxy_agent_id", jsonutils.NewString(agent.Id))
+	return data, nil
+}
+
+func (man *SForwardManager) validatePortReq(
+	ctx context.Context,
+	typ string, portReq int, agentId, epId string,
+	data *jsonutils.JSONDict,
+) (*jsonutils.JSONDict, error) {
+	validateOne := func(portReq int) (*jsonutils.JSONDict, error) {
+		var err error
+		switch typ {
+		case cloudproxy_api.FORWARD_TYPE_LOCAL:
+			if agentId == "" {
+				data, err = man.validateLocalSelectAgent(ctx, data, portReq)
+			} else {
+				data, err = man.validateLocalSetPort(ctx, data, agentId, portReq)
+			}
+		case cloudproxy_api.FORWARD_TYPE_REMOTE:
+			data, err = man.validateRemoteSetPort(ctx, data, epId, portReq)
+		}
+		return data, err
+	}
+
+	if typ == cloudproxy_api.FORWARD_TYPE_REMOTE && agentId == "" {
+		var err error
+		data, err = man.validateRemoteSelectAgent(ctx, data)
+		if err != nil {
+			return nil, httperrors.NewResourceNotFoundError("select proxy agent: %v", err)
+		}
+	}
+
+	var err error
+	if portReq <= 0 {
+		portTotal := cloudproxy_api.BindPortMax - cloudproxy_api.BindPortMin + 1
+		portReqStart := rand.Intn(portTotal)
+		for portInc := portReqStart; ; {
+			data, err = validateOne(cloudproxy_api.BindPortMin + portInc)
+			if err == nil {
+				break
+			}
+			portInc += 1
+			if portInc == portTotal {
+				portInc = 0
+			}
+			if portInc == portReqStart {
+				return nil, httperrors.NewOutOfResourceError("no available port for bind")
+			}
+		}
+	} else {
+		data, err = validateOne(portReq)
+	}
+	return data, err
+}
+
+func (man *SForwardManager) AllowPerformCreateFromServer(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAllowClassPerform(rbacutils.ScopeProject, userCred, man, "create-from-server")
+}
+
+func (man *SForwardManager) PerformCreateFromServer(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *cloudproxy_api.ForwardCreateFromServerInput) (jsonutils.JSONObject, error) {
+	data := jsonutils.Marshal(input).(*jsonutils.JSONDict)
+
+	typeV := validators.NewStringChoicesValidator("type", cloudproxy_api.FORWARD_TYPES)
+	portReqV := validators.NewRangeValidator("bind_port_req", cloudproxy_api.BindPortMin, cloudproxy_api.BindPortMax)
+	remotePortV := validators.NewPortValidator("remote_port")
+	{
+		for _, v := range []validators.IValidator{
+			typeV,
+			portReqV.Optional(true),
+			remotePortV,
+
+			validators.NewNonNegativeValidator("last_seen_timeout").Optional(true),
+		} {
+			if err := v.Validate(data); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	serverId := input.ServerId
+	if serverId == "" {
+		return nil, httperrors.NewBadRequestError("server_id is required")
+	}
+	serverInfo, err := getServerInfo(ctx, userCred, serverId)
+	if err != nil {
+		return nil, err
+	}
+	nic := serverInfo.GetNic()
+	if nic == nil {
+		return nil, httperrors.NewBadRequestError("cannot find network interface for this server")
+	}
+	proxymatch := ProxyMatchManager.findMatch(ctx, nic.NetworkId, nic.VpcId)
+	if proxymatch == nil {
+		return nil, httperrors.NewBadRequestError("cannot find an endpoint for this server")
+	}
+	data.Set("opaque", jsonutils.NewString(serverInfo.Server.Id))
+	data.Set("remote_addr", jsonutils.NewString(nic.IpAddr))
+	data.Set("proxy_endpoint_id", jsonutils.NewString(proxymatch.ProxyEndpointId))
+
+	typ := typeV.Value
+	agentId := ""
+	epId := proxymatch.ProxyEndpointId
+	if data.Contains("bind_port_req") {
+		portReq := int(portReqV.Value)
+		data, err = man.validatePortReq(ctx, typ, portReq, agentId, epId, data)
+	} else {
+		data, err = man.validatePortReq(ctx, typ, -1, agentId, epId, data)
+	}
+
+	forward := &SForward{}
+	if err := data.Unmarshal(forward); err != nil {
+		return nil, httperrors.NewServerError("unmarshal create params: %v", err)
+	}
+	forward.Name = fmt.Sprintf("%s-%s-%d", serverInfo.Server.Name, typ, forward.RemotePort)
+	forward.DomainId = userCred.GetProjectDomainId()
+	forward.ProjectId = userCred.GetProjectId()
+	if err := man.TableSpec().Insert(ctx, forward); err != nil {
+		return nil, httperrors.NewServerError("database insertion error: %v", err)
+	}
+
+	return jsonutils.Marshal(forward), nil
+}
+
+func (man *SForwardManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	endpointV := validators.NewModelIdOrNameValidator("proxy_endpoint", ProxyEndpointManager.Keyword(), ownerId)
+	agentV := validators.NewModelIdOrNameValidator("proxy_agent", ProxyAgentManager.Keyword(), ownerId)
+	typeV := validators.NewStringChoicesValidator("type", cloudproxy_api.FORWARD_TYPES)
+	portReqV := validators.NewRangeValidator("bind_port_req", cloudproxy_api.BindPortMin, cloudproxy_api.BindPortMax)
+	for _, v := range []validators.IValidator{
+		endpointV,
+		agentV.Optional(true),
+
+		typeV,
+		validators.NewIPv4AddrValidator("remote_addr"),
+		validators.NewPortValidator("remote_port"),
+		portReqV.Optional(true),
+
+		validators.NewNonNegativeValidator("last_seen_timeout").Optional(true),
+	} {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+
+	typ := typeV.Value
+	epId := endpointV.Model.GetId()
+	var agentId string
+	if agentV.Model != nil {
+		agentId = agentV.Model.GetId()
+	}
+
+	var err error
+	if data.Contains("bind_port_req") {
+		portReq := int(portReqV.Value)
+		data, err = man.validatePortReq(ctx, typ, portReq, agentId, epId, data)
+	} else {
+		data, err = man.validatePortReq(ctx, typ, -1, agentId, epId, data)
+	}
+	return data, err
+}
+
+func (fwd *SForward) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	endpointV := validators.NewModelIdOrNameValidator("proxy_endpoint", ProxyEndpointManager.Keyword(), userCred)
+	agentV := validators.NewModelIdOrNameValidator("proxy_agent", ProxyAgentManager.Keyword(), userCred)
+	portReqV := validators.NewRangeValidator("bind_port_req", cloudproxy_api.BindPortMin, cloudproxy_api.BindPortMax)
+	for _, v := range []validators.IValidator{
+		endpointV,
+		agentV.Optional(true),
+
+		validators.NewIPv4AddrValidator("remote_addr"),
+		validators.NewPortValidator("remote_port"),
+		portReqV,
+
+		validators.NewNonNegativeValidator("last_seen_timeout"),
+	} {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+
+	portReq := int(portReqV.Value)
+	if portReq != fwd.BindPortReq {
+		var err error
+		var agentId string
+		if agentV.Model == nil {
+			agentId = fwd.ProxyAgentId
+		} else {
+			agentId = agentV.Model.GetId()
+		}
+		switch typ := fwd.Type; typ {
+		case cloudproxy_api.FORWARD_TYPE_LOCAL:
+			data, err = ForwardManager.validateLocalSetPort(ctx, data, agentId, portReq)
+		case cloudproxy_api.FORWARD_TYPE_REMOTE:
+			data, err = ForwardManager.validateRemoteSetPort(ctx, data, agentId, portReq)
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func (man *SForwardManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	input cloudproxy_api.ForwardListInput,
+) (*sqlchemy.SQuery, error) {
+	filters := [][2]string{
+		[2]string{"type", input.Type},
+		[2]string{"proxy_endpoint_id", input.ProxyEndpointId},
+		[2]string{"proxy_agent_id", input.ProxyAgentId},
+		[2]string{"opaque", input.Opaque},
+	}
+	for _, filter := range filters {
+		if v := filter[1]; v != "" {
+			q = q.Equals(filter[0], v)
+		}
+	}
+	return q, nil
+}
+
+func (man *SForwardManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []*jsonutils.JSONDict {
+	fwds := gotypes.ConvertSliceElemType(objs, (**SForward)(nil)).([]*SForward)
+
+	paMap := map[string]*SProxyAgent{}
+	peMap := map[string]*SProxyEndpoint{}
+	{
+		var paIds []string
+		var peIds []string
+		{
+			paIdMap := map[string]string{}
+			peIdMap := map[string]string{}
+			for _, fwd := range fwds {
+				paIdMap[fwd.ProxyAgentId] = ""
+				peIdMap[fwd.ProxyEndpointId] = ""
+			}
+			for id := range paIdMap {
+				if id != "" {
+					paIds = append(paIds, id)
+				}
+			}
+			for id := range peIdMap {
+				if id != "" {
+					peIds = append(peIds, id)
+				}
+			}
+		}
+
+		var pas []SProxyAgent
+		var pes []SProxyEndpoint
+		{
+			paQ := ProxyAgentManager.Query().In("id", paIds)
+			if err := db.FetchModelObjects(ProxyAgentManager, paQ, &pas); err != nil {
+				return nil
+			}
+			peQ := ProxyEndpointManager.Query().In("id", peIds)
+			if err := db.FetchModelObjects(ProxyEndpointManager, peQ, &pes); err != nil {
+				return nil
+			}
+		}
+
+		for i := range pas {
+			pa := &pas[i]
+			paMap[pa.Id] = pa
+		}
+		for i := range pes {
+			pe := &pes[i]
+			peMap[pe.Id] = pe
+		}
+	}
+
+	r := make([]*jsonutils.JSONDict, len(objs))
+	for i, fwd := range fwds {
+		d := jsonutils.NewDict()
+		pa, paOK := paMap[fwd.ProxyAgentId]
+		pe, peOK := peMap[fwd.ProxyEndpointId]
+		if paOK || peOK {
+			if paOK {
+				d.Set("proxy_agent", jsonutils.NewString(pa.Name))
+			}
+			if peOK {
+				d.Set("proxy_endpoint", jsonutils.NewString(pe.Name))
+			}
+			switch fwd.Type {
+			case cloudproxy_api.FORWARD_TYPE_LOCAL:
+				if paOK {
+					d.Set("bind_addr", jsonutils.NewString(pa.AdvertiseAddr))
+				}
+			case cloudproxy_api.FORWARD_TYPE_REMOTE:
+				if peOK {
+					d.Set("bind_addr", jsonutils.NewString(pe.IntranetIpAddr))
+				}
+			}
+			r[i] = d
+		}
+	}
+	return r
+}
+
+func (man *SForwardManager) AllowPerformHeartbeat(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAllowClassPerform(rbacutils.ScopeProject, userCred, man, "heartbeat")
+}
+
+func (fwd *SForward) PerformHeartbeat(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *cloudproxy_api.ForwardHeartbeatInput) (jsonutils.JSONObject, error) {
+	if _, err := db.Update(fwd, func() error {
+		fwd.LastSeen = time.Now()
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return nil, nil
+}

pkg/cloudproxy/models/initdb.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+func InitDB() error {
+	return nil
+}

pkg/cloudproxy/models/proxy_agents.go

@@ -0,0 +1,104 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+// bind_addr, default 0.0.0.0
+// advertise_addr, default default route adddr, maybe k8s cluster ip
+type SProxyAgent struct {
+	db.SStandaloneResourceBase
+
+	BindAddr      string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"admin"`
+	AdvertiseAddr string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"admin"`
+}
+
+type SProxyAgentManager struct {
+	db.SStandaloneResourceBaseManager
+}
+
+var ProxyAgentManager *SProxyAgentManager
+
+func init() {
+	ProxyAgentManager = &SProxyAgentManager{
+		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+			SProxyAgent{},
+			"proxy_agents_tbl",
+			"proxy_agent",
+			"proxy_agents",
+		),
+	}
+	ProxyAgentManager.SetVirtualObject(ProxyAgentManager)
+}
+
+func (man *SProxyAgentManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	vs := []validators.IValidator{
+		validators.NewIPv4AddrValidator("bind_addr").Optional(true),
+		validators.NewIPv4AddrValidator("advertise_addr").Optional(true),
+	}
+	for _, v := range vs {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func (proxyagent *SProxyAgent) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	vs := []validators.IValidator{
+		validators.NewIPv4AddrValidator("bind_addr"),
+		validators.NewIPv4AddrValidator("advertise_addr"),
+	}
+	for _, v := range vs {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func (proxyagent *SProxyAgent) ValidateDeleteCondition(ctx context.Context) error {
+	q := ForwardManager.Query().Equals("proxy_agent_id", proxyagent.Id)
+	if count, err := q.CountWithError(); err != nil {
+		return httperrors.NewServerError("count forwards using proxy endpoint %s(%s)",
+			proxyagent.Name, proxyagent.Id)
+	} else if count > 0 {
+		return httperrors.NewConflictError("proxy endpoint %s(%s) is still used by %d forward(s)",
+			proxyagent.Name, proxyagent.Id, count)
+	} else {
+		return nil
+	}
+}
+
+func (man *SProxyAgentManager) allAgents(ctx context.Context) ([]SProxyAgent, error) {
+	var (
+		agents []SProxyAgent
+		q      = man.Query()
+	)
+	if err := db.FetchModelObjects(man, q, &agents); err != nil {
+		return nil, httperrors.NewServerError("query forwards by agent failed: %v", err)
+	}
+	return agents, nil
+}

pkg/cloudproxy/models/proxy_endpoints.go

@@ -0,0 +1,260 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	cloudproxy_api "yunion.io/x/onecloud/pkg/apis/cloudproxy"
+	compute_apis "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
+
+// Add revision?
+type SProxyEndpoint struct {
+	db.SVirtualResourceBase
+
+	User       string `nullable:"false" list:"user" update:"user" create:"optional"`
+	Host       string `nullable:"false" list:"user" update:"user" create:"required"`
+	Port       int    `nullable:"false" list:"user" update:"user" create:"optional"`
+	PrivateKey string `nullable:"false" update:"user" list:"admin" get:"admin" create:"required"` // do not allow get, list
+
+	IntranetIpAddr string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"required"`
+
+	StatusDetail string `width:"128" charset:"ascii" nullable:"false" default:"init" list:"user" create:"optional" json:"status_detail"`
+}
+
+type SProxyEndpointManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var ProxyEndpointManager *SProxyEndpointManager
+
+func init() {
+	ProxyEndpointManager = &SProxyEndpointManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SProxyEndpoint{},
+			"proxy_endpoints_tbl",
+			"proxy_endpoint",
+			"proxy_endpoints",
+		),
+	}
+	ProxyEndpointManager.SetVirtualObject(ProxyEndpointManager)
+}
+
+func (man *SProxyEndpointManager) AllowPerformCreateFromServer(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAllowClassPerform(rbacutils.ScopeProject, userCred, man, "create-from-server")
+}
+
+func (man *SProxyEndpointManager) PerformCreateFromServer(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *cloudproxy_api.ProxyEndpointCreateFromServerInput) (jsonutils.JSONObject, error) {
+	serverId := input.ServerId
+	if serverId == "" {
+		return nil, httperrors.NewBadRequestError("server_id is required")
+	}
+
+	serverInfo, err := getServerInfo(ctx, userCred, serverId)
+	if err != nil {
+		return nil, err
+	}
+	if serverInfo.PrivateKey == "" {
+		return nil, httperrors.NewBadRequestError("cannot find ssh private key for this server")
+	}
+
+	nic := serverInfo.GetNic()
+	if nic == nil {
+		return nil, httperrors.NewBadRequestError("cannot find usable network interface for this server")
+	}
+	host := serverInfo.Server.Eip
+	if host == "" && nic.VpcId == compute_apis.DEFAULT_VPC_ID {
+		host = nic.IpAddr
+	}
+	if host == "" {
+		return nil, httperrors.NewBadRequestError("cannot find ssh host ip address for this server")
+	}
+
+	proxyendpoint := &SProxyEndpoint{
+		User:       "cloudroot",
+		Host:       host,
+		Port:       22,
+		PrivateKey: serverInfo.PrivateKey,
+
+		IntranetIpAddr: nic.IpAddr,
+	}
+	proxyendpoint.Name = serverInfo.Server.Name
+	proxyendpoint.DomainId = userCred.GetProjectDomainId()
+	proxyendpoint.ProjectId = userCred.GetProjectId()
+	if err := man.TableSpec().Insert(ctx, proxyendpoint); err != nil {
+		return nil, httperrors.NewServerError("database insertion error: %v", err)
+	}
+
+	var proxymatches []*SProxyMatch
+	if nic.VpcId != "" {
+		pm := &SProxyMatch{
+			ProxyEndpointId: proxyendpoint.Id,
+			MatchScope:      cloudproxy_api.PM_SCOPE_VPC,
+			MatchValue:      nic.VpcId,
+		}
+		pm.Name = "vpc-" + nic.VpcId
+		proxymatches = append(proxymatches, pm)
+	}
+
+	if nic.NetworkId != "" {
+		pm := &SProxyMatch{
+			ProxyEndpointId: proxyendpoint.Id,
+			MatchScope:      cloudproxy_api.PM_SCOPE_NETWORK,
+			MatchValue:      nic.NetworkId,
+		}
+		pm.Name = "network-" + nic.NetworkId
+		proxymatches = append(proxymatches, pm)
+	}
+	for _, proxymatch := range proxymatches {
+		proxymatch.DomainId = userCred.GetProjectDomainId()
+		proxymatch.ProjectId = userCred.GetProjectId()
+		if err := ProxyMatchManager.TableSpec().Insert(ctx, proxymatch); err != nil {
+			log.Errorf("failed insertion of proxy match %s: %v", proxymatch.Name, err)
+		}
+	}
+
+	return jsonutils.Marshal(proxyendpoint), nil
+}
+
+func (man *SProxyEndpointManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input cloudproxy_api.ProxyEndpointCreateInput,
+) (*jsonutils.JSONDict, error) {
+	data := jsonutils.Marshal(input).(*jsonutils.JSONDict)
+
+	if input, err := man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput); err != nil {
+		return nil, err
+	} else {
+		data.Update(jsonutils.Marshal(input))
+	}
+
+	vs := []validators.IValidator{
+		validators.NewStringNonEmptyValidator("user").Default("cloudroot"),
+		validators.NewStringNonEmptyValidator("host"),
+		validators.NewPortValidator("port").Default(22),
+		validators.NewSSHKeyValidator("private_key").Optional(true),
+
+		validators.NewIPv4AddrValidator("intranet_ip_addr"),
+	}
+	for _, v := range vs {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	// populate ssh credential through "cloudhost"
+	//
+	// if ! skip validation {
+	// 	ssh credential validation
+	// }
+	return data, nil
+}
+
+func (man *SProxyEndpointManager) getById(id string) (*SProxyEndpoint, error) {
+	m, err := db.FetchById(man, id)
+	if err != nil {
+		return nil, err
+	}
+	proxyendpoint := m.(*SProxyEndpoint)
+	return proxyendpoint, err
+}
+
+func (proxyendpoint *SProxyEndpoint) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input cloudproxy_api.ProxyEndpointUpdateInput) (cloudproxy_api.ProxyEndpointUpdateInput, error) {
+	var err error
+	input.VirtualResourceBaseUpdateInput, err = proxyendpoint.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.VirtualResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SVirtualResourceBase.ValidateUpdateData")
+	}
+
+	data := jsonutils.Marshal(input).(*jsonutils.JSONDict)
+	vs := []validators.IValidator{
+		validators.NewStringNonEmptyValidator("user"),
+		validators.NewStringNonEmptyValidator("host"),
+		validators.NewPortValidator("port"),
+		validators.NewSSHKeyValidator("private_key").Optional(true),
+	}
+	for _, v := range vs {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return input, err
+		}
+	}
+	return input, nil
+}
+
+func (proxyendpoint *SProxyEndpoint) ValidateDeleteCondition(ctx context.Context) error {
+	q := ForwardManager.Query().Equals("proxy_endpoint_id", proxyendpoint.Id)
+	if count, err := q.CountWithError(); err != nil {
+		return httperrors.NewServerError("count forwards using proxy endpoint %s(%s)",
+			proxyendpoint.Name, proxyendpoint.Id)
+	} else if count > 0 {
+		return httperrors.NewConflictError("proxy endpoint %s(%s) is still used by %d forward(s)",
+			proxyendpoint.Name, proxyendpoint.Id, count)
+	} else {
+		return nil
+	}
+}
+
+func (proxyendpoint *SProxyEndpoint) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	var pms []SProxyMatch
+	q := ProxyMatchManager.Query().Equals("proxy_endpoint_id", proxyendpoint.Id)
+	if err := db.FetchModelObjects(ProxyMatchManager, q, &pms); err != nil {
+		return httperrors.NewServerError("fetch proxy matches for endpoint %s(%s)",
+			proxyendpoint.Name, proxyendpoint.Id)
+	}
+
+	for i := range pms {
+		pm := &pms[i]
+		err := db.DeleteModel(ctx, userCred, pm)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (man *SProxyEndpointManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	input cloudproxy_api.ProxyEndpointListInput,
+) (*sqlchemy.SQuery, error) {
+	filters := [][2]string{
+		[2]string{cloudproxy_api.PM_SCOPE_VPC, input.VpcId},
+		[2]string{cloudproxy_api.PM_SCOPE_NETWORK, input.NetworkId},
+	}
+	for _, filter := range filters {
+		if v := filter[1]; v != "" {
+			pmQ := ProxyMatchManager.Query("proxy_endpoint_id").
+				Equals("match_scope", filter[0]).
+				Equals("match_value", v)
+			q = q.In("id", pmQ.SubQuery())
+		}
+	}
+	return q, nil
+}

pkg/cloudproxy/models/proxy_matches.go

@@ -0,0 +1,113 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/cloudproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SProxyMatch struct {
+	db.SVirtualResourceBase
+
+	ProxyEndpointId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	MatchScope      string `width:"16" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	MatchValue      string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+}
+
+type SProxyMatchManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var ProxyMatchManager *SProxyMatchManager
+
+func init() {
+	ProxyMatchManager = &SProxyMatchManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SProxyMatch{},
+			"proxy_matches_tbl",
+			"proxy_match",
+			"proxy_matches",
+		),
+	}
+	ProxyMatchManager.SetVirtualObject(ProxyMatchManager)
+}
+
+func (man *SProxyMatchManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	matchScopeV := validators.NewStringChoicesValidator("match_scope", api.PM_SCOPES)
+	endpointV := validators.NewModelIdOrNameValidator("proxy_endpoint", ProxyEndpointManager.Keyword(), ownerId)
+	for _, v := range []validators.IValidator{
+		matchScopeV,
+		endpointV,
+	} {
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func (pm *SProxyMatch) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	matchScopeV := validators.NewStringChoicesValidator("match_scope", api.PM_SCOPES)
+	endpointV := validators.NewModelIdOrNameValidator("proxy_endpoint", ProxyEndpointManager.Keyword(), userCred)
+	for _, v := range []validators.IValidator{
+		matchScopeV,
+		endpointV,
+	} {
+		v.Optional(true)
+		if err := v.Validate(data); err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func (man *SProxyMatchManager) findMatch(ctx context.Context, networkId, vpcId string) *SProxyMatch {
+	q := man.Query()
+	qfScope := q.Field("match_scope")
+	qfValue := q.Field("match_value")
+	q = q.Filter(
+		sqlchemy.OR(
+			sqlchemy.AND(
+				sqlchemy.Equals(qfScope, api.PM_SCOPE_VPC),
+				sqlchemy.Equals(qfValue, vpcId),
+			),
+			sqlchemy.AND(
+				sqlchemy.Equals(qfScope, api.PM_SCOPE_NETWORK),
+				sqlchemy.Equals(qfValue, networkId),
+			),
+		),
+	)
+
+	var pms []SProxyMatch
+	if err := db.FetchModelObjects(man, q, &pms); err != nil {
+		return nil
+	}
+	var r *SProxyMatch
+	for _, pm := range pms {
+		if pm.MatchScope == api.PM_SCOPE_NETWORK {
+			return &pm
+		}
+		r = &pm
+	}
+	return r
+}

pkg/cloudproxy/models/server_util.go

@@ -0,0 +1,82 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+
+	compute_apis "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	compute_models "yunion.io/x/onecloud/pkg/mcclient/models"
+	compute_modules "yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type serverInfo struct {
+	Server *compute_models.Server
+
+	// PrivateKey is the one corresponds to userCred when getting this
+	// serverInfo instance.  It can be empty
+	PrivateKey string
+}
+
+func (si *serverInfo) GetNic() *compute_models.ServerNic {
+	nic := si.getVPCNic()
+	if nic != nil {
+		return nic
+	}
+
+	for _, nic := range si.Server.Nics {
+		if nic.IpAddr != "" && nic.VpcId == compute_apis.DEFAULT_VPC_ID {
+			return &nic
+		}
+	}
+	return nil
+}
+
+func (si *serverInfo) getVPCNic() *compute_models.ServerNic {
+	for _, nic := range si.Server.Nics {
+		if nic.IpAddr != "" && nic.VpcId != compute_apis.DEFAULT_VPC_ID {
+			return &nic
+		}
+	}
+	return nil
+}
+
+func getServerInfo(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	serverId string,
+) (*serverInfo, error) {
+	sess := auth.GetSession(ctx, userCred, "", "")
+	serverJson, err := compute_modules.Servers.Get(sess, serverId, nil)
+	if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+
+	server := &compute_models.Server{}
+	if err := serverJson.Unmarshal(server); err != nil {
+		return nil, httperrors.NewServerError("unmarshal server %s: %v", serverId, err)
+	}
+
+	privateKey, _ := compute_modules.Sshkeypairs.FetchPrivateKey(ctx, userCred)
+
+	serverInfo := &serverInfo{
+		Server:     server,
+		PrivateKey: privateKey,
+	}
+	return serverInfo, nil
+}

pkg/cloudproxy/models/util_errors.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+type (
+	errNotFound    error
+	errMoreThanOne error
+)
+
+func IsNotFound(err error) bool {
+	_, ok := err.(errNotFound)
+	return ok
+}
+
+func IsMoreThanOne(err error) bool {
+	_, ok := err.(errMoreThanOne)
+	return ok
+}

pkg/cloudproxy/models/util_subnets.go

@@ -0,0 +1,52 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"strings"
+
+	"yunion.io/x/pkg/util/netutils"
+)
+
+type Subnets []*netutils.IPV4Prefix
+
+func (nets Subnets) StrList() []string {
+	r := make([]string, 0, len(nets))
+	for _, p := range nets {
+		r = append(r, p.String())
+	}
+	return r
+}
+
+func (nets Subnets) String() string {
+	r := nets.StrList()
+	return strings.Join(r, ",")
+}
+
+func (nets Subnets) ContainsAny(nets1 Subnets) bool {
+	contains, _ := nets.ContainsAnyEx(nets1)
+	return contains
+}
+
+func (nets Subnets) ContainsAnyEx(nets1 Subnets) (bool, *netutils.IPV4Prefix) {
+	for _, p0 := range nets {
+		for _, p1 := range nets1 {
+			if p0.Equals(p1) {
+				return true, p0
+			}
+		}
+	}
+	return false, nil
+}