From 2e82399204a35b1ef114ba2527c7cbacf87b50ce Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 19:17:39 +0800 Subject: [PATCH 1/6] db: add GetItemDetails --- pkg/cloudcommon/db/db_dispatcher.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 494ecc6587..2ae9e9e804 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -898,6 +898,10 @@ func getModelItemDetails(manager IModelManager, item IModel, ctx context.Context } } +func GetItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) { + return getItemDetails(manager, item, ctx, userCred, nil) +} + func getItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { metaFields, excludeFields := GetDetailFields(manager, userCred) fieldFilter := jsonutils.GetQueryStringArray(query, "field") From 92a4bebb2e060d233a00797f35753128ecfd6beb Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 19:17:45 +0800 Subject: [PATCH 2/6] cloudproxy: forwards: show details for fwd returned from create-from-server --- pkg/cloudproxy/models/forwards.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/cloudproxy/models/forwards.go b/pkg/cloudproxy/models/forwards.go index 5949c57a36..9fcce616ca 100644 --- a/pkg/cloudproxy/models/forwards.go +++ b/pkg/cloudproxy/models/forwards.go @@ -256,7 +256,11 @@ func (man *SForwardManager) PerformCreateFromServer(ctx context.Context, userCre data, err = man.validatePortReq(ctx, typ, -1, agentId, epId, data) } - forward := &SForward{} + forwardObj, err := db.NewModelObject(man) + if err != nil { + return nil, httperrors.NewGeneralError(err) + } + forward := forwardObj.(*SForward) if err := data.Unmarshal(forward); err != nil { return nil, httperrors.NewServerError("unmarshal create params: %v", err) } @@ -267,7 +271,7 @@ func (man *SForwardManager) PerformCreateFromServer(ctx context.Context, userCre return nil, httperrors.NewServerError("database insertion error: %v", err) } - return jsonutils.Marshal(forward), nil + return db.GetItemDetails(man, forward, ctx, userCred) } func (man *SForwardManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { From 2cfdb0a08a10c3d6f056bc4cca29afc7dc8210fd Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 10:03:24 +0800 Subject: [PATCH 3/6] cloudproxy: agent: wait a while on connect error --- pkg/cloudproxy/agent/ssh/client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/cloudproxy/agent/ssh/client.go b/pkg/cloudproxy/agent/ssh/client.go index 5f19a11036..99c10212e0 100644 --- a/pkg/cloudproxy/agent/ssh/client.go +++ b/pkg/cloudproxy/agent/ssh/client.go @@ -190,6 +190,12 @@ func (c *Client) runClientState(ctx context.Context, sshClientC chan *ssh.Client sshc, err := c.cc.ConnectContext(tmoCtx) if err != nil { log.Errorf("ssh connect: %v", err) + waitTmo := time.NewTimer(13 * time.Second) + select { + case <-ctx.Done(): + return + case <-waitTmo.C: + } continue } From 20f04d2fc545b76d3ba146672b2df0c7e4b91c9d Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 19:28:16 +0800 Subject: [PATCH 4/6] cloudproxy: agent: more verbose log for ssh connect failure --- pkg/cloudproxy/agent/ssh/client.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cloudproxy/agent/ssh/client.go b/pkg/cloudproxy/agent/ssh/client.go index 99c10212e0..75e4560ea0 100644 --- a/pkg/cloudproxy/agent/ssh/client.go +++ b/pkg/cloudproxy/agent/ssh/client.go @@ -186,10 +186,11 @@ func (c *Client) runClientState(ctx context.Context, sshClientC chan *ssh.Client default: } + cc := c.cc tmoCtx, _ := context.WithTimeout(ctx, 31*time.Second) - sshc, err := c.cc.ConnectContext(tmoCtx) + sshc, err := cc.ConnectContext(tmoCtx) if err != nil { - log.Errorf("ssh connect: %v", err) + log.Errorf("ssh connect: %s@%s, port %d: %v", cc.Username, cc.Host, cc.Port, err) waitTmo := time.NewTimer(13 * time.Second) select { case <-ctx.Done(): From b242af75f40c7383b1031e505decc016a85a6449 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 18:55:54 +0800 Subject: [PATCH 5/6] guests: sshable: log error should unmarshal fail --- pkg/compute/models/guest_sshable.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/compute/models/guest_sshable.go b/pkg/compute/models/guest_sshable.go index a2cd23d04b..987ab1ed6d 100644 --- a/pkg/compute/models/guest_sshable.go +++ b/pkg/compute/models/guest_sshable.go @@ -207,10 +207,11 @@ func (guest *SGuest) sshableTryEach( res, err := cloudproxy_module.Forwards.PerformClassAction(sess, "create-from-server", fwdCreateParams) if err == nil { var fwd cloudproxy_api.ForwardDetails - if err := res.Unmarshal(&fwd); err == nil { - if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok { - return nil - } + if err := res.Unmarshal(&fwd); err != nil { + log.Errorf("unmarshal fwd details: %q", res.String()) + } + if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok { + return nil } } else { var reason string From bfe665bbf67df2af00bc825413db6e4bc92b4a35 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 7 May 2021 19:40:01 +0800 Subject: [PATCH 6/6] guests: sshable: wait and retry for newly created forward --- pkg/compute/models/guest_sshable.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/compute/models/guest_sshable.go b/pkg/compute/models/guest_sshable.go index 987ab1ed6d..4ce414fb59 100644 --- a/pkg/compute/models/guest_sshable.go +++ b/pkg/compute/models/guest_sshable.go @@ -210,8 +210,20 @@ func (guest *SGuest) sshableTryEach( if err := res.Unmarshal(&fwd); err != nil { log.Errorf("unmarshal fwd details: %q", res.String()) } - if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok { - return nil + + tmo := time.NewTimer(13 * time.Second) + tick := time.NewTicker(3 * time.Second) + for { + select { + case <-tmo.C: + break + case <-tick.C: + if ok := guest.sshableTryForward(ctx, tryData, &fwd); ok { + return nil + } + case <-ctx.Done(): + break + } } } else { var reason string