From c3733e282b4de11b0982e48b662333d82ca1dd9e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 26 Jul 2019 10:53:47 +0000 Subject: [PATCH 1/4] =?UTF-8?q?ansibleplaybooks:=20=E6=94=B9=E6=AD=A3?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/ansibleserver/models/ansibleplaybooks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ansibleserver/models/ansibleplaybooks.go b/pkg/ansibleserver/models/ansibleplaybooks.go index ed08156557..bab0417f33 100644 --- a/pkg/ansibleserver/models/ansibleplaybooks.go +++ b/pkg/ansibleserver/models/ansibleplaybooks.go @@ -79,6 +79,7 @@ func init() { sessions: ansible.SessionManager{}, sessionsMux: &sync.Mutex{}, } + AnsiblePlaybookManager.SetVirtualObject(AnsiblePlaybookManager) } func (man *SAnsiblePlaybookManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { From ab873ef90d25b2b1c910dd9910129624c1f9cc30 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 26 Jul 2019 10:52:07 +0000 Subject: [PATCH 2/4] =?UTF-8?q?ansibleplaybooks:=20Output=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E4=BD=BF=E7=94=A8mediumtext?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/ansibleserver/models/ansibleplaybooks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ansibleserver/models/ansibleplaybooks.go b/pkg/ansibleserver/models/ansibleplaybooks.go index bab0417f33..912e4a385e 100644 --- a/pkg/ansibleserver/models/ansibleplaybooks.go +++ b/pkg/ansibleserver/models/ansibleplaybooks.go @@ -54,7 +54,7 @@ type SAnsiblePlaybook struct { db.SVirtualResourceBase Playbook *ansible.Playbook `nullable:"false" create:"required" get:"user" update:"user"` - Output string `get:"user"` + Output string `length:"medium" get:"user"` StartTime time.Time `list:"user"` EndTime time.Time `list:"user"` } @@ -253,7 +253,7 @@ func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient. apb.EndTime = time.Now() // truncate to preserve the tail output := pb.Output() - textMax := 64*1024 - 1 + textMax := 64*1024*1024 - 1 if len(output) > textMax { output = output[len(output)-textMax:] } From c3e82b9042c3338a938ffacbf971da86743cec5e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 26 Jul 2019 12:22:56 +0000 Subject: [PATCH 3/4] =?UTF-8?q?pkg:=20ansible:=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=8C=87=E5=AE=9AOutputWriter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/ansible/playbook.go | 30 +++++++----------------------- pkg/util/ansible/playbook_test.go | 5 ++++- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/pkg/util/ansible/playbook.go b/pkg/util/ansible/playbook.go index 337522b81e..d80363111d 100644 --- a/pkg/util/ansible/playbook.go +++ b/pkg/util/ansible/playbook.go @@ -15,7 +15,6 @@ package ansible import ( - "bytes" "context" "io" "io/ioutil" @@ -59,7 +58,7 @@ type Playbook struct { tmpdir string noCleanOnExit bool - stdio *bytes.Buffer + outputWriter io.Writer state pbState stateMux *sync.Mutex } @@ -68,7 +67,6 @@ func NewPlaybook() *Playbook { pb := &Playbook{ state: pbStateInit, stateMux: &sync.Mutex{}, - stdio: &bytes.Buffer{}, } return pb } @@ -212,21 +210,11 @@ func (pb *Playbook) Run(ctx context.Context) (err error) { errs = append(errs, errors.WithMessagef(err1, "run module %q, args %q", m.Name, modArgs)) return } - f := func(r io.Reader) { - b := make([]byte, 4096) - for { - n, err := r.Read(b) - if n > 0 { - // Mix stdout, stderr - pb.stdio.Write(b[:n]) - } - if err != nil { - return - } - } + // Mix stdout, stderr + if pb.outputWriter != nil { + go io.Copy(pb.outputWriter, stdout) + go io.Copy(pb.outputWriter, stderr) } - go f(stdout) - go f(stderr) if err1 := cmd.Wait(); err1 != nil { errs = append(errs, errors.WithMessagef(err1, "wait module %q, args %q", m.Name, modArgs)) // continue to next @@ -235,10 +223,6 @@ func (pb *Playbook) Run(ctx context.Context) (err error) { return nil } -// Output returns the stdio output of the playbook -func (pb *Playbook) Output() []byte { - if pb.stdio != nil { - return pb.stdio.Bytes() - } - return nil +func (pb *Playbook) OutputWriter(w io.Writer) { + pb.outputWriter = w } diff --git a/pkg/util/ansible/playbook_test.go b/pkg/util/ansible/playbook_test.go index d0ebcc3616..66f8db6bc4 100644 --- a/pkg/util/ansible/playbook_test.go +++ b/pkg/util/ansible/playbook_test.go @@ -15,6 +15,7 @@ package ansible import ( + "bytes" "context" "os/exec" "reflect" @@ -73,8 +74,10 @@ func TestPlaybook(t *testing.T) { } }) t.Run("run", func(t *testing.T) { + b := &bytes.Buffer{} + pb.OutputWriter(b) err := pb.Run(context.TODO()) - t.Logf("%s", pb.Output()) + t.Logf("%s", b.String()) if err != nil { t.Fatalf("not expecting err: %v", err) } From e6c3e98a6d7dbd3b4650b4763a0b0743431d30d5 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 26 Jul 2019 12:39:47 +0000 Subject: [PATCH 4/4] =?UTF-8?q?ansibleplaybooks:=20=E5=A2=9E=E9=87=8F?= =?UTF-8?q?=E5=BC=8F=E8=AE=B0=E5=BD=95=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/ansibleserver/models/ansibleplaybooks.go | 36 ++++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/pkg/ansibleserver/models/ansibleplaybooks.go b/pkg/ansibleserver/models/ansibleplaybooks.go index 912e4a385e..797953eed1 100644 --- a/pkg/ansibleserver/models/ansibleplaybooks.go +++ b/pkg/ansibleserver/models/ansibleplaybooks.go @@ -59,6 +59,10 @@ type SAnsiblePlaybook struct { EndTime time.Time `list:"user"` } +const ( + OutputMaxBytes = 64*1024*1024 - 1 +) + type SAnsiblePlaybookManager struct { db.SVirtualResourceBaseManager @@ -218,6 +222,7 @@ func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient. } else { pb.PrivateKey = []byte(k) } + pb.OutputWriter(&ansiblePlaybookOutputRecorder{apb}) man.sessions.Add(apb.Id, pb) @@ -251,13 +256,6 @@ func (apb *SAnsiblePlaybook) runPlaybook(ctx context.Context, userCred mcclient. apb.Status = AnsiblePlaybookStatusSucceeded } apb.EndTime = time.Now() - // truncate to preserve the tail - output := pb.Output() - textMax := 64*1024*1024 - 1 - if len(output) > textMax { - output = output[len(output)-textMax:] - } - apb.Output = string(output) return nil }) if err != nil { @@ -278,3 +276,27 @@ func (apb *SAnsiblePlaybook) stopPlaybook(ctx context.Context, userCred mcclient man.sessions.Stop(apb.Id) return nil } + +type ansiblePlaybookOutputRecorder struct { + apb *SAnsiblePlaybook +} + +func (w *ansiblePlaybookOutputRecorder) Write(p []byte) (n int, err error) { + apb := w.apb + _, err = db.Update(apb, func() error { + cur := apb.Output + i := len(p) + len(cur) - OutputMaxBytes + if i > 0 { + // truncate to preserve the tail + apb.Output = cur[:len(cur)-i] + string(p) + } else { + apb.Output += string(p) + } + return nil + }) + if err != nil { + log.Errorf("ansibleplaybook %s(%s): record output: %v", apb.Name, apb.Id, err) + return 0, err + } + return len(p), nil +}