diff --git a/Makefile b/Makefile index ce8e1b7b511..e2c17eaeba2 100644 --- a/Makefile +++ b/Makefile @@ -281,6 +281,7 @@ endif .PHONY: clean clean: @echo "---> Cleaning up OSS build artifacts." + rm -rf build.assets/build rm -rf $(BUILDDIR) rm -rf $(ER_BPF_BUILDDIR) rm -rf $(RS_BPF_BUILDDIR) @@ -441,10 +442,8 @@ test-go: PACKAGES := $(shell go list ./... | grep -v integration) test-go: CHAOS_FOLDERS := $(shell find . -type f -name '*chaos*.go' -not -path '*/vendor/*' | xargs dirname | uniq) test-go: $(VERSRC) $(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" $(PACKAGES) $(FLAGS) $(ADDFLAGS) \ - | tee tests-unit.json \ | go run build.assets/render-tests/main.go - $(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) -cover \ - | tee -a tests-unit.json \ + $(CGOFLAG) go test -p 4 -cover -json -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(ROLETESTER_TAG) $(DESKTOP_ACCESS_BETA_TAG)" -test.run=TestChaos $(CHAOS_FOLDERS) \ | go run build.assets/render-tests/main.go # diff --git a/build.assets/render-tests/main.go b/build.assets/render-tests/main.go index a28d1c8d51a..03afbfd8e2b 100644 --- a/build.assets/render-tests/main.go +++ b/build.assets/render-tests/main.go @@ -81,11 +81,82 @@ func readInput(input io.Reader, ch chan<- TestEvent) { } } -func main() { +type packageOutput struct { + output []string + subtestOutput map[string][]string + failedSubtests map[string][]string +} - testOutput := make(map[string][]string) - failedTests := make(map[string][]string) - actionCounts := make(map[string]int) +func (pkg *packageOutput) FailedTests() []string { + result := []string{} + for testName := range pkg.failedSubtests { + result = append(result, testName) + } + sort.Strings(result) + return result +} + +type outputMap struct { + pkgs map[string]*packageOutput + actionCounts map[string]int +} + +func newOutputMap() *outputMap { + return &outputMap{ + pkgs: make(map[string]*packageOutput), + actionCounts: make(map[string]int), + } +} + +func (m *outputMap) record(event TestEvent) { + m.actionCounts[event.Action]++ + + var pkgOutput *packageOutput + var exists bool + if pkgOutput, exists = m.pkgs[event.Package]; !exists { + pkgOutput = &packageOutput{ + subtestOutput: make(map[string][]string), + failedSubtests: make(map[string][]string), + } + m.pkgs[event.Package] = pkgOutput + } + + switch event.Action { + case actionOutput: + pkgOutput.output = append(pkgOutput.output, event.Output) + if event.Test != "" { + pkgOutput.subtestOutput[event.Test] = append(pkgOutput.subtestOutput[event.Test], event.Output) + } + + case actionFail: + // If this is a single test result + if event.Test != "" { + pkgOutput.failedSubtests[event.Test] = pkgOutput.subtestOutput[event.Test] + } else { + // If this is a package result, we only want to preserve the package output if + // there are no failed subtests + if len(pkgOutput.failedSubtests) > 0 { + pkgOutput.output = nil + } + } + fallthrough + + case actionPass, actionSkip: + delete(pkgOutput.subtestOutput, event.Test) + } +} + +func (m *outputMap) getPkg(pkgName string) *packageOutput { + return m.pkgs[pkgName] +} + +func (m *outputMap) deletePkg(pkgName string) { + delete(m.pkgs, pkgName) +} + +func main() { + testOutput := newOutputMap() + failedPackages := make(map[string]*packageOutput) coverage := make(map[string]float64) events := make(chan TestEvent) @@ -109,20 +180,26 @@ func main() { testName := event.FullName() - switch event.Action { - case actionOutput: - if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 { - value, err := strconv.ParseFloat(matches[1], 64) - if err != nil { - panic("Malformed coverage value: " + err.Error()) - } - coverage[testName] = value - } - testOutput[testName] = append(testOutput[testName], event.Output) + testOutput.record(event) - case actionPass, actionFail, actionSkip: - // If this is a whole-package summary result - if event.Test == "" { + // if this is whole-package summary result + if event.Test == "" { + switch event.Action { + case actionOutput: + if matches := covPattern.FindStringSubmatch(event.Output); len(matches) != 0 { + value, err := strconv.ParseFloat(matches[1], 64) + if err != nil { + panic("Malformed coverage value: " + err.Error()) + } + coverage[testName] = value + } + + case actionFail: + // cache the failed test output + failedPackages[event.Package] = testOutput.getPkg(event.Package) + fallthrough + + case actionPass, actionSkip: // extract and format coverage value covText := "------" if covValue, ok := coverage[testName]; ok { @@ -131,20 +208,10 @@ func main() { // only display package results as progress messages fmt.Printf("%s %s: %s\n", covText, event.Action, event.Package) - } else { - // packages results don't count towards our test count. - actionCounts[event.Action]++ - // we want to preserve the log of our failed tests for - // later examination - if event.Action == actionFail { - failedTests[testName] = testOutput[testName] - } + // Don't need this no more + testOutput.deletePkg(event.Package) } - // we don't need the test output any more, it's either in the - // errors collection, or the test passed and we don't need to - // display it. - delete(testOutput, testName) } } } @@ -152,33 +219,58 @@ func main() { fmt.Println(separator) fmt.Printf("%d tests passed. %d failed, %d skipped\n", - actionCounts[actionPass], actionCounts[actionFail], actionCounts[actionSkip]) + testOutput.actionCounts[actionPass], testOutput.actionCounts[actionFail], testOutput.actionCounts[actionSkip]) fmt.Println(separator) - if len(failedTests) == 0 { + if len(failedPackages) == 0 { fmt.Println("All tests pass. Yay!") - return + os.Exit(0) } - names := make([]string, 0, len(failedTests)) - for k := range failedTests { + // Generate a sorted list of package names, so that we present the + // packages that fail in a repeatable order. + names := make([]string, 0, len(failedPackages)) + for k := range failedPackages { names = append(names, k) } sort.Strings(names) - for _, testName := range names { - fmt.Printf("FAIL: %s\n", testName) - } + // Print a summary list of the failed tests and packages. + for _, pkgName := range names { + pkg := failedPackages[pkgName] - for _, testName := range names { - fmt.Println(separator) - fmt.Printf("TEST %s OUTPUT\n", testName) - fmt.Println(separator) - for _, l := range failedTests[testName] { - fmt.Print(l) + fmt.Printf("FAIL: %s\n", pkgName) + + for testName := range pkg.failedSubtests { + fmt.Printf("FAIL: %s.%s\n", pkgName, testName) } } - os.Exit(1) + fmt.Println(separator) + + // Print the output of each failed package or test. Note that we only print + // the package output if there is no identifiable test that caused the + // failure, as it will probably swamp the individual test output. + for _, pkgName := range names { + pkg := failedPackages[pkgName] + + if len(pkg.failedSubtests) == 0 { + fmt.Printf("OUTPUT %s\n", pkgName) + fmt.Println(separator) + for _, l := range pkg.output { + fmt.Print(l) + } + fmt.Println(separator) + } else { + for _, testName := range pkg.FailedTests() { + fmt.Printf("OUTPUT %s.%s\n", pkgName, testName) + fmt.Println(separator) + for _, l := range pkg.failedSubtests[testName] { + fmt.Print(l) + } + fmt.Println(separator) + } + } + } }