mirror of https://github.com/coder/coder.git
fix(scaletest): output error and trace instead of {} for json output (#10075)
This commit is contained in:
Mathias Fredriksson
GitHub
parent
ab9276bd08
commit
5d5a7da67f
|
|
@ -2,11 +2,15 @@ package harness
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/httpapi"
|
||||
)
|
||||
|
||||
|
|
@ -33,6 +37,18 @@ type RunResult struct {
|
|||
DurationMS int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marhshaler for RunResult.
|
||||
func (r RunResult) MarshalJSON() ([]byte, error) {
|
||||
type alias RunResult
|
||||
return json.Marshal(&struct {
|
||||
alias
|
||||
Error string `json:"error"`
|
||||
}{
|
||||
alias: alias(r),
|
||||
Error: fmt.Sprintf("%+v", r.Error),
|
||||
})
|
||||
}
|
||||
|
||||
// Results returns the results of the test run. Panics if the test run is not
|
||||
// done yet.
|
||||
func (r *TestRun) Result() RunResult {
|
||||
|
|
@ -88,7 +104,10 @@ func (h *TestHarness) Results() Results {
|
|||
// PrintText prints the results as human-readable text to the given writer.
|
||||
func (r *Results) PrintText(w io.Writer) {
|
||||
var totalDuration time.Duration
|
||||
for _, run := range r.Runs {
|
||||
keys := maps.Keys(r.Runs)
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
run := r.Runs[key]
|
||||
totalDuration += time.Duration(run.Duration)
|
||||
if run.Error == nil {
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -2,9 +2,15 @@ package harness_test
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
|
|
@ -12,9 +18,18 @@ import (
|
|||
"github.com/coder/coder/v2/scaletest/harness"
|
||||
)
|
||||
|
||||
type testError struct {
|
||||
hidden error
|
||||
}
|
||||
|
||||
func (e testError) Error() string {
|
||||
return e.hidden.Error()
|
||||
}
|
||||
|
||||
func Test_Results(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2023, 10, 5, 12, 3, 56, 395813665, time.UTC)
|
||||
results := harness.Results{
|
||||
TotalRuns: 10,
|
||||
TotalPass: 8,
|
||||
|
|
@ -26,7 +41,7 @@ func Test_Results(t *testing.T) {
|
|||
ID: "0",
|
||||
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
|
||||
Error: xerrors.New("test-0/0 error"),
|
||||
StartedAt: time.Now(),
|
||||
StartedAt: now,
|
||||
Duration: httpapi.Duration(time.Second),
|
||||
DurationMS: 1000,
|
||||
},
|
||||
|
|
@ -36,7 +51,17 @@ func Test_Results(t *testing.T) {
|
|||
ID: "1",
|
||||
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
|
||||
Error: nil,
|
||||
StartedAt: time.Now(),
|
||||
StartedAt: now.Add(333 * time.Millisecond),
|
||||
Duration: httpapi.Duration(time.Second),
|
||||
DurationMS: 1000,
|
||||
},
|
||||
"test-0/2": {
|
||||
FullID: "test-0/2",
|
||||
TestName: "test-0",
|
||||
ID: "2",
|
||||
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
|
||||
Error: testError{hidden: xerrors.New("test-0/2 error")},
|
||||
StartedAt: now.Add(666 * time.Millisecond),
|
||||
Duration: httpapi.Duration(time.Second),
|
||||
DurationMS: 1000,
|
||||
},
|
||||
|
|
@ -45,7 +70,7 @@ func Test_Results(t *testing.T) {
|
|||
ElapsedMS: 1000,
|
||||
}
|
||||
|
||||
expected := `
|
||||
wantText := `
|
||||
== FAIL: test-0/0
|
||||
|
||||
Error: test-0/0 error
|
||||
|
|
@ -53,6 +78,13 @@ func Test_Results(t *testing.T) {
|
|||
Log:
|
||||
test-0/0 log line 1
|
||||
|
||||
== FAIL: test-0/2
|
||||
|
||||
Error: test-0/2 error
|
||||
|
||||
Log:
|
||||
test-0/2 log line 1
|
||||
|
||||
|
||||
Test results:
|
||||
Pass: 8
|
||||
|
|
@ -60,11 +92,63 @@ Test results:
|
|||
Total: 10
|
||||
|
||||
Total duration: 1s
|
||||
Avg. duration: 200ms
|
||||
Avg. duration: 300ms
|
||||
`
|
||||
wantJSON := `{
|
||||
"total_runs": 10,
|
||||
"total_pass": 8,
|
||||
"total_fail": 2,
|
||||
"elapsed": "1s",
|
||||
"elapsed_ms": 1000,
|
||||
"runs": {
|
||||
"test-0/0": {
|
||||
"full_id": "test-0/0",
|
||||
"test_name": "test-0",
|
||||
"id": "0",
|
||||
"logs": "test-0/0 log line 1\ntest-0/0 log line 2",
|
||||
"started_at": "2023-10-05T12:03:56.395813665Z",
|
||||
"duration": "1s",
|
||||
"duration_ms": 1000,
|
||||
"error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]/results_test.go:43"
|
||||
},
|
||||
"test-0/1": {
|
||||
"full_id": "test-0/1",
|
||||
"test_name": "test-0",
|
||||
"id": "1",
|
||||
"logs": "test-0/1 log line 1\ntest-0/1 log line 2",
|
||||
"started_at": "2023-10-05T12:03:56.728813665Z",
|
||||
"duration": "1s",
|
||||
"duration_ms": 1000,
|
||||
"error": "\u003cnil\u003e"
|
||||
},
|
||||
"test-0/2": {
|
||||
"full_id": "test-0/2",
|
||||
"test_name": "test-0",
|
||||
"id": "2",
|
||||
"logs": "test-0/2 log line 1\ntest-0/2 log line 2",
|
||||
"started_at": "2023-10-05T12:03:57.061813665Z",
|
||||
"duration": "1s",
|
||||
"duration_ms": 1000,
|
||||
"error": "test-0/2 error"
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
wd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
wd = filepath.ToSlash(wd) // Hello there Windows, my friend...
|
||||
wantJSON = strings.Replace(wantJSON, "[working_directory]", wd, 1)
|
||||
|
||||
out := bytes.NewBuffer(nil)
|
||||
results.PrintText(out)
|
||||
|
||||
require.Equal(t, expected, out.String())
|
||||
assert.Empty(t, cmp.Diff(wantText, out.String()), "text result does not match (-want +got)")
|
||||
|
||||
out.Reset()
|
||||
enc := json.NewEncoder(out)
|
||||
enc.SetIndent("", "\t")
|
||||
err = enc.Encode(results)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Empty(t, cmp.Diff(wantJSON, out.String()), "JSON result does not match (-want +got)")
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue