1# Shuffle order of tests and benchmarks
2
3[short] skip 'builds and repeatedly runs a test binary'
4
5# Run tests
6go test -v foo_test.go
7! stdout '-test.shuffle '
8stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
9
10go test -v -shuffle=off foo_test.go
11! stdout '-test.shuffle '
12stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
13
14go test -v -shuffle=42 foo_test.go
15stdout '^-test.shuffle 42'
16stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
17
18go test -v -shuffle=0 foo_test.go
19stdout '^-test.shuffle 0'
20stdout '(?s)TestTwo(.*)TestOne(.*)TestThree'
21
22go test -v -shuffle -1 foo_test.go
23stdout '^-test.shuffle -1'
24stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
25
26go test -v -shuffle=on foo_test.go
27stdout '^-test.shuffle '
28stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
29stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
30stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
31
32
33# Run tests and benchmarks
34go test -v -bench=. foo_test.go
35! stdout '-test.shuffle '
36stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
37
38go test -v -bench=. -shuffle=off foo_test.go
39! stdout '-test.shuffle '
40stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
41
42go test -v -bench=. -shuffle=42 foo_test.go
43stdout '^-test.shuffle 42'
44stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
45
46go test -v -bench=. -shuffle=0 foo_test.go
47stdout '^-test.shuffle 0'
48stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
49
50go test -v -bench=. -shuffle -1 foo_test.go
51stdout '^-test.shuffle -1'
52stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
53
54go test -v -bench=. -shuffle=on foo_test.go
55stdout '^-test.shuffle '
56stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
57stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
58stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
59stdout -count=2 'BenchmarkOne'
60stdout -count=2 'BenchmarkTwo'
61stdout -count=2 'BenchmarkThree'
62
63
64# When running go test -count=N, each of the N runs distinct runs should maintain the same
65# shuffled order of these tests.
66go test -v -shuffle=43 -count=4 foo_test.go
67stdout '^-test.shuffle 43'
68stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne'
69
70go test -v -bench=. -shuffle=44 -count=2 foo_test.go
71stdout '^-test.shuffle 44'
72stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
73
74
75# The feature should work with test binaries as well
76go test -c
77exec ./m.test -test.shuffle=off
78! stdout '^-test.shuffle '
79
80exec ./m.test -test.shuffle=on
81stdout '^-test.shuffle '
82
83exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go
84stdout '^-test.shuffle 0'
85stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
86
87exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go
88stdout '^-test.shuffle 123'
89stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne'
90
91exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go
92stdout '^-test.shuffle -1'
93stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
94
95exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go
96stdout '^-test.shuffle 44'
97stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
98
99
100# Negative testcases for invalid input
101! go test -shuffle -count=2
102stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax'
103
104! go test -shuffle=
105stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax'
106
107! go test -shuffle=' '
108stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax'
109
110! go test -shuffle=true
111stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax'
112
113! go test -shuffle='abc'
114stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax'
115
116-- go.mod --
117module m
118
119go 1.16
120-- foo_test.go --
121package foo
122
123import "testing"
124
125func TestOne(t *testing.T) {}
126func TestTwo(t *testing.T) {}
127func TestThree(t *testing.T) {}
128
129func BenchmarkOne(b *testing.B) {}
130func BenchmarkTwo(b *testing.B) {}
131func BenchmarkThree(b *testing.B) {}
132
133-- foo.go --
134package foo
View as plain text