ALL

For reading and verification only. You may not copy, use, modify or redistribute this code. Full notice on the main page.

tunnel/selfcheck_test.go

SHA-256: e565d1b1a3a13656ce1b84e4fae18f66c495019780bc7ce9c69a078c5c2a6231
452 lines · 19872 bytes · View on GitHub

1package main
2
3import (
4 "bytes"
5 "context"
6 "crypto/ecdsa"
7 "crypto/elliptic"
8 "crypto/rand"
9 "crypto/sha256"
10 "crypto/tls"
11 "crypto/x509"
12 "crypto/x509/pkix"
13 "encoding/base64"
14 "encoding/json"
15 "fmt"
16 "math/big"
17 "net"
18 "net/http"
19 "net/http/httptest"
20 "strings"
21 "sync"
22 "testing"
23 "time"
24)
25
26// ALL 2.0, WP26: the self-check. A real TLS server with a real certificate chain is dialed the way ChatGPT will, and the first step that fails is the answer: a wrong chain,
27// an expired certificate, a key that is not the one the computer expects, a name that is not its own, a TLS version below 1.3, a /healthz that does not say ok, a port
28// that is closed and a server that never answers. Every answer is the contract's selfcheck-result.
29
30const selfCheckHost = "abcdefghijklmnopqrst.devices.all-e2e.test"
31
32type selfCheckAuthority struct {
33 pool *x509.CertPool
34 cert *x509.Certificate
35 key *ecdsa.PrivateKey
36}
37
38func newSelfCheckAuthority(t *testing.T, name string) *selfCheckAuthority {
39 t.Helper()
40 key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
41 if err != nil {
42 t.Fatal(err)
43 }
44 template := &x509.Certificate{
45 SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-48 * time.Hour), NotAfter: time.Now().Add(240 * time.Hour),
46 IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
47 }
48 der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
49 if err != nil {
50 t.Fatal(err)
51 }
52 cert, err := x509.ParseCertificate(der)
53 if err != nil {
54 t.Fatal(err)
55 }
56 pool := x509.NewCertPool()
57 pool.AddCert(cert)
58 return &selfCheckAuthority{pool: pool, cert: cert, key: key}
59}
60
61type selfCheckLeaf struct {
62 certificate tls.Certificate
63 spki string
64 notAfter int64
65}
66
67// issue signs a leaf for `host` that is valid from `notBefore` to `notAfter`.
68func (a *selfCheckAuthority) issue(t *testing.T, host string, notBefore, notAfter time.Time) selfCheckLeaf {
69 t.Helper()
70 key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
71 if err != nil {
72 t.Fatal(err)
73 }
74 template := &x509.Certificate{
75 SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: host}, DNSNames: []string{host}, NotBefore: notBefore, NotAfter: notAfter,
76 KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
77 }
78 der, err := x509.CreateCertificate(rand.Reader, template, a.cert, &key.PublicKey, a.key)
79 if err != nil {
80 t.Fatal(err)
81 }
82 leaf, err := x509.ParseCertificate(der)
83 if err != nil {
84 t.Fatal(err)
85 }
86 sum := sha256.Sum256(leaf.RawSubjectPublicKeyInfo)
87 return selfCheckLeaf{certificate: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, spki: base64.RawURLEncoding.EncodeToString(sum[:]), notAfter: leaf.NotAfter.Unix()}
88}
89
90func (a *selfCheckAuthority) good(t *testing.T) selfCheckLeaf {
91 return a.issue(t, selfCheckHost, time.Now().Add(-time.Hour), time.Now().Add(45*24*time.Hour))
92}
93
94// serveSelfCheck runs a TLS server (HTTP/1.1 behind it) on a loopback port with the certificate and the TLS limits given, and returns its address.
95func serveSelfCheck(t *testing.T, leaf selfCheckLeaf, maxVersion uint16, healthz http.HandlerFunc) string {
96 t.Helper()
97 mux := http.NewServeMux()
98 mux.HandleFunc("/healthz", healthz)
99 server := httptest.NewUnstartedServer(mux)
100 server.TLS = &tls.Config{Certificates: []tls.Certificate{leaf.certificate}, MinVersion: tls.VersionTLS12, MaxVersion: maxVersion}
101 server.StartTLS()
102 t.Cleanup(server.Close)
103 return server.Listener.Addr().String()
104}
105
106func okHealthz(w http.ResponseWriter, _ *http.Request) {
107 w.Header().Set("Content-Type", "application/json")
108 _, _ = w.Write([]byte(`{"ok":true}`))
109}
110
111func checkWith(t *testing.T, authority *selfCheckAuthority, addr, host string, expect []string, change func(*SelfCheckConfig)) SelfCheckResult {
112 t.Helper()
113 cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
114 if change != nil {
115 change(&cfg)
116 }
117 result := SelfCheck(context.Background(), cfg, addr, host, expect)
118 // every answer, whatever it says, is the contract and carries no host and no address
119 encoded, err := json.Marshal(result)
120 if err != nil {
121 t.Fatal(err)
122 }
123 var instance any
124 if err := json.Unmarshal(encoded, &instance); err != nil {
125 t.Fatal(err)
126 }
127 if problems := ctLoadSchemas().validate("selfcheck-result.schema.json", instance); len(problems) != 0 {
128 t.Fatalf("not the contract: %v in %s", problems, encoded)
129 }
130 if strings.Contains(string(encoded), host) || strings.Contains(string(encoded), "127.0.0.1") {
131 t.Fatalf("the answer names a host or an address: %s", encoded)
132 }
133 return result
134}
135
136func reasonOf(r SelfCheckResult) string {
137 if r.Reason == nil {
138 return ""
139 }
140 return *r.Reason
141}
142
143func TestSelfCheckPassesForAGoodChainTheRightKeyAndAHealthyPath(t *testing.T) {
144 authority := newSelfCheckAuthority(t, "good root")
145 leaf := authority.good(t)
146 addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
147 result := checkWith(t, authority, addr, selfCheckHost, []string{leaf.spki}, nil)
148 if !result.OK || result.Reason != nil || !result.Healthz {
149 t.Fatalf("a good path: %+v", result)
150 }
151 if result.TLSVersion == nil || *result.TLSVersion != "1.3" || result.SPKISHA256 == nil || *result.SPKISHA256 != leaf.spki || result.NotAfter == nil || *result.NotAfter != leaf.notAfter {
152 t.Fatalf("what it saw: %+v", result)
153 }
154 // without an expectation (the operator's canary asks only whether the chain is good) the key is not compared
155 if anyKey := checkWith(t, authority, addr, selfCheckHost, nil, nil); !anyKey.OK {
156 t.Fatalf("no expectation: %+v", anyKey)
157 }
158 // the expected set may hold the key of the generation that is about to replace this one
159 if two := checkWith(t, authority, addr, selfCheckHost, []string{strings.Repeat("A", 43), leaf.spki}, nil); !two.OK {
160 t.Fatalf("a set of two: %+v", two)
161 }
162}
163
164func TestSelfCheckFailsForAChainThatDoesNotLeadToAPublicRoot(t *testing.T) {
165 trusted := newSelfCheckAuthority(t, "the root the checker trusts")
166 stranger := newSelfCheckAuthority(t, "another root")
167 addr := serveSelfCheck(t, stranger.good(t), tls.VersionTLS13, okHealthz)
168 result := checkWith(t, trusted, addr, selfCheckHost, nil, nil)
169 if result.OK || reasonOf(result) != selfCheckChain || result.Healthz {
170 t.Fatalf("a wrong chain: %+v", result)
171 }
172 // and a self-signed leaf with no authority at all
173 selfSigned := trusted.issue(t, selfCheckHost, time.Now().Add(-time.Hour), time.Now().Add(time.Hour))
174 selfSigned.certificate.Certificate = selfSigned.certificate.Certificate[:1]
175 key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
176 template := &x509.Certificate{SerialNumber: big.NewInt(7), Subject: pkix.Name{CommonName: selfCheckHost}, DNSNames: []string{selfCheckHost}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
177 der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
178 if err != nil {
179 t.Fatal(err)
180 }
181 own := serveSelfCheck(t, selfCheckLeaf{certificate: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}}, tls.VersionTLS13, okHealthz)
182 if again := checkWith(t, trusted, own, selfCheckHost, nil, nil); reasonOf(again) != selfCheckChain {
183 t.Fatalf("a self-signed leaf: %+v", again)
184 }
185}
186
187func TestSelfCheckFailsForAnExpiredCertificateAndForOneThatIsNotYetValid(t *testing.T) {
188 authority := newSelfCheckAuthority(t, "root")
189 expired := authority.issue(t, selfCheckHost, time.Now().Add(-90*24*time.Hour), time.Now().Add(-time.Hour))
190 result := checkWith(t, authority, serveSelfCheck(t, expired, tls.VersionTLS13, okHealthz), selfCheckHost, []string{expired.spki}, nil)
191 if result.OK || reasonOf(result) != selfCheckExpired {
192 t.Fatalf("expired: %+v", result)
193 }
194 if result.NotAfter == nil || *result.NotAfter != expired.notAfter {
195 t.Fatalf("the answer still says when it expired: %+v", result)
196 }
197 future := authority.issue(t, selfCheckHost, time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour))
198 if early := checkWith(t, authority, serveSelfCheck(t, future, tls.VersionTLS13, okHealthz), selfCheckHost, nil, nil); reasonOf(early) != selfCheckExpired {
199 t.Fatalf("not yet valid: %+v", early)
200 }
201 // validity is judged at the moment the check says: a good certificate seen ten years from now is an expired one
202 good := authority.good(t)
203 later := checkWith(t, authority, serveSelfCheck(t, good, tls.VersionTLS13, okHealthz), selfCheckHost, nil, func(cfg *SelfCheckConfig) {
204 cfg.Now = func() time.Time { return time.Now().Add(10 * 365 * 24 * time.Hour) }
205 })
206 if reasonOf(later) != selfCheckExpired {
207 t.Fatalf("in ten years: %+v", later)
208 }
209}
210
211func TestSelfCheckFailsForAKeyThatIsNotTheOneTheComputerExpects(t *testing.T) {
212 authority := newSelfCheckAuthority(t, "root")
213 leaf := authority.good(t)
214 other := authority.good(t)
215 addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
216 result := checkWith(t, authority, addr, selfCheckHost, []string{other.spki}, nil)
217 if result.OK || reasonOf(result) != selfCheckSPKI || result.Healthz {
218 t.Fatalf("a mismatching key: %+v", result)
219 }
220 if result.SPKISHA256 == nil || *result.SPKISHA256 != leaf.spki {
221 t.Fatalf("the answer says which key it saw (so the operator can compare): %+v", result)
222 }
223}
224
225func TestSelfCheckFailsForANameThatIsNotTheComputersOwn(t *testing.T) {
226 authority := newSelfCheckAuthority(t, "root")
227 wrong := authority.issue(t, "tsrqponmlkjihgfedcba.devices.all-e2e.test", time.Now().Add(-time.Hour), time.Now().Add(time.Hour))
228 result := checkWith(t, authority, serveSelfCheck(t, wrong, tls.VersionTLS13, okHealthz), selfCheckHost, []string{wrong.spki}, nil)
229 if result.OK || reasonOf(result) != selfCheckName {
230 t.Fatalf("a certificate for another label: %+v", result)
231 }
232}
233
234func TestSelfCheckRequiresTLS13(t *testing.T) {
235 authority := newSelfCheckAuthority(t, "root")
236 leaf := authority.good(t)
237 result := checkWith(t, authority, serveSelfCheck(t, leaf, tls.VersionTLS12, okHealthz), selfCheckHost, []string{leaf.spki}, nil)
238 if result.OK || reasonOf(result) != selfCheckTLSVersion || result.TLSVersion == nil || *result.TLSVersion != "1.2" {
239 t.Fatalf("a TLS 1.2 server: %+v", result)
240 }
241}
242
243func TestSelfCheckFailsWhenHealthzDoesNotSayOk(t *testing.T) {
244 authority := newSelfCheckAuthority(t, "root")
245 leaf := authority.good(t)
246 for name, handler := range map[string]http.HandlerFunc{
247 "a server error": func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "down", http.StatusInternalServerError) },
248 "not ok": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"ok":false}`)) },
249 "not json": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`<html>welcome</html>`)) },
250 "no member": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{}`)) },
251 "a redirect": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/elsewhere", http.StatusFound) },
252 } {
253 result := checkWith(t, authority, serveSelfCheck(t, leaf, tls.VersionTLS13, handler), selfCheckHost, []string{leaf.spki}, nil)
254 if result.OK || reasonOf(result) != selfCheckHealthz || result.Healthz {
255 t.Fatalf("%s: %+v", name, result)
256 }
257 if result.SPKISHA256 == nil {
258 t.Fatalf("%s: the TLS part passed, so the answer still carries what it saw", name)
259 }
260 }
261}
262
263func TestSelfCheckSaysUnreachableForAClosedPortAndTimeoutForAServerThatNeverAnswers(t *testing.T) {
264 authority := newSelfCheckAuthority(t, "root")
265 closed, err := net.Listen("tcp", "127.0.0.1:0")
266 if err != nil {
267 t.Fatal(err)
268 }
269 address := closed.Addr().String()
270 closed.Close()
271 if result := checkWith(t, authority, address, selfCheckHost, nil, nil); result.OK || reasonOf(result) != selfCheckUnreachable {
272 t.Fatalf("a closed port: %+v", result)
273 }
274 silent, err := net.Listen("tcp", "127.0.0.1:0")
275 if err != nil {
276 t.Fatal(err)
277 }
278 defer silent.Close()
279 var mu sync.Mutex
280 var held []net.Conn
281 go func() {
282 for {
283 conn, err := silent.Accept()
284 if err != nil {
285 return
286 }
287 mu.Lock()
288 held = append(held, conn)
289 mu.Unlock()
290 }
291 }()
292 defer func() {
293 mu.Lock()
294 for _, conn := range held {
295 conn.Close()
296 }
297 mu.Unlock()
298 }()
299 started := time.Now()
300 result := checkWith(t, authority, silent.Addr().String(), selfCheckHost, nil, func(cfg *SelfCheckConfig) { cfg.Timeout = 300 * time.Millisecond })
301 if result.OK || reasonOf(result) != selfCheckTimeout {
302 t.Fatalf("a server that never answers: %+v", result)
303 }
304 if elapsed := time.Since(started); elapsed > 3*time.Second {
305 t.Fatalf("the timeout did not bound the check: %v", elapsed)
306 }
307 // a server that answers the handshake and then says nothing about /healthz is a timeout too, not a hang
308 leaf := authority.good(t)
309 stalled := serveSelfCheck(t, leaf, tls.VersionTLS13, func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })
310 started = time.Now()
311 hung := checkWith(t, authority, stalled, selfCheckHost, []string{leaf.spki}, func(cfg *SelfCheckConfig) { cfg.Timeout = 400 * time.Millisecond })
312 if hung.OK || reasonOf(hung) != selfCheckTimeout || time.Since(started) > 3*time.Second {
313 t.Fatalf("a healthz that never answers: %+v after %v", hung, time.Since(started))
314 }
315}
316
317func TestSelfCheckReasonsAreThePolicysEnumInTheOrderTheStepsRun(t *testing.T) {
318 policy := ctPolStrings("enums.selfcheck_reason")
319 ours := []string{selfCheckChain, selfCheckExpired, selfCheckSPKI, selfCheckName, selfCheckTLSVersion, selfCheckHealthz, selfCheckUnreachable, selfCheckTimeout}
320 if fmt.Sprint(policy) != fmt.Sprint(ours) {
321 t.Fatalf("policy %v, code %v", policy, ours)
322 }
323}
324
325func TestSelfCheckTunnelsThroughTheDialerItIsGiven(t *testing.T) {
326 // the node dials its own public path through whatever Dial gives: here, a function that ignores the address and goes to the test server
327 authority := newSelfCheckAuthority(t, "root")
328 leaf := authority.good(t)
329 real := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
330 var dialed string
331 result := checkWith(t, authority, "203.0.113.9:443", selfCheckHost, []string{leaf.spki}, func(cfg *SelfCheckConfig) {
332 cfg.Dial = func(ctx context.Context, network, addr string) (net.Conn, error) {
333 dialed = addr
334 return (&net.Dialer{}).DialContext(ctx, network, real)
335 }
336 })
337 if !result.OK || dialed != "203.0.113.9:443" {
338 t.Fatalf("%+v dialed %q", result, dialed)
339 }
340}
341
342// ---- the HTTP shape ----------------------------------------------------------------------------------------------------------------------------
343
344func selfCheckPost(handler http.Handler, body string) *httptest.ResponseRecorder {
345 request := httptest.NewRequest(http.MethodPost, "/selfcheck", bytes.NewBufferString(body))
346 recorder := httptest.NewRecorder()
347 handler.ServeHTTP(recorder, request)
348 return recorder
349}
350
351func TestSelfCheckHandlerAnswersTheContractAndRefusesWhatItDoesNotKnow(t *testing.T) {
352 authority := newSelfCheckAuthority(t, "root")
353 leaf := authority.good(t)
354 addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
355 cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
356 labelOK := func(label string) bool { return ctLabelOK(label) }
357 lookup := func(label string) (string, bool) {
358 return label + ".devices.all-e2e.test", label == "abcdefghijklmnopqrst"
359 }
360 handler := NewSelfCheckHandler(cfg, addr, lookup, labelOK, 0)
361 good := selfCheckPost(handler, fmt.Sprintf(`{"v":1,"label":"abcdefghijklmnopqrst","expect":["%s"]}`, leaf.spki))
362 var answer map[string]any
363 if good.Code != http.StatusOK || json.Unmarshal(good.Body.Bytes(), &answer) != nil || answer["ok"] != true {
364 t.Fatalf("%d %s", good.Code, good.Body.String())
365 }
366 if problems := ctLoadSchemas().validate("selfcheck-result.schema.json", answer); len(problems) != 0 {
367 t.Fatalf("not the contract: %v", problems)
368 }
369 if good.Header().Get("Cache-Control") != "no-store" {
370 t.Fatal("an answer about one computer is never cached")
371 }
372 // an unknown or offline label is the same kind of answer as any other failure: unreachable, never a different status that tells which labels exist
373 offline := selfCheckPost(handler, `{"v":1,"label":"tsrqponmlkjihgfedcba","expect":[]}`)
374 var gone map[string]any
375 if offline.Code != http.StatusOK || json.Unmarshal(offline.Body.Bytes(), &gone) != nil || gone["ok"] != false || gone["reason"] != "unreachable" {
376 t.Fatalf("%d %s", offline.Code, offline.Body.String())
377 }
378 for _, body := range []string{
379 ``, `{`, `[]`, `{"v":2,"label":"abcdefghijklmnopqrst","expect":[]}`, `{"v":1,"label":"short","expect":[]}`, `{"v":1,"label":"ABCDEFGHIJKLMNOPQRST","expect":[]}`,
380 `{"v":1,"label":"abcdefghijklmnopqrst","expect":["short"]}`, `{"v":1,"label":"abcdefghijklmnopqrst","expect":[],"host":"evil.example"}`,
381 `{"v":1,"label":"abcdefghijklmnopqrst","expect":[]} {}`, fmt.Sprintf(`{"v":1,"label":"abcdefghijklmnopqrst","expect":["%[1]s","%[1]s","%[1]s","%[1]s","%[1]s"]}`, strings.Repeat("A", 43)),
382 `{"v":1,"label":"abcdefghijklmnopqrst","expect":[],"pad":"` + strings.Repeat("x", 2000) + `"}`,
383 } {
384 if refused := selfCheckPost(handler, body); refused.Code != http.StatusBadRequest {
385 t.Fatalf("%q: %d %s", truncate(body), refused.Code, refused.Body.String())
386 }
387 }
388 for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} {
389 recorder := httptest.NewRecorder()
390 handler.ServeHTTP(recorder, httptest.NewRequest(method, "/selfcheck", nil))
391 if recorder.Code != http.StatusMethodNotAllowed {
392 t.Fatalf("%s: %d", method, recorder.Code)
393 }
394 }
395}
396
397func TestSelfCheckHandlerSpacesTheChecksOfOneLabelAndCapsTheOnesInFlight(t *testing.T) {
398 authority := newSelfCheckAuthority(t, "root")
399 leaf := authority.good(t)
400 addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
401 cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
402 lookup := func(label string) (string, bool) { return selfCheckHost, true }
403 spaced := NewSelfCheckHandler(cfg, addr, lookup, ctLabelOK, time.Hour)
404 body := func(label string) string {
405 return fmt.Sprintf(`{"v":1,"label":"%s","expect":["%s"]}`, label, leaf.spki)
406 }
407 if first := selfCheckPost(spaced, body("abcdefghijklmnopqrst")); first.Code != http.StatusOK {
408 t.Fatalf("%d %s", first.Code, first.Body.String())
409 }
410 if again := selfCheckPost(spaced, body("abcdefghijklmnopqrst")); again.Code != http.StatusTooManyRequests || !strings.Contains(again.Body.String(), `"busy"`) {
411 t.Fatalf("a second check of the same label within the spacing: %d %s", again.Code, again.Body.String())
412 }
413 if other := selfCheckPost(spaced, body("tsrqponmlkjihgfedcba")); other.Code != http.StatusOK {
414 t.Fatalf("another label is its own: %d", other.Code)
415 }
416 // at most selfCheckInFlight checks run at once, whatever labels they are for: the rest are busy, none waits
417 release := make(chan struct{})
418 started := make(chan struct{}, 16)
419 slow := func(ctx context.Context, network, addr string) (net.Conn, error) {
420 started <- struct{}{}
421 select {
422 case <-release:
423 case <-ctx.Done():
424 }
425 return nil, context.Canceled
426 }
427 capped := NewSelfCheckHandler(SelfCheckConfig{Dial: slow, Roots: authority.pool, Timeout: 5 * time.Second}, addr, lookup, ctLabelOK, 0)
428 var wg sync.WaitGroup
429 results := make(chan int, 16)
430 labels := []string{"aaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbb", "cccccccccccccccccccc", "dddddddddddddddddddd", "eeeeeeeeeeeeeeeeeeee", "ffffffffffffffffffff"}
431 for _, label := range labels[:selfCheckInFlight] {
432 wg.Add(1)
433 go func(label string) {
434 defer wg.Done()
435 results <- selfCheckPost(capped, body(label)).Code
436 }(label)
437 }
438 for range labels[:selfCheckInFlight] {
439 <-started
440 }
441 if extra := selfCheckPost(capped, body(labels[selfCheckInFlight])); extra.Code != http.StatusTooManyRequests {
442 t.Fatalf("one more than the cap: %d", extra.Code)
443 }
444 close(release)
445 wg.Wait()
446 close(results)
447 for code := range results {
448 if code != http.StatusOK {
449 t.Fatalf("the checks in flight finish normally: %d", code)
450 }
451 }
452}