ALL

Solo para leer y verificar. No puedes copiar, usar, modificar ni redistribuir este código. Aviso completo en la página principal.

tunnel/jwks_test.go

SHA-256: 2388e1cc03f5d5924f96cfb84da3046a9b3965a5cf6da8a2114d448d6a9aa03d
484 líneas · 17053 bytes · Ver en GitHub

1package main
2
3import (
4 "bufio"
5 "context"
6 "crypto/ed25519"
7 "crypto/rand"
8 "errors"
9 "fmt"
10 "net/http"
11 "net/http/httptest"
12 "strings"
13 "sync"
14 "sync/atomic"
15 "testing"
16 "time"
17)
18
19// The JWKS cache: the control plane's keys are the root of trust of every session the node verifies offline.
20
21// jwksSource is a control plane's key endpoint as the cache sees it: a set that the tests change and a switch that makes it fail.
22type jwksSource struct {
23 mu sync.Mutex
24 keys map[string]ed25519.PublicKey
25 fail error
26 raw []byte
27 calls atomic.Int64
28}
29
30func newJWKSSource(kids ...string) *jwksSource {
31 s := &jwksSource{keys: map[string]ed25519.PublicKey{}}
32 for _, kid := range kids {
33 s.add(kid)
34 }
35 return s
36}
37
38func (s *jwksSource) add(kid string) ed25519.PublicKey {
39 pub, _, _ := ed25519.GenerateKey(rand.Reader)
40 s.mu.Lock()
41 s.keys[kid] = pub
42 s.mu.Unlock()
43 return pub
44}
45
46func (s *jwksSource) drop(kid string) {
47 s.mu.Lock()
48 delete(s.keys, kid)
49 s.mu.Unlock()
50}
51
52func (s *jwksSource) setFail(err error) {
53 s.mu.Lock()
54 s.fail = err
55 s.mu.Unlock()
56}
57
58func (s *jwksSource) fetch(ctx context.Context) ([]byte, error) {
59 s.calls.Add(1)
60 s.mu.Lock()
61 defer s.mu.Unlock()
62 if s.fail != nil {
63 return nil, s.fail
64 }
65 if s.raw != nil {
66 return s.raw, nil
67 }
68 list := []any{}
69 for kid, key := range s.keys {
70 list = append(list, map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(key), "kid": kid, "alg": "EdDSA", "use": "sig", "key_ops": []any{"verify"}})
71 }
72 return []byte(canonicalJSON(map[string]any{"keys": list})), nil
73}
74
75func newTestCache(src *jwksSource, clk *fakeClock) *JWKSCache {
76 return NewJWKSCache(JWKSConfig{Fetch: src.fetch, Clock: newClock(clk.Now, clk.AfterFunc), MinReload: 5 * time.Second})
77}
78
79func TestJWKS_FetchesOnDemandCachesFor300sAndRefreshesBehindTheLookup(t *testing.T) {
80 src := newJWKSSource("a")
81 clk := newFakeClock()
82 c := newTestCache(src, clk)
83 if policyData.JWKS.CacheS != 300 || c.cfg.TTL != 300*time.Second {
84 t.Fatalf("the cache lives %s; the plan says 300 s", c.cfg.TTL)
85 }
86 if _, ok := c.Key("a"); !ok || src.calls.Load() != 1 {
87 t.Fatalf("first lookup: fetched %d", src.calls.Load())
88 }
89 for i := 0; i < 50; i++ {
90 if _, ok := c.Key("a"); !ok {
91 t.Fatal("a cached key was lost")
92 }
93 }
94 if src.calls.Load() != 1 {
95 t.Fatalf("a cached set was fetched again: %d", src.calls.Load())
96 }
97 // past its TTL a lookup still answers from the set (it is vouched for) and refreshes behind it
98 clk.Advance(301 * time.Second)
99 if _, ok := c.Key("a"); !ok {
100 t.Fatal("a set past its TTL but inside the stale window must still serve")
101 }
102 eventually(t, "the refresh behind the lookup", 2*time.Second, func() bool { return src.calls.Load() == 2 })
103}
104
105func TestJWKS_ARotationIsFollowedAtTheNextLookupAndADroppedKeyStopsBeingTrusted(t *testing.T) {
106 src := newJWKSSource("old")
107 clk := newFakeClock()
108 c := newTestCache(src, clk)
109 if _, ok := c.Key("old"); !ok {
110 t.Fatal("old")
111 }
112 newPub := src.add("new")
113 clk.Advance(6 * time.Second) // past the minimum gap between forced reloads
114 got, ok := c.Key("new") // an unknown kid reloads
115 if !ok || !got.Equal(newPub) {
116 t.Fatal("a rotated-in key was not picked up at the first lookup that needed it")
117 }
118 if _, ok := c.Key("old"); !ok {
119 t.Fatal("the old key is still published, so still trusted")
120 }
121 src.drop("old")
122 clk.Advance(301 * time.Second)
123 if err := c.Refresh(context.Background()); err != nil {
124 t.Fatal(err)
125 }
126 if _, ok := c.Key("old"); ok {
127 t.Fatal("a key the control plane stopped publishing is still trusted")
128 }
129 if _, ok := c.Key("new"); !ok {
130 t.Fatal("new")
131 }
132}
133
134func TestJWKS_AFloodOfUnknownKidsIsOneFetchPerMinimumGapNotAFetchPerSession(t *testing.T) {
135 src := newJWKSSource("a")
136 clk := newFakeClock()
137 c := newTestCache(src, clk)
138 if _, ok := c.Key("a"); !ok {
139 t.Fatal("a")
140 }
141 before := src.calls.Load()
142 var wg sync.WaitGroup
143 var wrongly atomic.Int64
144 for i := 0; i < 4000; i++ {
145 wg.Add(1)
146 go func(i int) {
147 defer wg.Done()
148 if _, ok := c.Key(fmt.Sprintf("random-%d", i)); ok {
149 wrongly.Add(1)
150 }
151 }(i)
152 }
153 wg.Wait()
154 if wrongly.Load() != 0 {
155 t.Fatal("an unknown kid was accepted")
156 }
157 // the first of them may fetch (the cache had not looked for a while); the rest never do
158 if n := src.calls.Load() - before; n > 1 {
159 t.Fatalf("4000 sessions with unknown kids made %d fetches", n)
160 }
161 // a legitimate rotation right after the flood waits for the gap, and is picked up after it
162 src.add("rotated")
163 if _, ok := c.Key("rotated"); ok {
164 t.Fatal("served inside the minimum gap")
165 }
166 clk.Advance(6 * time.Second)
167 if _, ok := c.Key("rotated"); !ok {
168 t.Fatal("not picked up after the gap")
169 }
170 // and the known keys were never refused while the flood ran
171 if _, ok := c.Key("a"); !ok {
172 t.Fatal("a known key was refused")
173 }
174}
175
176// waitIdle waits until the source has been asked n times and no fetch is in flight.
177func (c *JWKSCache) waitIdle(t *testing.T, src *jwksSource, n int64) {
178 t.Helper()
179 eventually(t, "the background refresh to end", 3*time.Second, func() bool {
180 c.mu.Lock()
181 defer c.mu.Unlock()
182 return src.calls.Load() >= n && c.flight == nil
183 })
184}
185
186func TestJWKS_ALookupPastTheTTLStampsItsRefreshBeforeItReturns(t *testing.T) {
187 src := newJWKSSource("a")
188 clk := newFakeClock()
189 c := newTestCache(src, clk)
190 if _, ok := c.Key("a"); !ok {
191 t.Fatal("start")
192 }
193 clk.Advance(6 * time.Minute) // past the 5 minute TTL, still vouched for
194 at := clk.Now()
195 if _, ok := c.Key("a"); !ok {
196 t.Fatal("a set past its TTL but not stale must still be served")
197 }
198 // the gap to the next attempt counts from this lookup, whenever the refresh goroutine happens to run
199 c.mu.Lock()
200 stamped := c.attemptedAt
201 c.mu.Unlock()
202 if !stamped.Equal(at) {
203 t.Fatalf("the attempt is stamped %s, the lookup was at %s", stamped, at)
204 }
205 c.waitIdle(t, src, 2)
206 if _, ok := c.Key("zzz"); ok || src.calls.Load() != 2 {
207 t.Fatalf("an unknown kid inside the gap fetched again (%d fetches)", src.calls.Load())
208 }
209}
210
211func TestJWKS_FailsClosedAfterThirtyMinutesWithoutARefreshAndRecovers(t *testing.T) {
212 if policyData.JWKS.StaleFailClosedS != 1800 {
213 t.Fatalf("the plan says 30 minutes, the policy says %d s", policyData.JWKS.StaleFailClosedS)
214 }
215 src := newJWKSSource("a")
216 clk := newFakeClock()
217 c := newTestCache(src, clk)
218 if _, ok := c.Key("a"); !ok || !c.Healthy() {
219 t.Fatal("start")
220 }
221 src.setFail(errors.New("unreachable"))
222 // the control plane is down: the set keeps serving for the stale window
223 clk.Advance(29 * time.Minute)
224 if _, ok := c.Key("a"); !ok || !c.Healthy() {
225 t.Fatal("a set 29 minutes old must still be trusted")
226 }
227 // that lookup refreshed the set behind itself (and failed): the fake clock moves only once it is over
228 c.waitIdle(t, src, 2)
229 clk.Advance(2 * time.Minute) // 31 minutes without a refresh
230 if _, ok := c.Key("a"); ok {
231 t.Fatal("a set 31 minutes old was trusted: the cache must fail closed")
232 }
233 if c.Healthy() {
234 t.Fatal("healthy while stale")
235 }
236 if age := c.Age(); age < 31*time.Minute {
237 t.Fatalf("age %s", age)
238 }
239 // the control plane is back: the next lookup (after the gap) recovers by itself
240 src.setFail(nil)
241 clk.Advance(6 * time.Second)
242 if _, ok := c.Key("a"); !ok || !c.Healthy() {
243 t.Fatal("the cache did not recover when the control plane came back")
244 }
245}
246
247func TestJWKS_ABadSetIsNeverUsedAndNeverReplacesAGoodOne(t *testing.T) {
248 src := newJWKSSource("good")
249 clk := newFakeClock()
250 c := newTestCache(src, clk)
251 if _, ok := c.Key("good"); !ok {
252 t.Fatal("start")
253 }
254 pub, _, _ := ed25519.GenerateKey(rand.Reader)
255 x := b64Encode(pub)
256 key := func(kid string) string {
257 return fmt.Sprintf(`{"kty":"OKP","crv":"Ed25519","x":%q,"kid":%q,"alg":"EdDSA","use":"sig","key_ops":["verify"]}`, x, kid)
258 }
259 many := make([]string, 17)
260 for i := range many {
261 many[i] = key(fmt.Sprintf("k%d", i))
262 }
263 for name, raw := range map[string]string{
264 "17 keys": `{"keys":[` + strings.Join(many, ",") + `]}`,
265 "a repeated kid": `{"keys":[` + key("dup") + `,` + key("dup") + `]}`,
266 "no keys": `{"keys":[]}`,
267 "a private part": `{"keys":[{"kty":"OKP","crv":"Ed25519","x":"` + x + `","d":"` + x + `","kid":"p","alg":"EdDSA","use":"sig","key_ops":["verify"]}]}`,
268 "not json": `<html>`,
269 "an extra member": `{"keys":[` + key("a") + `],"issuer":"https://elsewhere.example"}`,
270 } {
271 src.mu.Lock()
272 src.raw = []byte(raw)
273 src.mu.Unlock()
274 clk.Advance(6 * time.Second)
275 if err := c.Refresh(context.Background()); err == nil {
276 t.Errorf("%s: accepted", name)
277 }
278 if _, ok := c.Key("good"); !ok {
279 t.Errorf("%s: the good set was lost", name)
280 }
281 if _, ok := c.Key("dup"); ok {
282 t.Errorf("%s: part of a refused set was used", name)
283 }
284 }
285 // sixteen keys is the most
286 sixteen := many[:16]
287 src.mu.Lock()
288 src.raw = []byte(`{"keys":[` + strings.Join(sixteen, ",") + `]}`)
289 src.mu.Unlock()
290 if err := c.Refresh(context.Background()); err != nil {
291 t.Fatalf("16 keys: %v", err)
292 }
293 if _, ok := c.Key("k15"); !ok {
294 t.Fatal("k15")
295 }
296}
297
298func TestJWKS_RunKeepsTheSetFreshAndBacksOffAfterAFailure(t *testing.T) {
299 src := newJWKSSource("a")
300 clk := newFakeClock()
301 var observed []bool
302 var mu sync.Mutex
303 c := NewJWKSCache(JWKSConfig{Fetch: src.fetch, Clock: newClock(clk.Now, clk.AfterFunc), Observe: func(ok bool) {
304 mu.Lock()
305 observed = append(observed, ok)
306 mu.Unlock()
307 }})
308 ctx, cancel := context.WithCancel(context.Background())
309 done := make(chan struct{})
310 go func() { c.Run(ctx); close(done) }()
311 eventually(t, "the first fetch", 2*time.Second, func() bool { return src.calls.Load() == 1 })
312 eventually(t, "the loop sleeping", 2*time.Second, func() bool { return clk.pending() == 1 })
313 clk.Advance(5 * time.Minute) // the TTL, spread by at most 10 percent: not yet at 270 s, surely at 330 s
314 clk.Advance(30 * time.Second)
315 eventually(t, "the periodic refresh", 2*time.Second, func() bool { return src.calls.Load() == 2 })
316 eventually(t, "the loop sleeping again", 2*time.Second, func() bool { return clk.pending() == 1 })
317 src.setFail(errors.New("down"))
318 clk.Advance(6 * time.Minute)
319 eventually(t, "a failed refresh", 2*time.Second, func() bool { return src.calls.Load() == 3 })
320 eventually(t, "the backoff timer", 2*time.Second, func() bool { return clk.pending() == 1 })
321 // after a failure the next try comes after the policy's backoff (1 s, then 2 s), not after another 5 minutes
322 clk.Advance(2 * time.Second)
323 eventually(t, "the retry after the backoff", 2*time.Second, func() bool { return src.calls.Load() == 4 })
324 cancel()
325 select {
326 case <-done:
327 case <-time.After(2 * time.Second):
328 t.Fatal("Run did not end with its context")
329 }
330 mu.Lock()
331 defer mu.Unlock()
332 if len(observed) < 4 || !observed[0] || observed[2] {
333 t.Fatalf("observed outcomes %v", observed)
334 }
335}
336
337func TestJWKS_ThePinnedControlPlaneIsTheOnlyPlaceTheKeysComeFrom(t *testing.T) {
338 var landed atomic.Int64
339 elsewhere := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
340 landed.Add(1)
341 _, _ = w.Write([]byte(`{"keys":[]}`))
342 }))
343 defer elsewhere.Close()
344 redirecting := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
345 http.Redirect(w, r, elsewhere.URL+jwksPath, http.StatusFound)
346 }))
347 defer redirecting.Close()
348 _, priv, _ := ed25519.GenerateKey(rand.Reader)
349 hc, err := NewHTTPControl(HTTPControlConfig{BaseURL: redirecting.URL, NodeID: "t1", KID: "k1", Key: priv, Client: redirecting.Client()})
350 if err != nil {
351 t.Fatal(err)
352 }
353 if _, err := hc.FetchJWKS(context.Background()); err == nil || landed.Load() != 0 {
354 t.Fatalf("the keys were read through a redirect: err=%v landed=%d", err, landed.Load())
355 }
356
357 // an oversized or failing answer is no set
358 big := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
359 _, _ = w.Write([]byte(strings.Repeat("x", maxJWKSBody+10)))
360 }))
361 defer big.Close()
362 hc, _ = NewHTTPControl(HTTPControlConfig{BaseURL: big.URL, NodeID: "t1", KID: "k1", Key: priv, Client: big.Client()})
363 if _, err := hc.FetchJWKS(context.Background()); err == nil {
364 t.Fatal("an oversized document was read")
365 }
366}
367
368func TestJWKS_ARotationMidConnectionWorksAndAStaleSetFailsClosedForNewSessions(t *testing.T) {
369 clk := newFakeClock()
370 e, cp := httpNode(t, clk, nil)
371 d := e.newDevice()
372 conn := e.connectOK(d)
373
374 // The control plane rotates: a new key signs the sessions from now on, the old one stays published for a while.
375 newPub, newPriv, _ := ed25519.GenerateKey(rand.Reader)
376 const newKID = "cp-test-2"
377 cp.mu.Lock()
378 cp.jwks[newKID] = newPub
379 cp.mu.Unlock()
380 clk.Advance(2 * time.Second) // past the cache's minimum gap between two forced reloads
381 rotated := func(dev *testDevice) string {
382 return e.sessionSignedBy(newPriv, dev, func(h, c map[string]any) { h["kid"] = newKID })
383 }
384
385 // the connection that was up before the rotation renews with a session of the new key
386 conn.session = rotated(d)
387 conn.send("RENEW " + conn.session)
388 line := conn.recv()
389 m, err := parseLine(line, dirN2D)
390 if err != nil || m.str("verb") != "CHALLENGE" {
391 t.Fatalf("a session of the rotated-in key was refused at RENEW: %q", line)
392 }
393 conn.send("PROOF " + e.proof(d, conn.session, m.str("nonce"), nil))
394 if line = conn.recv(); !strings.HasPrefix(line, "OK ") {
395 t.Fatalf("renew after a rotation: %q", line)
396 }
397 // and so does a new device
398 d2 := e.newDevice()
399 conn2 := e.connect(d2)
400 if line := conn2.hello(func() string { return rotated(d2) }, func(s, n string) string { return e.proof(d2, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
401 t.Fatalf("hello after a rotation: %q", line)
402 }
403
404 // The control plane goes away. For 29 minutes the node still vouches for the set it has; at 31 it vouches for nothing.
405 cp.setDown(503)
406 clk.Advance(29 * time.Minute)
407 d3 := e.newDevice()
408 e.cp.set(d3.id, &devicePlan{host: d3.host, err: &ControlError{Status: 503, Code: "end_to_end_unavailable"}})
409 if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); line != "ERR control_unavailable" {
410 t.Fatalf("29 minutes of silence, authorize down: %q", line) // the keys still verify; it is authorize that is unavailable
411 }
412 if strings.Contains(e.logs.String(), `"msg":"session_refused"`) {
413 t.Fatal("the session was refused at verification: the set should still have been trusted at 29 minutes")
414 }
415 clk.Advance(2 * time.Minute)
416 if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); line != "ERR control_unavailable" {
417 t.Fatalf("31 minutes of silence: %q", line)
418 }
419 if !strings.Contains(e.logs.String(), `"msg":"session_refused"`) || !strings.Contains(e.logs.String(), `"reason":"unknown_kid"`) {
420 t.Fatal("a session was verified against a set older than the policy's window")
421 }
422 // told to retry later, not told its credentials are wrong
423 if !strings.Contains(e.logs.String(), `"answer":"control_unavailable"`) {
424 t.Fatal("the refusal because of stale keys is not logged as such")
425 }
426 // the connections that were already authenticated are untouched
427 if !e.srv.Online(d.host) || !e.srv.Online(d2.host) {
428 t.Fatal("a stale JWKS closed connections that were already authenticated")
429 }
430 // and a RENEW on them is refused as unavailable too, keeping the connection
431 conn.session = rotated(d)
432 conn.send("RENEW " + conn.session)
433 if line := conn.recv(); line != "ERR control_unavailable" {
434 t.Fatalf("renew on a stale set: %q", line)
435 }
436
437 // back to normal
438 cp.setDown(0)
439 e.cp.set(d3.id, &devicePlan{host: d3.host, validFor: time.Hour})
440 clk.Advance(6 * time.Second)
441 if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
442 t.Fatalf("after the control plane came back: %q", line)
443 }
444}
445
446func TestJWKS_AFloodOfHellosWithRandomKidsCostsTheControlPlaneAtMostOneFetchAndHarmsNoLegitimateDevice(t *testing.T) {
447 clk := newFakeClock()
448 e, cp := httpNode(t, clk, nil)
449 d := e.newDevice()
450 _ = e.connectOK(d) // the cache has its keys
451 before := cp.callCount(jwksPath)
452
453 var wg sync.WaitGroup
454 var wrong atomic.Int64
455 for i := 0; i < 300; i++ {
456 wg.Add(1)
457 go func() {
458 defer wg.Done()
459 attacker := &testDevice{id: randomHex(16), key: d.key}
460 c, err := e.dialNode()
461 if err != nil {
462 return // the node was full: also fine
463 }
464 defer c.Close()
465 session := e.session(attacker, func(h, claims map[string]any) { h["kid"] = "rogue-" + randomHex(6) })
466 _, _ = c.Write([]byte("HELLO 2 " + session + "\n"))
467 _ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
468 line, _ := bufio.NewReader(c).ReadString('\n')
469 if strings.TrimSpace(line) != "ERR auth" {
470 wrong.Add(1)
471 }
472 }()
473 }
474 wg.Wait()
475 if wrong.Load() != 0 {
476 t.Fatalf("%d of the 300 sessions with a made-up kid were not refused with auth", wrong.Load())
477 }
478 if n := cp.callCount(jwksPath) - before; n > 1 {
479 t.Fatalf("300 sessions with random kids made the node fetch the keys %d times", n)
480 }
481 // a legitimate device is served as before
482 d2 := e.newDevice()
483 _ = e.connectOK(d2)
484}