For reading and verification only. You may not copy, use, modify or redistribute this code. Full notice on the main page.
tunnel/drain.go
| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "time" |
| 5 | ) |
| 6 | |
| 7 | // Draining (the design plan, section 5): what a node does on SIGTERM, so that a deploy or a restart costs the people on it as little as |
| 8 | // possible. In this order: |
| 9 | // |
| 10 | // 1. It stops taking anything new. A client that connects now gets a TLS alert and no OPEN (the DNS no longer points at a node being taken down, but |
| 11 | // a client that resolved it a minute ago still arrives); a device that sends a HELLO gets ERR control_unavailable and goes to the other node. |
| 12 | // The listeners stay open on purpose: a device's ATTACH for a stream that already has its OPEN must still be served, and closing the port would |
| 13 | // turn a drain into a reset for every client that is a handshake away from its stream. |
| 14 | // 2. It tells every device it serves DRAIN <quiet_s>: prefer the other node for new connections; the open streams continue here for the quiet period. |
| 15 | // 3. It waits until the streams are gone or the quiet period (policy timing.drain_quiet_s, 20 minutes) is over, whichever comes first, then |
| 16 | // closes what is left and exits. A call of 900 s that started before SIGTERM finishes; one that would run past the quiet period is cut, and the |
| 17 | // runbook says so (it may already have run on the computer). |
| 18 | // |
| 19 | // The control connections stay up for the whole drain (a RENEW is answered), and the feed keeps being applied: a revoke during a drain still cuts. |
| 20 | |
| 21 | // drainPoll is how often a drain looks at whether the streams are gone. |
| 22 | const drainPoll = 250 * time.Millisecond |
| 23 | |
| 24 | // Draining reports whether the node is draining. |
| 25 | func (s *Server) Draining() bool { return s.draining.Load() } |
| 26 | |
| 27 | // Drain runs a drain to its end (the streams gone, or quiet over) and shuts the node down. It returns when the node is down. A second call is a no-op. |
| 28 | func (s *Server) Drain(quiet time.Duration) { |
| 29 | s.drainFor.CompareAndSwap(0, int64(quiet)) |
| 30 | if !s.draining.CompareAndSwap(false, true) { |
| 31 | return |
| 32 | } |
| 33 | s.log.Info("draining", "quiet_s", int64(quiet/time.Second), "streams", s.streams.Load()) |
| 34 | line := drainLine(quiet) |
| 35 | for _, cc := range s.routes.controls("") { |
| 36 | if cc.isReady() { |
| 37 | _ = cc.write(line) // a device that cannot be told is no worse off: it learns when the connection closes |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | over := make(chan struct{}) |
| 42 | stop := s.clk.AfterFunc(quiet, func() { close(over) }) |
| 43 | defer stop() |
| 44 | for s.streams.Load() > 0 || s.pendingCount() > 0 { |
| 45 | select { |
| 46 | case <-over: |
| 47 | s.log.Warn("drain_quiet_over", "streams", s.streams.Load()) |
| 48 | s.shutdown(0, closeDrain) |
| 49 | return |
| 50 | case <-s.ctx.Done(): |
| 51 | return |
| 52 | default: |
| 53 | } |
| 54 | time.Sleep(drainPoll) |
| 55 | } |
| 56 | s.log.Info("drained") |
| 57 | s.shutdown(s.cfg.ShutdownGrace, closeDrain) |
| 58 | } |
| 59 | |
| 60 | func (s *Server) pendingCount() int { |
| 61 | s.mu.Lock() |
| 62 | defer s.mu.Unlock() |
| 63 | return len(s.pending) |
| 64 | } |