ALL

For reading and verification only. You may not copy, use, modify or redistribute this code. Full notice on the main page.

tunnel/metrics_test.go

SHA-256: 827f6856c5692a88d6698d72ce28fcfe4583aecaa7d9ead562d8cc9062c69586
248 lines · 9061 bytes · View on GitHub

1package main
2
3import (
4 "net/http"
5 "regexp"
6 "strconv"
7 "strings"
8 "testing"
9 "time"
10)
11
12// /metrics and /healthz (metrics.go): loopback, aggregate, no series per device.
13
14func metricsEnv(t *testing.T, mutate func(*Config)) *testEnv {
15 return newTestEnv(t, func(c *Config) {
16 c.MetricsAddr = "127.0.0.1:0"
17 if mutate != nil {
18 mutate(c)
19 }
20 })
21}
22
23func TestMetrics_ServedOnLoopbackToGETAndHEADOnly(t *testing.T) {
24 e := metricsEnv(t, nil)
25 addr := e.srv.MetricsAddr()
26 if !strings.HasPrefix(addr, "127.0.0.1:") {
27 t.Fatalf("listening on %q", addr)
28 }
29 code, body, header := httpGet(t, "http://"+addr+"/metrics")
30 if code != 200 || !strings.HasPrefix(header.Get("content-type"), "text/plain") || !strings.Contains(body, "# TYPE all_tunnel_streams_open gauge") {
31 t.Fatalf("%d %v\n%s", code, header, body)
32 }
33 if code, _, _ := httpGet(t, "http://"+addr+"/nothing"); code != 404 {
34 t.Fatalf("an unknown path: %d", code)
35 }
36 for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
37 req, _ := http.NewRequest(method, "http://"+addr+"/metrics", strings.NewReader("x"))
38 resp, err := http.DefaultClient.Do(req)
39 if err != nil {
40 t.Fatal(err)
41 }
42 resp.Body.Close()
43 if resp.StatusCode != http.StatusMethodNotAllowed {
44 t.Fatalf("%s: %d", method, resp.StatusCode)
45 }
46 }
47 req, _ := http.NewRequest(http.MethodHead, "http://"+addr+"/healthz", nil)
48 resp, err := http.DefaultClient.Do(req)
49 if err != nil || resp.StatusCode != 200 {
50 t.Fatalf("HEAD /healthz: %v %v", resp, err)
51 }
52 resp.Body.Close()
53 // without the option there is no listener at all
54 if off := newTestEnv(t, nil); off.srv.MetricsAddr() != "" {
55 t.Fatalf("a node without -metrics-addr listens on %q", off.srv.MetricsAddr())
56 }
57}
58
59func TestMetrics_HealthzSaysWhetherTheNodeServes(t *testing.T) {
60 e := metricsEnv(t, nil)
61 url := "http://" + e.srv.MetricsAddr() + "/healthz"
62 if code, body, _ := httpGet(t, url); code != 200 || !strings.Contains(body, `"state":"ok"`) {
63 t.Fatalf("%d %s", code, body)
64 }
65 e.srv.setKilled(true)
66 if code, body, _ := httpGet(t, url); code != 503 || !strings.Contains(body, `"state":"kill_switch"`) {
67 t.Fatalf("%d %s", code, body)
68 }
69 e.srv.setKilled(false)
70 d := e.newDevice()
71 dc := e.connectOK(d)
72 release := make(chan struct{})
73 defer close(release)
74 dc.serve(holdHandler(d, release))
75 if _, err := e.clientTLS(d); err != nil { // a stream that stays open: the drain has something to wait for
76 t.Fatal(err)
77 }
78 go e.srv.Drain(10 * time.Second)
79 eventually(t, "draining", 3*time.Second, func() bool { return e.srv.Draining() })
80 if code, body, _ := httpGet(t, url); code != 503 || !strings.Contains(body, `"state":"draining"`) {
81 t.Fatalf("%d %s", code, body)
82 }
83 if v := e.metricValue("all_tunnel_draining"); v != 1 {
84 t.Fatalf("draining gauge %d", v)
85 }
86}
87
88// seriesOf is the set of series of a scrape without their values: name and label.
89func seriesOf(text string) map[string]bool {
90 out := map[string]bool{}
91 for _, line := range strings.Split(text, "\n") {
92 if line == "" || strings.HasPrefix(line, "#") {
93 continue
94 }
95 if i := strings.LastIndexByte(line, ' '); i > 0 {
96 out[line[:i]] = true
97 }
98 }
99 return out
100}
101
102func TestMetrics_ThereIsNoSeriesPerLabelDeviceOrAddress(t *testing.T) {
103 scrape := func(devices int) string {
104 // (the devices of the test all connect from loopback, which is one source: its rate is out of the way)
105 e := newVIPEnv(t, func(e *testEnv, c *Config) { c.PerIPNewPerS, c.PerIPConcurrent = 1<<20, 1<<20 })
106 var held []*devConn
107 for i := 0; i < devices; i++ {
108 d := e.newDevice()
109 dc := e.connectOK(d)
110 dc.serve(nil)
111 held = append(held, dc)
112 c, err := e.clientFrom(randomIP(i), d)
113 if err != nil {
114 t.Fatalf("%v\n%s", err, e.logs.String())
115 }
116 if _, err := postOver(c, d.host, []byte("x")); err != nil {
117 t.Fatalf("%v\n%s", err, e.logs.String())
118 }
119 e.helloFrom(randomIP(i+500), "nobody"+d.label[6:]+".devices.all-e2e.test", time.Second) // a name that is not a device
120 dc.send("HINT " + randomIP(i+900))
121 }
122 text := e.srv.renderMetrics()
123 if devices > 0 {
124 if got := e.metricValue("all_tunnel_devices_online"); got != int64(devices) {
125 t.Fatalf("online %d", got)
126 }
127 }
128 _ = held
129 return text
130 }
131 one, many := scrape(1), scrape(25)
132 a, b := seriesOf(one), seriesOf(many)
133 if len(a) != len(b) {
134 t.Fatalf("%d series with one device, %d with 25", len(a), len(b))
135 }
136 for s := range b {
137 if !a[s] {
138 t.Errorf("series %s exists only with more devices", s)
139 }
140 }
141 for name, re := range map[string]*regexp.Regexp{
142 "a label": regexp.MustCompile(`[a-z2-7]{20}`),
143 "a zone": regexp.MustCompile(`devices\.all-e2e|\.test`),
144 "an address": regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b|[0-9a-f]*:[0-9a-f]*:[0-9a-f:]+`),
145 "a device id": regexp.MustCompile(`[0-9a-f]{32}`),
146 "a stream id": regexp.MustCompile(`stream`),
147 } {
148 if name == "a stream id" {
149 continue // the word "stream" is in metric names; stream ids are the 32-hex check above
150 }
151 if m := re.FindString(many); m != "" {
152 t.Errorf("the metrics hold %s: %q", name, m)
153 }
154 }
155 // every label key of every series is one of a fixed few
156 keys := regexp.MustCompile(`\{([a-z_]+)=`)
157 allowed := map[string]bool{"reason": true, "direction": true, "result": true, "type": true, "group": true, "version": true, "cipher": true}
158 for s := range b {
159 if m := keys.FindStringSubmatch(s); m != nil && !allowed[m[1]] && !strings.HasPrefix(s, "all_tunnel_build_info") {
160 t.Errorf("series %s has a label key that is not in the closed list", s)
161 }
162 }
163}
164
165func randomIP(i int) string { return "198.51." + itoa(100+i/250) + "." + itoa(1+i%250) }
166
167func itoa(n int) string { return strconv.Itoa(n) }
168
169func TestMetrics_TheNodeCertificateAndTheKeysAreReported(t *testing.T) {
170 end := time.Now().Add(30 * 24 * time.Hour).Truncate(time.Second)
171 e, _ := httpNode(t, nil, func(c *Config) { c.CertNotAfter = func() time.Time { return end } })
172 if got := e.metricValue("all_tunnel_node_cert_not_after_seconds"); got != end.Unix() {
173 t.Fatalf("not_after %d, want %d", got, end.Unix())
174 }
175 left := e.metricValue("all_tunnel_node_cert_seconds_left")
176 if left < 29*24*3600 || left > 30*24*3600 {
177 t.Fatalf("seconds left %d", left)
178 }
179 eventually(t, "the keys were fetched", 5*time.Second, func() bool { return e.metricValue("all_tunnel_jwks_healthy") == 1 })
180 if age := e.metricValue("all_tunnel_jwks_age_seconds"); age < 0 || age > 60 {
181 t.Fatalf("age %d", age)
182 }
183 // without a certificate source of the kind that reports, the series is simply absent (a test node's fake keys report no age either)
184 plain := newTestEnv(t, nil)
185 if strings.Contains(plain.srv.renderMetrics(), "node_cert_not_after") || strings.Contains(plain.srv.renderMetrics(), "jwks_age") {
186 t.Fatal("series reported without a source")
187 }
188}
189
190func TestMetrics_CountersFollowWhatHappens(t *testing.T) {
191 e := metricsEnv(t, nil)
192 d := e.newDevice()
193 dc := e.connectOK(d)
194 dc.serve(nil)
195 c, err := e.clientTLS(d)
196 if err != nil {
197 t.Fatal(err)
198 }
199 if _, err := postOver(c, d.host, make([]byte, 5000)); err != nil {
200 t.Fatal(err)
201 }
202 c.Close()
203 e.waitIdle()
204 if e.metricValue("all_tunnel_streams_opened_total") != 1 || e.metricValue(`all_tunnel_authorize_total{result="ok"}`) != 1 {
205 t.Fatalf("\n%s", e.srv.renderMetrics())
206 }
207 if e.metricValue(`all_tunnel_bytes_total{direction="up"}`) < 5000 || e.metricValue(`all_tunnel_bytes_total{direction="down"}`) < 5000 {
208 t.Fatalf("bytes\n%s", e.srv.renderMetrics())
209 }
210 if e.metricValue("all_tunnel_control_connections") != 1 || e.metricValue("all_tunnel_devices_online") != 1 {
211 t.Fatalf("\n%s", e.srv.renderMetrics())
212 }
213 e.plan(d).err = &ControlError{Status: 402, Code: codeLicenseInactive}
214 other := e.newDeviceNoPlan()
215 oc := e.connect(other)
216 oc.hello(func() string { return e.session(other, nil) }, func(s, n string) string { return e.proof(other, s, n, nil) })
217 if e.metricValue(`all_tunnel_authorize_total{result="refused"}`) != 1 {
218 t.Fatalf("\n%s", e.srv.renderMetrics())
219 }
220 // something that is not TLS, and a name that is nobody's
221 e.helloFromLoopback("nobody.devices.all-e2e.test")
222 if e.rejected("unknown_host") != 1 {
223 t.Fatalf("\n%s", e.srv.renderMetrics())
224 }
225}
226
227func (e *testEnv) helloFromLoopback(host string) {
228 e.t.Helper()
229 c := sendRaw(e.t, e, helloOf(host, 0, 16000), 0)
230 buf := make([]byte, 16)
231 _ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
232 _, _ = c.Read(buf)
233}
234
235func TestMetrics_TheTestNodeProcessServesThemAndLogsNameNobody(t *testing.T) {
236 p := launchTestNode(t, 3600, "-metrics-addr", "127.0.0.1:0")
237 addr, _ := p.ready["metrics"].(string)
238 if addr == "" {
239 t.Fatalf("the ready line has no metrics address: %v", p.ready)
240 }
241 code, body, _ := httpGet(t, "http://"+addr+"/metrics")
242 if code != 200 || !strings.Contains(body, "all_tunnel_node_cert_not_after_seconds") || !strings.Contains(body, `all_tunnel_build_info{version=`) {
243 t.Fatalf("%d\n%s", code, body)
244 }
245 if code, body, _ := httpGet(t, "http://"+addr+"/healthz"); code != 200 || !strings.Contains(body, `"ok":true`) {
246 t.Fatalf("%d %s", code, body)
247 }
248}